From 2cb6fc5ffed66ef6a0dfb48b71c6ca084dae6458 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Apr 2025 10:25:41 +0200 Subject: [PATCH 1/6] switch using runsOn runners --- .github/workflows/create-lint-wf.yml | 12 +++------- .../create-test-lint-wf-template.yml | 23 ++++--------------- .github/workflows/create-test-wf.yml | 15 ++++-------- .github/workflows/pytest.yml | 18 +++++---------- .github/workflows/sync.yml | 12 +++------- 5 files changed, 20 insertions(+), 60 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 997ad084a9..a0d0bf820a 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -15,14 +15,6 @@ on: release: types: [published] workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" # Cancel if a newer run is started concurrency: @@ -31,7 +23,9 @@ concurrency: jobs: MakeTestWorkflow: - runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} + runs-on: + - runs-on=${{ github.run_id }}-make-test-worfklow + - runner=4cpu-linux-x64 env: NXF_ANSI_LOG: false diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 2c804196fb..54ca1ae81a 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -14,14 +14,6 @@ on: release: types: [published] workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" # Cancel if a newer run is started concurrency: @@ -49,7 +41,9 @@ jobs: echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/template_features.yml)" >> $GITHUB_OUTPUT RunTestWorkflow: - runs-on: ${{ matrix.runner }} + runs-on: + - runs-on=${{ github.run_id }}-run-test-worfklow + - runner=4cpu-linux-x64 needs: prepare-matrix env: NXF_ANSI_LOG: false @@ -57,20 +51,11 @@ jobs: strategy: matrix: TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }} - runner: - # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default - - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} - profile: ["self_hosted_runner"] include: - TEMPLATE: all - runner: ubuntu-latest profile: "docker" - TEMPLATE: nf_core_configs - runner: ubuntu-latest profile: "docker" - exclude: - - TEMPLATE: nf_core_configs - profile: "self_hosted_runner" fail-fast: false steps: @@ -119,7 +104,7 @@ jobs: - name: run the pipeline run: | cd create-test-lint-wf - nextflow run my-prefix-testpipeline -profile test,${{matrix.profile}} --outdir ./results + nextflow run my-prefix-testpipeline -profile test,docker --outdir ./results # Remove results folder before linting - name: remove results folder diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 1166bc0107..171c6d5565 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -15,14 +15,6 @@ on: release: types: [published] workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" # Cancel if a newer run is started concurrency: @@ -31,8 +23,9 @@ concurrency: jobs: RunTestWorkflow: - # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default - runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} + runs-on: + - runs-on=${{ github.run_id }}-run-test-worfklow + - runner=4cpu-linux-x64 env: NXF_ANSI_LOG: false @@ -71,7 +64,7 @@ jobs: mkdir create-test-wf && cd create-test-wf export NXF_WORK=$(pwd) nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nextflow run nf-core-testpipeline -profile self_hosted_runner,test --outdir ./results + nextflow run nf-core-testpipeline -profile docker,test --outdir ./results - name: Upload log file artifact if: ${{ always() }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 6b46c99762..a36680d32f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,14 +20,6 @@ on: release: types: [published] workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" # Cancel if a newer run with the same workflow name is queued concurrency: @@ -77,8 +69,9 @@ jobs: name: Run ${{matrix.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} needs: [setup, list_tests] if: ${{ needs.setup.outputs.run-tests }} - # run on self-hosted runners for test_components_generate_snapshot.py (because of the gitlab branch), based on the input if it is dispatched manually, on github if it is a rerun or on self-hosted by default - runs-on: ${{ matrix.test == 'components/test_components_generate_snapshot.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }} + runs-on: + - runs-on=${{ github.run_id }}-run-test + - runner=4cpu-linux-x64 strategy: matrix: ${{ fromJson(needs.list_tests.outputs.tests) }} fail-fast: false # run all tests even if one fails @@ -166,8 +159,9 @@ jobs: coverage: needs: test - # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default - runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} + runs-on: + - runs-on=${{ github.run_id }}-coverage + - runner=2cpu-linux-x64 steps: - name: go to subdirectory run: | diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 83cf6f061f..a870814830 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -8,13 +8,6 @@ on: type: boolean description: Only run on nf-core/testpipeline? required: true - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" force_pr: description: "Force a PR to be created" type: boolean @@ -58,8 +51,9 @@ jobs: sync: needs: get-pipelines - # use the github runner on release otherwise use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default - runs-on: ${{github.event_name == 'release' && 'self-hosted' || github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} + runs-on: + - runs-on=${{ github.run_id }}-sync + - runner=4cpu-linux-x64 strategy: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false From 6dedc4d19e28aed5de0bbd4cfcbec98eef0618ae Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 15 Apr 2025 08:29:03 +0000 Subject: [PATCH 2/6] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2c92c6526..688fad70c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ - Update pre-commit hook astral-sh/ruff-pre-commit to v0.11.2 ([#3521](https://github.com/nf-core/tools/pull/3521)) - Fix indentation in included_configs API docs ([#3523](https://github.com/nf-core/tools/pull/3523)) - chore(deps): update python:3.12-slim docker digest to a866731 ([#3527](https://github.com/nf-core/tools/pull/3527)) +- switch to using runsOn runners in nf-core/tools repo ([#3537](https://github.com/nf-core/tools/pull/3537)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] From 6d636c6e0779e31b1ebff33a7e51f7dc8206be17 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Apr 2025 15:59:15 +0200 Subject: [PATCH 3/6] try different gitlab branch --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 5ecd0b3f55..bba3bc09ed 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -32,7 +32,7 @@ GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca" GITLAB_BRANCH_TEST_NEW_SHA = "7d73e21f30041297ea44367f2b4fd4e045c0b991" -GITLAB_NFTEST_BRANCH = "nf-test-tests-self-hosted-runners" +GITLAB_NFTEST_BRANCH = "nf-test-tests" def with_temporary_folder(func: Callable[..., Any]) -> Callable[..., Any]: From 5eb0b5b48700bf0db2c5625e7c37cf4e4ff0bd98 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Apr 2025 16:16:46 +0200 Subject: [PATCH 4/6] Revert "try different gitlab branch" This reverts commit 6d636c6e0779e31b1ebff33a7e51f7dc8206be17. --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index bba3bc09ed..5ecd0b3f55 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -32,7 +32,7 @@ GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca" GITLAB_BRANCH_TEST_NEW_SHA = "7d73e21f30041297ea44367f2b4fd4e045c0b991" -GITLAB_NFTEST_BRANCH = "nf-test-tests" +GITLAB_NFTEST_BRANCH = "nf-test-tests-self-hosted-runners" def with_temporary_folder(func: Callable[..., Any]) -> Callable[..., Any]: From e2741384de36ab3a953e493c442af07586743d5e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Apr 2025 16:27:30 +0200 Subject: [PATCH 5/6] rename it test branch one more time --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 5ecd0b3f55..bba3bc09ed 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -32,7 +32,7 @@ GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca" GITLAB_BRANCH_TEST_NEW_SHA = "7d73e21f30041297ea44367f2b4fd4e045c0b991" -GITLAB_NFTEST_BRANCH = "nf-test-tests-self-hosted-runners" +GITLAB_NFTEST_BRANCH = "nf-test-tests" def with_temporary_folder(func: Callable[..., Any]) -> Callable[..., Any]: From d61403b97dbd0884f617c944b117872a17ea6715 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 15 Apr 2025 17:00:05 +0200 Subject: [PATCH 6/6] try removing some extra steps --- .github/workflows/create-test-lint-wf-template.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 54ca1ae81a..64788f7829 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -53,9 +53,6 @@ jobs: TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }} include: - TEMPLATE: all - profile: "docker" - - TEMPLATE: nf_core_configs - profile: "docker" fail-fast: false steps: