nf-core · mashehu · Jun 30, 2025 · Jun 23, 2025 · Jun 23, 2025 · Jun 30, 2025
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -115,8 +115,9 @@ jobs:
           mv .github/.coveragerc .
 
       - name: Test with pytest
+        id: pytest
         run: |
-          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --cov-config=.coveragerc --durations=0 && exit_code=0|| exit_code=$?
+          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --cov-config=.coveragerc --durations=0 -n auto && exit_code=0|| exit_code=$?
           # don't fail if no tests were collected, e.g. for test_licence.py
           if [ "${exit_code}" -eq 5 ]; then
             echo "No tests were collected"
@@ -133,7 +134,7 @@ jobs:
 
       - name: Store snapshot report
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        if: always()
+        if: always() && contains(matrix.test, 'test_create_app') && steps.pytest.outcome == 'failure'
         with:
           include-hidden-files: true
           name: Snapshot Report ${{ env.test }}
@@ -152,39 +153,35 @@ jobs:
       - runs-on=${{ github.run_id }}-coverage
       - runner=2cpu-linux-x64
     steps:
-      - name: go to subdirectory
-        run: |
-          mkdir -p pytest
-          cd pytest
-
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
       - name: Set up Python 3.13
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        env:
-          AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/
         with:
           python-version: "3.13"
           cache: "pip"
 
-      - name: Install dependencies
+      - name: Install coverage
         run: |
-          python -m pip install --upgrade pip -r requirements-dev.txt
-          pip install -e .
+          python -m pip install --upgrade pip coverage
 
       - name: move coveragerc file up
         run: |
           mv .github/.coveragerc .
 
       - name: Download all artifacts
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          pattern: coverage_*
+
       - name: Run coverage
         run: |
-          coverage combine --keep coverage*/.coverage*
+          coverage combine --keep coverage_*/.coverage*
           coverage report
           coverage xml
 
       - uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5
         with:
           files: coverage.xml
+          disable_search: true # we already know the file to upload
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/update-textual-snapshots.yml b/.github/workflows/update-textual-snapshots.yml
@@ -46,7 +46,7 @@ jobs:
       - name: Run pytest to update snapshots
         id: pytest
         run: |
-          python3 -m pytest tests/pipelines/test_create_app.py --snapshot-update --color=yes  --durations=0
+          python3 -m pytest tests/pipelines/test_create_app.py --snapshot-update --color=yes --durations=0 -n auto
         continue-on-error: true
 
       # indication that the run has finished

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -36,6 +36,7 @@
 - Update zentered/bluesky-post-action action to v0.3.0 ([#3626](https://github.com/nf-core/tools/pull/3626))
 - Update dependency textual to v3.5.0 ([#3636](https://github.com/nf-core/tools/pull/3636))
 - Make changelog bot push to correct remote ([#3638](https://github.com/nf-core/tools/pull/3638))
+- Parallelize pytest runs and speed up coverage step ([#3635](https://github.com/nf-core/tools/pull/3635))
 
 ## [v3.3.1 - Tungsten Tamarin Patch](https://github.com/nf-core/tools/releases/tag/3.3.1) - [2025-06-02]
 

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -18,5 +18,6 @@ typing_extensions >=4.0.0
 pytest-asyncio
 pytest-textual-snapshot==1.1.0
 pytest-workflow>=2.0.0
+pytest-xdist>=3.7.0
 pytest>=8.0.0
 ruff
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -0,0 +1,53 @@
+"""Global pytest configuration for nf-core tests setting up worker-specific cache directories to avoid git lock issues."""
+
+import os
+import shutil
+import tempfile
+
+
+def pytest_configure(config):
+    """Configure pytest before any tests run - set up worker-specific cache directories."""
+    # Get worker ID for pytest-xdist, or 'main' if not using xdist
+    worker_id = getattr(config, "workerinput", {}).get("workerid", "main")
+
+    # Create temporary directories for this worker
+    cache_base = tempfile.mkdtemp(prefix=f"nfcore_cache_{worker_id}_")
+    config_base = tempfile.mkdtemp(prefix=f"nfcore_config_{worker_id}_")
+
+    # Store original values for later restoration
+    config._original_xdg_cache = os.environ.get("XDG_CACHE_HOME")
+    config._original_xdg_config = os.environ.get("XDG_CONFIG_HOME")
+    config._temp_cache_dir = cache_base
+    config._temp_config_dir = config_base
+
+    # Set environment variables to use worker-specific directories
+    os.environ["XDG_CACHE_HOME"] = cache_base
+    os.environ["XDG_CONFIG_HOME"] = config_base
+
+
+def pytest_unconfigure(config):
+    """Clean up after all tests are done."""
+    # Restore original environment variables
+    if hasattr(config, "_original_xdg_cache"):
+        if config._original_xdg_cache is not None:
+            os.environ["XDG_CACHE_HOME"] = config._original_xdg_cache
+        else:
+            os.environ.pop("XDG_CACHE_HOME", None)
+
+    if hasattr(config, "_original_xdg_config"):
+        if config._original_xdg_config is not None:
+            os.environ["XDG_CONFIG_HOME"] = config._original_xdg_config
+        else:
+            os.environ.pop("XDG_CONFIG_HOME", None)
+
+    # Clean up temporary directories
+    if hasattr(config, "_temp_cache_dir"):
+        try:
+            shutil.rmtree(config._temp_cache_dir)
+        except (OSError, FileNotFoundError):
+            pass
+    if hasattr(config, "_temp_config_dir"):
+        try:
+            shutil.rmtree(config._temp_config_dir)
+        except (OSError, FileNotFoundError):
+            pass