From 6d424b3ddb72454d9b09813d2bd812ef1f7cbc16 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Tue, 9 Sep 2025 08:40:02 +0000 Subject: [PATCH 1/6] impr: run nextflow inspect in tempdir --- nf_core/pipelines/download/download.py | 30 +++++++++++++++----------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/nf_core/pipelines/download/download.py b/nf_core/pipelines/download/download.py index 212c05a2e4..73f70dbfde 100644 --- a/nf_core/pipelines/download/download.py +++ b/nf_core/pipelines/download/download.py @@ -7,6 +7,7 @@ import re import shutil import tarfile +import tempfile from datetime import datetime from pathlib import Path from typing import Any, Literal, Optional, Union @@ -650,19 +651,22 @@ def find_container_images( profile_str += ",test,test_full" profile = f"-profile {profile_str}" if self.container_system else "" - # Run nextflow inspect - executable = "nextflow" - cmd_params = f"inspect -format json {profile} {workflow_directory / entrypoint}" - cmd_out = run_cmd(executable, cmd_params) - if cmd_out is None: - raise DownloadError("Failed to run `nextflow inspect`. Please check your Nextflow installation.") - - out, _ = cmd_out - out_json = json.loads(out) - # NOTE: Should we save the container name too to have more meta information? - named_containers = {proc["name"]: proc["container"] for proc in out_json["processes"]} - # We only want to process unique containers - self.containers = list(set(named_containers.values())) + with tempfile.TemporaryDirectory(dir=".", delete=True) as temp: + os.chdir(temp) + # Run nextflow inspect + executable = "nextflow" + cmd_params = f"inspect -format json {profile} {Path('..') / workflow_directory / entrypoint}" + cmd_out = run_cmd(executable, cmd_params) + if cmd_out is None: + raise DownloadError("Failed to run `nextflow inspect`. Please check your Nextflow installation.") + + out, _ = cmd_out + out_json = json.loads(out) + # NOTE: Should we save the container name too to have more meta information? + named_containers = {proc["name"]: proc["container"] for proc in out_json["processes"]} + # We only want to process unique containers + self.containers = list(set(named_containers.values())) + os.chdir("..") except RuntimeError as e: log.error("Running 'nextflow inspect' failed with the following error") From 89e628d6679ad5b1a583f20c0969f53614089926 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Tue, 9 Sep 2025 08:58:52 +0000 Subject: [PATCH 2/6] impr: Add contextmanager for changing into temp direcotry and handling exceptions --- nf_core/pipelines/download/download.py | 10 ++++------ nf_core/pipelines/download/utils.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/nf_core/pipelines/download/download.py b/nf_core/pipelines/download/download.py index 73f70dbfde..0a39c820b4 100644 --- a/nf_core/pipelines/download/download.py +++ b/nf_core/pipelines/download/download.py @@ -7,7 +7,6 @@ import re import shutil import tarfile -import tempfile from datetime import datetime from pathlib import Path from typing import Any, Literal, Optional, Union @@ -23,7 +22,7 @@ from nf_core.pipelines.download.container_fetcher import ContainerFetcher from nf_core.pipelines.download.docker import DockerFetcher from nf_core.pipelines.download.singularity import SINGULARITY_CACHE_DIR_ENV_VAR, SingularityFetcher -from nf_core.pipelines.download.utils import DownloadError +from nf_core.pipelines.download.utils import DownloadError, intermediate_dir_with_cd from nf_core.pipelines.download.workflow_repo import WorkflowRepo from nf_core.utils import ( NF_INSPECT_MIN_NF_VERSION, @@ -651,11 +650,11 @@ def find_container_images( profile_str += ",test,test_full" profile = f"-profile {profile_str}" if self.container_system else "" - with tempfile.TemporaryDirectory(dir=".", delete=True) as temp: - os.chdir(temp) + working_dir = Path().absolute() + with intermediate_dir_with_cd(working_dir): # Run nextflow inspect executable = "nextflow" - cmd_params = f"inspect -format json {profile} {Path('..') / workflow_directory / entrypoint}" + cmd_params = f"inspect -format json {profile} {working_dir / workflow_directory / entrypoint}" cmd_out = run_cmd(executable, cmd_params) if cmd_out is None: raise DownloadError("Failed to run `nextflow inspect`. Please check your Nextflow installation.") @@ -666,7 +665,6 @@ def find_container_images( named_containers = {proc["name"]: proc["container"] for proc in out_json["processes"]} # We only want to process unique containers self.containers = list(set(named_containers.values())) - os.chdir("..") except RuntimeError as e: log.error("Running 'nextflow inspect' failed with the following error") diff --git a/nf_core/pipelines/download/utils.py b/nf_core/pipelines/download/utils.py index c29abde806..fca8cdbaa1 100644 --- a/nf_core/pipelines/download/utils.py +++ b/nf_core/pipelines/download/utils.py @@ -1,6 +1,7 @@ import contextlib import importlib.resources import logging +import os import shutil import tempfile from collections.abc import Generator @@ -81,3 +82,24 @@ def intermediate_file_no_creation(output_path: Path) -> Generator[Path, None, No except: tmp.cleanup() raise + + +@contextlib.contextmanager +def intermediate_dir_with_cd(original_dir: Path, base_dir: Path = Path(".")): + """ + Context manager to provide and change into a tempdir and ensure its removal and retrun to the + original_dir upon exceptions. + """ + + if not original_dir.is_dir(): + raise DownloadError(f"Unable to setup temporary dir, original_dir does not exist: {original_dir}") + + tmp = tempfile.TemporaryDirectory(dir=base_dir) + try: + os.chdir(tmp.name) + yield tmp + os.chdir(original_dir) + tmp.cleanup() + except: # noqa: E722 + os.chdir(original_dir) + tmp.cleanup() From 403fc2cc2ce8a28924845799cf2efd6d3f01e20a Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Tue, 9 Sep 2025 11:15:18 +0200 Subject: [PATCH 3/6] Add changes --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3480966630..868d6b3313 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ - Fix backwards compatibility with python 3.9 in use of Enum ([#3736](https://github.com/nf-core/tools/pull/3736)) - Fix downloads: temporary files not moved and cleaned up correctly after singularity pull ([#3749](https://github.com/nf-core/tools/pull/3749)) - impr devcontainer: Add hostRequirements to run with 4CPUs and 16GB ram by default ([#3746](https://github.com/nf-core/tools/pull/3746)) +- Fix Issues/3729: Remove temporary folders created from nextflow inspect during downloads ([#3750](https://github.com/nf-core/tools/pull/3750)) ## [v3.3.2 - Tungsten Tamarin Patch 2](https://github.com/nf-core/tools/releases/tag/3.3.2) - [2025-07-08] From 573e8796bd702ca9c334650d15974611abc97459 Mon Sep 17 00:00:00 2001 From: JulianFlesch Date: Tue, 7 Oct 2025 16:02:56 +0200 Subject: [PATCH 4/6] Update nf_core/pipelines/download/utils.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipelines/download/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/download/utils.py b/nf_core/pipelines/download/utils.py index 7fa33ad3fb..2c9e1c29dc 100644 --- a/nf_core/pipelines/download/utils.py +++ b/nf_core/pipelines/download/utils.py @@ -87,7 +87,7 @@ def intermediate_file_no_creation(output_path: Path) -> Generator[Path, None, No @contextlib.contextmanager def intermediate_dir_with_cd(original_dir: Path, base_dir: Path = Path(".")): """ - Context manager to provide and change into a tempdir and ensure its removal and retrun to the + Context manager to provide and change into a tempdir and ensure its removal and return to the original_dir upon exceptions. """ From 6886d3694bd00b8c19e865d70ec237e3d27bd743 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Tue, 7 Oct 2025 19:05:50 +0200 Subject: [PATCH 5/6] Remove explicit tempdir cleanup --- nf_core/pipelines/download/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/pipelines/download/utils.py b/nf_core/pipelines/download/utils.py index 2c9e1c29dc..977ffa0c42 100644 --- a/nf_core/pipelines/download/utils.py +++ b/nf_core/pipelines/download/utils.py @@ -99,7 +99,5 @@ def intermediate_dir_with_cd(original_dir: Path, base_dir: Path = Path(".")): os.chdir(tmp.name) yield tmp os.chdir(original_dir) - tmp.cleanup() except: # noqa: E722 os.chdir(original_dir) - tmp.cleanup() From e397d50ec21d7fa28ff0aa577dccc37c6d331a80 Mon Sep 17 00:00:00 2001 From: Julian Flesch Date: Tue, 7 Oct 2025 19:10:59 +0200 Subject: [PATCH 6/6] add finally to simplify --- nf_core/pipelines/download/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/pipelines/download/utils.py b/nf_core/pipelines/download/utils.py index 977ffa0c42..17c3426d64 100644 --- a/nf_core/pipelines/download/utils.py +++ b/nf_core/pipelines/download/utils.py @@ -98,6 +98,5 @@ def intermediate_dir_with_cd(original_dir: Path, base_dir: Path = Path(".")): try: os.chdir(tmp.name) yield tmp - os.chdir(original_dir) - except: # noqa: E722 + finally: os.chdir(original_dir)