From 8fc05a576b940f02eac0c006deffc3dfb726d5c4 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 26 Sep 2025 13:52:06 +0200 Subject: [PATCH 01/20] update module and subworkflow template --- nf_core/module-template/main.nf | 17 ++++++----------- nf_core/subworkflow-template/main.nf | 5 ----- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/nf_core/module-template/main.nf b/nf_core/module-template/main.nf index f99145b94d..7b2e28dd5f 100644 --- a/nf_core/module-template/main.nf +++ b/nf_core/module-template/main.nf @@ -65,7 +65,12 @@ process {{ component_name_underscore|upper }} { {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output {%- endif %} {%- endif %} - path "versions.yml" , emit: versions + {% if not_empty_template -%} + // TODO nf-core: Update the command here to obtain the version number of the software used in this module + // TODO nf-core: If multiple software packages are used in this module then each MUST be added here + // by copying the line below and replacing the current tool with the extra tool(s) + {%- endif %} + tuple val("${task.process}"), val('{{ component }}'), eval("{{ component }} --version"), topic: versions when: task.ext.when == null || task.ext.when @@ -111,11 +116,6 @@ process {{ component_name_underscore|upper }} { $bam {%- endif %} {%- endif %} - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - {{ component }}: \$({{ component }} --version) - END_VERSIONS """ stub: @@ -146,10 +146,5 @@ process {{ component_name_underscore|upper }} { touch ${prefix}.bam {%- endif %} {%- endif %} - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - {{ component }}: \$({{ component }} --version) - END_VERSIONS """ } diff --git a/nf_core/subworkflow-template/main.nf b/nf_core/subworkflow-template/main.nf index d9c3ed687c..17356a00af 100644 --- a/nf_core/subworkflow-template/main.nf +++ b/nf_core/subworkflow-template/main.nf @@ -14,9 +14,6 @@ workflow {{ component_name_underscore|upper }} { ch_bam // channel: [ val(meta), [ bam ] ] main: - - ch_versions = Channel.empty() - // TODO nf-core: substitute modules here for the modules of your subworkflow SAMTOOLS_SORT ( ch_bam ) @@ -30,6 +27,4 @@ workflow {{ component_name_underscore|upper }} { bam = SAMTOOLS_SORT.out.bam // channel: [ val(meta), [ bam ] ] bai = SAMTOOLS_INDEX.out.bai // channel: [ val(meta), [ bai ] ] csi = SAMTOOLS_INDEX.out.csi // channel: [ val(meta), [ csi ] ] - - versions = ch_versions // channel: [ versions.yml ] } From de4908fb33669efba4dfe092d1f0aa450848e53b Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 26 Sep 2025 11:55:21 +0000 Subject: [PATCH 02/20] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c337e7ad7a..45b13f9c5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - ignore files in gitignore also for pipeline_if_empty_null lint test ([#3722](https://github.com/nf-core/tools/pull/3722)) - do not check pytest_modules.yml file, deprecating ([#3748](https://github.com/nf-core/tools/pull/3748)) - Use the org from the .nf-core.yml when linting manifest name and homePage. ([#3767](https://github.com/nf-core/tools/pull/3767)) +- Add `topics` to the template + update linting ([#3779](https://github.com/nf-core/tools/pull/3779)) ### Modules From c48f7acc3e2408c414cdad020aed49952a53f46a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 29 Sep 2025 16:56:46 +0200 Subject: [PATCH 03/20] also emit versions --- nf_core/module-template/main.nf | 2 +- nf_core/module-template/meta.yml | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/nf_core/module-template/main.nf b/nf_core/module-template/main.nf index 7b2e28dd5f..0769f3309a 100644 --- a/nf_core/module-template/main.nf +++ b/nf_core/module-template/main.nf @@ -70,7 +70,7 @@ process {{ component_name_underscore|upper }} { // TODO nf-core: If multiple software packages are used in this module then each MUST be added here // by copying the line below and replacing the current tool with the extra tool(s) {%- endif %} - tuple val("${task.process}"), val('{{ component }}'), eval("{{ component }} --version"), topic: versions + tuple val("${task.process}"), val('{{ component }}'), eval("{{ component }} --version"), topic: versions, emit: versions1 when: task.ext.when == null || task.ext.when diff --git a/nf_core/module-template/meta.yml b/nf_core/module-template/meta.yml index 266b7e2882..965f0111ed 100644 --- a/nf_core/module-template/meta.yml +++ b/nf_core/module-template/meta.yml @@ -46,13 +46,16 @@ output: - edam: "http://edamontology.org/format_2572" # BAM - edam: "http://edamontology.org/format_2573" # CRAM - edam: "http://edamontology.org/format_3462" # SAM - versions: - - "versions.yml": - type: file - description: File containing software versions - pattern: "versions.yml" - ontologies: - - edam: "http://edamontology.org/format_3750" # YAML + versions1: + - - "${task.process}": + type: string + description: The name of the process + - "{{ component }}": + type: string + description: The name of the tool + - "{{ component }} --version": + type: string + description: The version of the tool authors: - "{{ author }}" From 62ea7f0f35e93a21a031b96eab5a2163d37a1aec Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 30 Sep 2025 16:28:45 +0200 Subject: [PATCH 04/20] add main.nf linting for topics --- nf_core/modules/lint/main_nf.py | 62 ++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 063fc354fa..338ff5dfee 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -36,14 +36,15 @@ def main_nf( * The module has a process label and it is among the standard ones. * If a ``meta`` map is defined as one of the modules - inputs it should be defined as one of the outputs, + inputs it should be defined as one of the emits, and be correctly configured in the ``saveAs`` function. * The module script section should contain definitions of ``software`` and ``prefix`` """ inputs: list[str] = [] - outputs: list[str] = [] + emits: list[str] = [] + topics: list[str] = [] # Check if we have a patch file affecting the 'main.nf' file # otherwise read the lines directly from the module @@ -128,8 +129,9 @@ def main_nf( line = joint_tuple inputs.extend(_parse_input(module, line)) if state == "output" and not _is_empty(line): - outputs += _parse_output(module, line) - outputs = list(set(outputs)) # remove duplicate 'meta's + emits += _parse_output_emits(module, line) + emits = list(set(emits)) # remove duplicate 'meta's + topics += _parse_output_topics(module, line) if state == "when" and not _is_empty(line): when_lines.append(line) if state == "script" and not _is_empty(line): @@ -140,7 +142,7 @@ def main_nf( exec_lines.append(line) # Check that we have required sections - if not len(outputs): + if not len(emits): module.failed.append(("main_nf_script_outputs", "No process 'output' block found", module.main_nf)) else: module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) @@ -175,8 +177,8 @@ def main_nf( if inputs: if "meta" in inputs: module.has_meta = True - if outputs: - if "meta" in outputs: + if emits: + if "meta" in emits: module.passed.append( ("main_nf_meta_output", "'meta' map emitted in output channel(s)", module.main_nf) ) @@ -186,13 +188,22 @@ def main_nf( ) # Check that a software version is emitted - if outputs: - if "versions" in outputs: - module.passed.append(("main_nf_version_emitted", "Module emits software version", module.main_nf)) + if topics: + if "versions" in topics: + module.passed.append(("main_nf_version_topic", "Module emits software versions as topic", module.main_nf)) else: - module.warned.append(("main_nf_version_emitted", "Module does not emit software version", module.main_nf)) + module.failed.append(("main_nf_version_topic", "Module does not emit software versions as topic", module.main_nf)) - return inputs, outputs + if emits: + topic_versions_amount = sum(1 for t in topics if t == "versions") + emit_versions_amount = sum(1 for e in emits if e.startswith("versions")) + if topic_versions_amount == emit_versions_amount: + module.passed.append(("main_nf_version_emit", "Module emits each software version", module.main_nf)) + else: + module.failed.append(("main_nf_version_emit", "Module does not have an `emit:` and `topic:` for each software version", module.main_nf)) + + + return inputs, emits def check_script_section(self, lines): @@ -202,12 +213,6 @@ def check_script_section(self, lines): """ script = "".join(lines) - # check that process name is used for `versions.yml` - if re.search(r"\$\{\s*task\.process\s*\}", script): - self.passed.append(("main_nf_version_script", "Process name used for versions.yml", self.main_nf)) - else: - self.warned.append(("main_nf_version_script", "Process name not used for versions.yml", self.main_nf)) - # check for prefix (only if module has a meta map as input) if self.has_meta: if re.search(r"\s*prefix\s*=\s*task.ext.prefix", script): @@ -607,15 +612,30 @@ def _parse_input(self, line_raw): return inputs -def _parse_output(self, line): +def _parse_output_emits(self, line): output = [] if "meta" in line: output.append("meta") - if "emit:" not in line: + emit_regex = re.search(r"^.*emit:\s*([^,\s]*)", line) + if not emit_regex: self.failed.append(("missing_emit", f"Missing emit statement: {line.strip()}", self.main_nf)) else: - output.append(line.split("emit:")[1].strip()) + output.append(emit_regex.group(1).strip()) + return output +def _parse_output_topics(self,line): + output = [] + if "meta" in line: + output.append("meta") + topic_regex = re.search(r"^.*topic:\s*([^,\s]*)", line) + if topic_regex: + topic_name = topic_regex.group(1).strip() + output.append(topic_name) + if topic_name == "versions": + if not re.search(r'tuple\s+val\("\${\s*task\.process\s*}"\),\s*val\(.*\),\s*eval\(.*\)', line): + self.failed.append(("wrong_version_output", 'Versions topic output is not correctly formatted, expected `tuple val("${task.process}"), val(\'\'), eval("")`', self.main_nf)) + if not re.search(r'emit:\s*versions\d+', line): + self.failed.append(("wrong_version_emit", 'Version emit should follow the format `versions`, e.g.: `versions1`, `versions2`', self.main_nf)) return output From e490c68c7d9d76c3bf3b752d1956ac5b886cf221 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 30 Sep 2025 17:02:59 +0200 Subject: [PATCH 05/20] Update modules lint --fix to work with eval --- nf_core/components/nfcore_component.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 0acd2fafad..35f69e41ea 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -253,7 +253,7 @@ def get_outputs_from_main_nf(self): return outputs output_data = data.split("output:")[1].split("when:")[0] regex_emit = r"emit:\s*([^)\s,]+)" - regex_elements = r"\b(val|path|env|stdout)\s*(\(([^)]+)\)|\s*([^)\s,]+))" + regex_elements = r"\b(val|path|env|stdout|eval)\s*(\(([^)]+)\)|\s*([^)\s,]+))" for line in output_data.split("\n"): match_emit = re.search(regex_emit, line) matches_elements = re.finditer(regex_elements, line) From 5f1af90d431eadf044b5e2485244acb1fc26cdb5 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Wed, 1 Oct 2025 13:04:26 +0200 Subject: [PATCH 06/20] Update nf_core/module-template/main.nf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/module-template/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/main.nf b/nf_core/module-template/main.nf index 0769f3309a..fd52b20fda 100644 --- a/nf_core/module-template/main.nf +++ b/nf_core/module-template/main.nf @@ -67,7 +67,7 @@ process {{ component_name_underscore|upper }} { {%- endif %} {% if not_empty_template -%} // TODO nf-core: Update the command here to obtain the version number of the software used in this module - // TODO nf-core: If multiple software packages are used in this module then each MUST be added here + // TODO nf-core: If multiple software packages are used in this module, all MUST be added here // by copying the line below and replacing the current tool with the extra tool(s) {%- endif %} tuple val("${task.process}"), val('{{ component }}'), eval("{{ component }} --version"), topic: versions, emit: versions1 From 1c6f6c934ff688ee95ebd49ee956d697d7229efb Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 1 Oct 2025 16:38:11 +0200 Subject: [PATCH 07/20] versions emit should now contain the tool name --- nf_core/modules/lint/main_nf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 338ff5dfee..010af475c8 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -634,8 +634,8 @@ def _parse_output_topics(self,line): if topic_name == "versions": if not re.search(r'tuple\s+val\("\${\s*task\.process\s*}"\),\s*val\(.*\),\s*eval\(.*\)', line): self.failed.append(("wrong_version_output", 'Versions topic output is not correctly formatted, expected `tuple val("${task.process}"), val(\'\'), eval("")`', self.main_nf)) - if not re.search(r'emit:\s*versions\d+', line): - self.failed.append(("wrong_version_emit", 'Version emit should follow the format `versions`, e.g.: `versions1`, `versions2`', self.main_nf)) + if not re.search(r'emit:\s*versions_[\d\w]+', line): + self.failed.append(("wrong_version_emit", 'Version emit should follow the format `versions_`, e.g.: `versions_samtools`, `versions_gatk4`', self.main_nf)) return output From 6cb96ee71c79e8645d168b54fcc99e70a7dcca5f Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 1 Oct 2025 17:22:02 +0200 Subject: [PATCH 08/20] linting fixes for new versions topics --- nf_core/components/nfcore_component.py | 37 ++++++++++++++++++++++++++ nf_core/module-template/main.nf | 2 +- nf_core/module-template/meta.yml | 14 +++++++++- nf_core/modules/lint/__init__.py | 2 ++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 35f69e41ea..63ec1f623d 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -294,3 +294,40 @@ def get_outputs_from_main_nf(self): pass log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") self.outputs = outputs + + def get_topics_from_main_nf(self): + with open(self.main_nf) as f: + data = f.read() + if self.component_type == "modules": + topics = {} + # get topic name from main.nf after "output:". the names are always after "topic:" + if "output:" not in data: + log.debug(f"Could not find any outputs in {self.main_nf}") + return topics + output_data = data.split("output:")[1].split("when:")[0] + regex_topic = r"topic:\s*([^)\s,]+)" + regex_elements = r"\b(val|path|env|stdout|eval)\s*(\(([^)]+)\)|\s*([^)\s,]+))" + for line in output_data.split("\n"): + match_topic = re.search(regex_topic, line) + matches_elements = re.finditer(regex_elements, line) + if not match_topic: + continue + channel_elements = [] + topic_name = match_topic.group(1) + if topic_name in topics: + continue + topics[match_topic.group(1)] = [] + for count, match_element in enumerate(matches_elements, start=1): + output_val = None + if match_element.group(3): + output_val = match_element.group(3) + elif match_element.group(4): + output_val = match_element.group(4) + if output_val: + channel_elements.append({f"value{count}": {}}) + if len(channel_elements) == 1: + topics[match_topic.group(1)].append(channel_elements[0]) + elif len(channel_elements) > 1: + topics[match_topic.group(1)].append(channel_elements) + log.debug(f"Found {len(list(topics.keys()))} topics in {self.main_nf}") + self.topics = topics \ No newline at end of file diff --git a/nf_core/module-template/main.nf b/nf_core/module-template/main.nf index fd52b20fda..49802b58c9 100644 --- a/nf_core/module-template/main.nf +++ b/nf_core/module-template/main.nf @@ -70,7 +70,7 @@ process {{ component_name_underscore|upper }} { // TODO nf-core: If multiple software packages are used in this module, all MUST be added here // by copying the line below and replacing the current tool with the extra tool(s) {%- endif %} - tuple val("${task.process}"), val('{{ component }}'), eval("{{ component }} --version"), topic: versions, emit: versions1 + tuple val("${task.process}"), val('{{ component }}'), eval("{{ component }} --version"), topic: versions, emit: versions_{{ component }} when: task.ext.when == null || task.ext.when diff --git a/nf_core/module-template/meta.yml b/nf_core/module-template/meta.yml index 965f0111ed..273f62e8a4 100644 --- a/nf_core/module-template/meta.yml +++ b/nf_core/module-template/meta.yml @@ -46,7 +46,7 @@ output: - edam: "http://edamontology.org/format_2572" # BAM - edam: "http://edamontology.org/format_2573" # CRAM - edam: "http://edamontology.org/format_3462" # SAM - versions1: + versions_{{ component }}: - - "${task.process}": type: string description: The name of the process @@ -57,6 +57,18 @@ output: type: string description: The version of the tool +topics: + - versions: + - - process: + type: string + description: The process the versions were collected from + - tool: + type: string + description: The tool name the version was collected for + - version: + type: string + description: The version of the tool + authors: - "{{ author }}" maintainers: diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 5cc9b8bf64..1ff65ff709 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -237,6 +237,7 @@ def lint_module( if local: mod.get_inputs_from_main_nf() mod.get_outputs_from_main_nf() + mod.get_topics_from_main_nf() # Update meta.yml file if requested if self.fix and mod.meta_yml is not None: self.update_meta_yml_file(mod) @@ -263,6 +264,7 @@ def lint_module( else: mod.get_inputs_from_main_nf() mod.get_outputs_from_main_nf() + mod.get_topics_from_main_nf() # Update meta.yml file if requested if self.fix: self.update_meta_yml_file(mod) From 5a31432e536451d0e50cfc2edefbcc38b4766a00 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 8 Oct 2025 15:58:06 +0200 Subject: [PATCH 09/20] add topic handling to the main workflow --- nf_core/pipeline-template/workflows/pipeline.nf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 6126f9ec69..3616b290e1 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -50,7 +50,19 @@ workflow {{ short_name|upper }} { // // Collate and save software versions // + def topic_versions = Channel.topic("versions") + .unique() + .map { process, tool, version -> + [ process[process.lastIndexOf(':')+1..-1], " ${tool}: ${version}" ] + } + .groupTuple(by:0) + .map { process, tool_versions -> + tool_versions.unique() + "${process}:\n${tool_versions.join('\n')}" + } + softwareVersionsToYAML(ch_versions) + .mix(topic_versions) .collectFile( storeDir: "${params.outdir}/pipeline_info", name: {% if is_nfcore %}'nf_core_' + {% endif %} '{{ short_name }}_software_' {% if multiqc %} + 'mqc_' {% endif %} + 'versions.yml', From 0e726e1c861a732f97d442a27c84fef4a378c9c0 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 8 Oct 2025 16:26:12 +0200 Subject: [PATCH 10/20] pre-commit --- CHANGELOG.md | 1 - nf_core/components/nfcore_component.py | 2 +- nf_core/module-template/meta.yml | 18 +++++++------- nf_core/modules/lint/main_nf.py | 34 ++++++++++++++++++++------ 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dce7451d9..9d1e1c128e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,6 @@ - Add `topics` to the template + update linting ([#3779](https://github.com/nf-core/tools/pull/3779)) - Use the org from .nf-core.yml when linting multiqc_config report_comment ([#3800](https://github.com/nf-core/tools/pull/3800)) - ### Modules - Support modules with `exec:` blocks ([#3633](https://github.com/nf-core/tools/pull/3633)) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 63ec1f623d..629b4d0c80 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -330,4 +330,4 @@ def get_topics_from_main_nf(self): elif len(channel_elements) > 1: topics[match_topic.group(1)].append(channel_elements) log.debug(f"Found {len(list(topics.keys()))} topics in {self.main_nf}") - self.topics = topics \ No newline at end of file + self.topics = topics diff --git a/nf_core/module-template/meta.yml b/nf_core/module-template/meta.yml index 273f62e8a4..9c484d9e73 100644 --- a/nf_core/module-template/meta.yml +++ b/nf_core/module-template/meta.yml @@ -59,15 +59,15 @@ output: topics: - versions: - - - process: - type: string - description: The process the versions were collected from - - tool: - type: string - description: The tool name the version was collected for - - version: - type: string - description: The version of the tool + - - process: + type: string + description: The process the versions were collected from + - tool: + type: string + description: The tool name the version was collected for + - version: + type: string + description: The version of the tool authors: - "{{ author }}" diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 010af475c8..f55d3e2cb3 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -192,7 +192,9 @@ def main_nf( if "versions" in topics: module.passed.append(("main_nf_version_topic", "Module emits software versions as topic", module.main_nf)) else: - module.failed.append(("main_nf_version_topic", "Module does not emit software versions as topic", module.main_nf)) + module.failed.append( + ("main_nf_version_topic", "Module does not emit software versions as topic", module.main_nf) + ) if emits: topic_versions_amount = sum(1 for t in topics if t == "versions") @@ -200,8 +202,13 @@ def main_nf( if topic_versions_amount == emit_versions_amount: module.passed.append(("main_nf_version_emit", "Module emits each software version", module.main_nf)) else: - module.failed.append(("main_nf_version_emit", "Module does not have an `emit:` and `topic:` for each software version", module.main_nf)) - + module.failed.append( + ( + "main_nf_version_emit", + "Module does not have an `emit:` and `topic:` for each software version", + module.main_nf, + ) + ) return inputs, emits @@ -623,7 +630,8 @@ def _parse_output_emits(self, line): output.append(emit_regex.group(1).strip()) return output -def _parse_output_topics(self,line): + +def _parse_output_topics(self, line): output = [] if "meta" in line: output.append("meta") @@ -633,9 +641,21 @@ def _parse_output_topics(self,line): output.append(topic_name) if topic_name == "versions": if not re.search(r'tuple\s+val\("\${\s*task\.process\s*}"\),\s*val\(.*\),\s*eval\(.*\)', line): - self.failed.append(("wrong_version_output", 'Versions topic output is not correctly formatted, expected `tuple val("${task.process}"), val(\'\'), eval("")`', self.main_nf)) - if not re.search(r'emit:\s*versions_[\d\w]+', line): - self.failed.append(("wrong_version_emit", 'Version emit should follow the format `versions_`, e.g.: `versions_samtools`, `versions_gatk4`', self.main_nf)) + self.failed.append( + ( + "wrong_version_output", + 'Versions topic output is not correctly formatted, expected `tuple val("${task.process}"), val(\'\'), eval("")`', + self.main_nf, + ) + ) + if not re.search(r"emit:\s*versions_[\d\w]+", line): + self.failed.append( + ( + "wrong_version_emit", + "Version emit should follow the format `versions_`, e.g.: `versions_samtools`, `versions_gatk4`", + self.main_nf, + ) + ) return output From 249c8eb63430c56f6a23d131d7414f66e57bf0b7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 9 Oct 2025 13:14:21 +0200 Subject: [PATCH 11/20] add versions file as valid topic output + fix meta yaml template --- nf_core/module-template/meta.yml | 20 +++++++++---------- .../pipeline-template/workflows/pipeline.nf | 10 ++++++++-- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/nf_core/module-template/meta.yml b/nf_core/module-template/meta.yml index 9c484d9e73..0d1c1a31a1 100644 --- a/nf_core/module-template/meta.yml +++ b/nf_core/module-template/meta.yml @@ -58,16 +58,16 @@ output: description: The version of the tool topics: - - versions: - - - process: - type: string - description: The process the versions were collected from - - tool: - type: string - description: The tool name the version was collected for - - version: - type: string - description: The version of the tool + versions: + - process: + type: string + description: The process the versions were collected from + - tool: + type: string + description: The tool name the version was collected for + - version: + type: string + description: The version of the tool authors: - "{{ author }}" diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 3616b290e1..38667e8a5b 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -51,7 +51,13 @@ workflow {{ short_name|upper }} { // Collate and save software versions // def topic_versions = Channel.topic("versions") - .unique() + .distinct() + .branch { entry -> + versions_file: entry instanceof Path + versions_tuple: true + } + + def topic_versions_string = topic_versions.versions_tuple .map { process, tool, version -> [ process[process.lastIndexOf(':')+1..-1], " ${tool}: ${version}" ] } @@ -61,7 +67,7 @@ workflow {{ short_name|upper }} { "${process}:\n${tool_versions.join('\n')}" } - softwareVersionsToYAML(ch_versions) + softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) .mix(topic_versions) .collectFile( storeDir: "${params.outdir}/pipeline_info", From 6b02333a2323a2369eb0f80135b1ee15f0cfc568 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 9 Oct 2025 15:49:10 +0200 Subject: [PATCH 12/20] update meta yaml versions output on module create --- nf_core/components/create.py | 44 ++++-- nf_core/module-template/meta.yml | 18 +-- tests/modules/test_create.py | 224 ++++++++++++++++++++++++++----- 3 files changed, 232 insertions(+), 54 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 03d04f2abf..a1b7cc07cd 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -530,20 +530,42 @@ def generate_meta_yml_file(self) -> None: meta_yml: ruamel.yaml.comments.CommentedMap = yaml.load(fh) versions: dict[str, list[dict[str, dict]]] = { + f"versions_{self.component}": [ + { + '${task.process}': { + "type": "string", + "description": "The name of the process" + }, + f"{self.component}": { + "type": "string", + "description": "The name of the tool" + }, + f"{self.component} --version": { + "type": "string", + "description": "The version of the tool" + } + } + ] + } + + versions_topic: dict[str, list[dict[str,dict]]] = { "versions": [ { - "versions.yml": { - "type": "file", - "description": "File containing software versions", - "pattern": "versions.yml", - "ontologies": [ - ruamel.yaml.comments.CommentedMap({"edam": "http://edamontology.org/format_3750"}) - ], + "process": { + "type": "string", + "description": "The process the versions were collected from" + }, + "tool": { + "type": "string", + "description": "The tool name the version was collected for" + }, + "version": { + "type": "string", + "description": "The version of the tool" } } ] } - versions["versions"][0]["versions.yml"]["ontologies"][0].yaml_add_eol_comment("YAML", "edam") if self.not_empty_template: meta_yml.yaml_set_comment_before_after_key( @@ -558,6 +580,9 @@ def generate_meta_yml_file(self) -> None: meta_yml["output"].yaml_set_start_comment( "### TODO nf-core: Add a description of all of the variables used as output", indent=2 ) + meta_yml["topics"].yaml_set_start_comment( + "### TODO nf-core: Add a description of all of the variables used as topics", indent=2 + ) if hasattr(self, "inputs"): inputs_array: list[Union[dict, list[dict]]] = [] @@ -669,6 +694,8 @@ def generate_meta_yml_file(self) -> None: meta_yml["output"]["bam"][0]["*.bam"]["ontologies"][2].yaml_add_eol_comment("SAM", "edam") meta_yml["output"].update(versions) + meta_yml["topics"] = versions_topic + else: input_entry: list[dict] = [ {"input": {"type": "file", "description": "", "pattern": "", "ontologies": [{"edam": ""}]}} @@ -691,6 +718,7 @@ def generate_meta_yml_file(self) -> None: meta_yml["input"] = input_entry meta_yml["output"] = {"output": output_entry} meta_yml["output"].update(versions) + meta_yml["topics"] = versions_topic with open(self.file_paths["meta.yml"], "w") as fh: yaml.dump(meta_yml, fh) diff --git a/nf_core/module-template/meta.yml b/nf_core/module-template/meta.yml index 0d1c1a31a1..8dc2af3c4b 100644 --- a/nf_core/module-template/meta.yml +++ b/nf_core/module-template/meta.yml @@ -59,15 +59,15 @@ output: topics: versions: - - process: - type: string - description: The process the versions were collected from - - tool: - type: string - description: The tool name the version was collected for - - version: - type: string - description: The version of the tool + - - process: + type: string + description: The process the versions were collected from + - tool: + type: string + description: The tool name the version was collected for + - version: + type: string + description: The version of the tool authors: - "{{ author }}" diff --git a/tests/modules/test_create.py b/tests/modules/test_create.py index 6ef7c3db3a..b7c876c123 100644 --- a/tests/modules/test_create.py +++ b/tests/modules/test_create.py @@ -236,17 +236,41 @@ def test_modules_meta_yml_structure_biotools_meta(self): }, ] ], - "versions": [ + "versions_bpipe": [ { - "versions.yml": { - "type": "file", - "description": "File containing software versions", - "pattern": "versions.yml", - "ontologies": [{"edam": "http://edamontology.org/format_3750"}], + "${task.process}": { + "type": "string", + "description": "The name of the process" + }, + "bpipe": { + "type": "string", + "description": "The name of the tool" + }, + "bpipe --version": { + "type": "string", + "description": "The version of the tool" } } ], }, + 'topics': { + 'versions': [ + { + 'process': { + 'description': 'The process the versions were collected from', + 'type': 'string', + }, + 'tool': { + 'description': 'The tool name the version was collected for', + 'type': 'string', + }, + 'version': { + 'description': 'The version of the tool', + 'type': 'string', + } + } + ] + }, "authors": ["@author"], "maintainers": ["@author"], } @@ -312,17 +336,41 @@ def test_modules_meta_yml_structure_biotools_nometa(self): } } ], - "versions": [ + "versions_bpipe": [ { - "versions.yml": { - "type": "file", - "description": "File containing software versions", - "pattern": "versions.yml", - "ontologies": [{"edam": "http://edamontology.org/format_3750"}], + "${task.process}": { + "type": "string", + "description": "The name of the process" + }, + "bpipe": { + "type": "string", + "description": "The name of the tool" + }, + "bpipe --version": { + "type": "string", + "description": "The version of the tool" } } ], }, + 'topics': { + 'versions': [ + { + 'process': { + 'description': 'The process the versions were collected from', + 'type': 'string', + }, + 'tool': { + 'description': 'The tool name the version was collected for', + 'type': 'string', + }, + 'version': { + 'description': 'The version of the tool', + 'type': 'string', + } + } + ] + }, "authors": ["@author"], "maintainers": ["@author"], } @@ -411,16 +459,46 @@ def test_modules_meta_yml_structure_template_meta( }, ] ], - "versions": [ + "versions_test": [ + [ + { + "${task.process}": { + "type": "string", + "description": "The name of the process" + } + }, + { + "test": { + "type": "string", + "description": "The name of the tool" + } + }, + { + "test --version": { + "type": "string", + "description": "The version of the tool" + } + } + ] + ], + }, + 'topics': { + 'versions': [ { - "versions.yml": { - "type": "file", - "description": "File containing software versions", - "pattern": "versions.yml", - "ontologies": [{"edam": "http://edamontology.org/format_3750"}], + 'process': { + 'description': 'The process the versions were collected from', + 'type': 'string', + }, + 'tool': { + 'description': 'The tool name the version was collected for', + 'type': 'string', + }, + 'version': { + 'description': 'The version of the tool', + 'type': 'string', } } - ], + ] }, "authors": ["@author"], "maintainers": ["@author"], @@ -494,17 +572,41 @@ def test_modules_meta_yml_structure_template_nometa( } } ], - "versions": [ + "versions_test": [ { - "versions.yml": { - "type": "file", - "description": "File containing software versions", - "pattern": "versions.yml", - "ontologies": [{"edam": "http://edamontology.org/format_3750"}], + "${task.process}": { + "type": "string", + "description": "The name of the process" + }, + "test": { + "type": "string", + "description": "The name of the tool" + }, + "test --version": { + "type": "string", + "description": "The version of the tool" } } ], }, + 'topics': { + 'versions': [ + { + 'process': { + 'description': 'The process the versions were collected from', + 'type': 'string', + }, + 'tool': { + 'description': 'The tool name the version was collected for', + 'type': 'string', + }, + 'version': { + 'description': 'The version of the tool', + 'type': 'string', + } + } + ] + }, "authors": ["@author"], "maintainers": ["@author"], } @@ -571,17 +673,41 @@ def test_modules_meta_yml_structure_empty_meta( {"*": {"type": "file", "description": "", "pattern": "", "ontologies": [{"edam": ""}]}}, ] ], - "versions": [ + "versions_test": [ { - "versions.yml": { - "type": "file", - "description": "File containing software versions", - "pattern": "versions.yml", - "ontologies": [{"edam": "http://edamontology.org/format_3750"}], + "${task.process}": { + "type": "string", + "description": "The name of the process" + }, + "test": { + "type": "string", + "description": "The name of the tool" + }, + "test --version": { + "type": "string", + "description": "The version of the tool" } } ], }, + 'topics': { + 'versions': [ + { + 'process': { + 'description': 'The process the versions were collected from', + 'type': 'string', + }, + 'tool': { + 'description': 'The tool name the version was collected for', + 'type': 'string', + }, + 'version': { + 'description': 'The version of the tool', + 'type': 'string', + } + } + ] + }, "authors": ["@author"], "maintainers": ["@author"], } @@ -628,17 +754,41 @@ def test_modules_meta_yml_structure_empty_nometa( "input": [{"input": {"type": "file", "description": "", "pattern": "", "ontologies": [{"edam": ""}]}}], "output": { "output": [{"*": {"type": "file", "description": "", "pattern": "", "ontologies": [{"edam": ""}]}}], - "versions": [ + "versions_test": [ { - "versions.yml": { - "type": "file", - "description": "File containing software versions", - "pattern": "versions.yml", - "ontologies": [{"edam": "http://edamontology.org/format_3750"}], + "${task.process}": { + "type": "string", + "description": "The name of the process" + }, + "test": { + "type": "string", + "description": "The name of the tool" + }, + "test --version": { + "type": "string", + "description": "The version of the tool" } } ], }, + 'topics': { + 'versions': [ + { + 'process': { + 'description': 'The process the versions were collected from', + 'type': 'string', + }, + 'tool': { + 'description': 'The tool name the version was collected for', + 'type': 'string', + }, + 'version': { + 'description': 'The version of the tool', + 'type': 'string', + } + } + ] + }, "authors": ["@author"], "maintainers": ["@author"], } From 309559752758351fed1af5a24acb47427665b5ea Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 9 Oct 2025 16:08:45 +0200 Subject: [PATCH 13/20] pre-commit --- nf_core/components/create.py | 32 +---- nf_core/modules/lint/main_nf.py | 1 + tests/modules/test_create.py | 240 ++++++++++++-------------------- 3 files changed, 98 insertions(+), 175 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index a1b7cc07cd..ef272ca6dd 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -532,37 +532,19 @@ def generate_meta_yml_file(self) -> None: versions: dict[str, list[dict[str, dict]]] = { f"versions_{self.component}": [ { - '${task.process}': { - "type": "string", - "description": "The name of the process" - }, - f"{self.component}": { - "type": "string", - "description": "The name of the tool" - }, - f"{self.component} --version": { - "type": "string", - "description": "The version of the tool" - } + "${task.process}": {"type": "string", "description": "The name of the process"}, + f"{self.component}": {"type": "string", "description": "The name of the tool"}, + f"{self.component} --version": {"type": "string", "description": "The version of the tool"}, } ] } - versions_topic: dict[str, list[dict[str,dict]]] = { + versions_topic: dict[str, list[dict[str, dict]]] = { "versions": [ { - "process": { - "type": "string", - "description": "The process the versions were collected from" - }, - "tool": { - "type": "string", - "description": "The tool name the version was collected for" - }, - "version": { - "type": "string", - "description": "The version of the tool" - } + "process": {"type": "string", "description": "The process the versions were collected from"}, + "tool": {"type": "string", "description": "The tool name the version was collected for"}, + "version": {"type": "string", "description": "The version of the tool"}, } ] } diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0799f04b3e..a918765a79 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -717,6 +717,7 @@ def _parse_output_emits(self, line): output.append(emit_regex.group(1).strip()) return output + def _parse_output_topics(self, line): output = [] if "meta" in line: diff --git a/tests/modules/test_create.py b/tests/modules/test_create.py index b7c876c123..2e1318d2f9 100644 --- a/tests/modules/test_create.py +++ b/tests/modules/test_create.py @@ -238,36 +238,27 @@ def test_modules_meta_yml_structure_biotools_meta(self): ], "versions_bpipe": [ { - "${task.process}": { - "type": "string", - "description": "The name of the process" - }, - "bpipe": { - "type": "string", - "description": "The name of the tool" - }, - "bpipe --version": { - "type": "string", - "description": "The version of the tool" - } + "${task.process}": {"type": "string", "description": "The name of the process"}, + "bpipe": {"type": "string", "description": "The name of the tool"}, + "bpipe --version": {"type": "string", "description": "The version of the tool"}, } ], }, - 'topics': { - 'versions': [ + "topics": { + "versions": [ { - 'process': { - 'description': 'The process the versions were collected from', - 'type': 'string', + "process": { + "description": "The process the versions were collected from", + "type": "string", }, - 'tool': { - 'description': 'The tool name the version was collected for', - 'type': 'string', + "tool": { + "description": "The tool name the version was collected for", + "type": "string", + }, + "version": { + "description": "The version of the tool", + "type": "string", }, - 'version': { - 'description': 'The version of the tool', - 'type': 'string', - } } ] }, @@ -338,36 +329,27 @@ def test_modules_meta_yml_structure_biotools_nometa(self): ], "versions_bpipe": [ { - "${task.process}": { - "type": "string", - "description": "The name of the process" - }, - "bpipe": { - "type": "string", - "description": "The name of the tool" - }, - "bpipe --version": { - "type": "string", - "description": "The version of the tool" - } + "${task.process}": {"type": "string", "description": "The name of the process"}, + "bpipe": {"type": "string", "description": "The name of the tool"}, + "bpipe --version": {"type": "string", "description": "The version of the tool"}, } ], }, - 'topics': { - 'versions': [ + "topics": { + "versions": [ { - 'process': { - 'description': 'The process the versions were collected from', - 'type': 'string', + "process": { + "description": "The process the versions were collected from", + "type": "string", + }, + "tool": { + "description": "The tool name the version was collected for", + "type": "string", }, - 'tool': { - 'description': 'The tool name the version was collected for', - 'type': 'string', + "version": { + "description": "The version of the tool", + "type": "string", }, - 'version': { - 'description': 'The version of the tool', - 'type': 'string', - } } ] }, @@ -461,42 +443,27 @@ def test_modules_meta_yml_structure_template_meta( ], "versions_test": [ [ - { - "${task.process}": { - "type": "string", - "description": "The name of the process" - } - }, - { - "test": { - "type": "string", - "description": "The name of the tool" - } - }, - { - "test --version": { - "type": "string", - "description": "The version of the tool" - } - } + {"${task.process}": {"type": "string", "description": "The name of the process"}}, + {"test": {"type": "string", "description": "The name of the tool"}}, + {"test --version": {"type": "string", "description": "The version of the tool"}}, ] ], }, - 'topics': { - 'versions': [ + "topics": { + "versions": [ { - 'process': { - 'description': 'The process the versions were collected from', - 'type': 'string', + "process": { + "description": "The process the versions were collected from", + "type": "string", }, - 'tool': { - 'description': 'The tool name the version was collected for', - 'type': 'string', + "tool": { + "description": "The tool name the version was collected for", + "type": "string", + }, + "version": { + "description": "The version of the tool", + "type": "string", }, - 'version': { - 'description': 'The version of the tool', - 'type': 'string', - } } ] }, @@ -574,36 +541,27 @@ def test_modules_meta_yml_structure_template_nometa( ], "versions_test": [ { - "${task.process}": { - "type": "string", - "description": "The name of the process" - }, - "test": { - "type": "string", - "description": "The name of the tool" - }, - "test --version": { - "type": "string", - "description": "The version of the tool" - } + "${task.process}": {"type": "string", "description": "The name of the process"}, + "test": {"type": "string", "description": "The name of the tool"}, + "test --version": {"type": "string", "description": "The version of the tool"}, } ], }, - 'topics': { - 'versions': [ + "topics": { + "versions": [ { - 'process': { - 'description': 'The process the versions were collected from', - 'type': 'string', + "process": { + "description": "The process the versions were collected from", + "type": "string", }, - 'tool': { - 'description': 'The tool name the version was collected for', - 'type': 'string', + "tool": { + "description": "The tool name the version was collected for", + "type": "string", + }, + "version": { + "description": "The version of the tool", + "type": "string", }, - 'version': { - 'description': 'The version of the tool', - 'type': 'string', - } } ] }, @@ -675,36 +633,27 @@ def test_modules_meta_yml_structure_empty_meta( ], "versions_test": [ { - "${task.process}": { - "type": "string", - "description": "The name of the process" - }, - "test": { - "type": "string", - "description": "The name of the tool" - }, - "test --version": { - "type": "string", - "description": "The version of the tool" - } + "${task.process}": {"type": "string", "description": "The name of the process"}, + "test": {"type": "string", "description": "The name of the tool"}, + "test --version": {"type": "string", "description": "The version of the tool"}, } ], }, - 'topics': { - 'versions': [ + "topics": { + "versions": [ { - 'process': { - 'description': 'The process the versions were collected from', - 'type': 'string', + "process": { + "description": "The process the versions were collected from", + "type": "string", }, - 'tool': { - 'description': 'The tool name the version was collected for', - 'type': 'string', + "tool": { + "description": "The tool name the version was collected for", + "type": "string", + }, + "version": { + "description": "The version of the tool", + "type": "string", }, - 'version': { - 'description': 'The version of the tool', - 'type': 'string', - } } ] }, @@ -756,36 +705,27 @@ def test_modules_meta_yml_structure_empty_nometa( "output": [{"*": {"type": "file", "description": "", "pattern": "", "ontologies": [{"edam": ""}]}}], "versions_test": [ { - "${task.process}": { - "type": "string", - "description": "The name of the process" - }, - "test": { - "type": "string", - "description": "The name of the tool" - }, - "test --version": { - "type": "string", - "description": "The version of the tool" - } + "${task.process}": {"type": "string", "description": "The name of the process"}, + "test": {"type": "string", "description": "The name of the tool"}, + "test --version": {"type": "string", "description": "The version of the tool"}, } ], }, - 'topics': { - 'versions': [ + "topics": { + "versions": [ { - 'process': { - 'description': 'The process the versions were collected from', - 'type': 'string', + "process": { + "description": "The process the versions were collected from", + "type": "string", }, - 'tool': { - 'description': 'The tool name the version was collected for', - 'type': 'string', + "tool": { + "description": "The tool name the version was collected for", + "type": "string", + }, + "version": { + "description": "The version of the tool", + "type": "string", }, - 'version': { - 'description': 'The version of the tool', - 'type': 'string', - } } ] }, From 0eea1b4f2fda3f80ca312c4ebdf28401cafa74cd Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 9 Oct 2025 16:16:04 +0200 Subject: [PATCH 14/20] pre-commit --- nf_core/modules/lint/main_nf.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index a918765a79..b2b28ee7ad 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -217,20 +217,25 @@ def main_nf( # Check that a software version is emitted if topics: if "versions" in topics: - module.passed.append(("main_nf_version_topic", "Module emits software versions as topic", module.main_nf)) + module.passed.append( + ("main_nf", "main_nf_version_topic", "Module emits software versions as topic", module.main_nf) + ) else: module.failed.append( - ("main_nf_version_topic", "Module does not emit software versions as topic", module.main_nf) + ("main_nf", "main_nf_version_topic", "Module does not emit software versions as topic", module.main_nf) ) if emits: topic_versions_amount = sum(1 for t in topics if t == "versions") emit_versions_amount = sum(1 for e in emits if e.startswith("versions")) if topic_versions_amount == emit_versions_amount: - module.passed.append(("main_nf_version_emit", "Module emits each software version", module.main_nf)) + module.passed.append( + ("main_nf", "main_nf_version_emit", "Module emits each software version", module.main_nf) + ) else: module.failed.append( ( + "main_nf", "main_nf_version_emit", "Module does not have an `emit:` and `topic:` for each software version", module.main_nf, From 2d49346ecd910778067f75d339cc4b6b778ba2be Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 13 Oct 2025 11:49:28 +0200 Subject: [PATCH 15/20] sort versions output --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 38667e8a5b..4f0cc49740 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -63,7 +63,7 @@ workflow {{ short_name|upper }} { } .groupTuple(by:0) .map { process, tool_versions -> - tool_versions.unique() + tool_versions.unique().sort() "${process}:\n${tool_versions.join('\n')}" } From c7be4d728fbd2656ce1c30505d77eafadbb56ec5 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 13 Oct 2025 13:31:00 +0200 Subject: [PATCH 16/20] try to add versions on module creation --- nf_core/components/create.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index ef272ca6dd..5fdcc6bf2e 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -529,13 +529,19 @@ def generate_meta_yml_file(self) -> None: with open(self.file_paths["meta.yml"]) as fh: meta_yml: ruamel.yaml.comments.CommentedMap = yaml.load(fh) - versions: dict[str, list[dict[str, dict]]] = { + versions: dict[str, list[list[dict[str, dict]]]] = { f"versions_{self.component}": [ - { - "${task.process}": {"type": "string", "description": "The name of the process"}, - f"{self.component}": {"type": "string", "description": "The name of the tool"}, - f"{self.component} --version": {"type": "string", "description": "The version of the tool"}, - } + [ + { + "${task.process}": {"type": "string", "description": "The name of the process"} + }, + { + f"{self.component}": {"type": "string", "description": "The name of the tool"} + }, + { + f"{self.component} --version": {"type": "string", "description": "The version of the tool"}, + } + ] ] } From 5e1928d3077857912f5b8ff359b16b5f8f25fe1d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 14 Oct 2025 15:58:30 +0200 Subject: [PATCH 17/20] update topics structure + added a check for empty input and output --- nf_core/components/create.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 5fdcc6bf2e..664500a151 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -529,7 +529,7 @@ def generate_meta_yml_file(self) -> None: with open(self.file_paths["meta.yml"]) as fh: meta_yml: ruamel.yaml.comments.CommentedMap = yaml.load(fh) - versions: dict[str, list[list[dict[str, dict]]]] = { + versions: dict[str, Union[list, dict]] = { f"versions_{self.component}": [ [ { @@ -545,13 +545,19 @@ def generate_meta_yml_file(self) -> None: ] } - versions_topic: dict[str, list[dict[str, dict]]] = { + versions_topic: dict[str, Union[list, dict]] = { "versions": [ - { - "process": {"type": "string", "description": "The process the versions were collected from"}, - "tool": {"type": "string", "description": "The tool name the version was collected for"}, - "version": {"type": "string", "description": "The version of the tool"}, - } + [ + { + "process": {"type": "string", "description": "The process the versions were collected from"} + }, + { + "tool": {"type": "string", "description": "The tool name the version was collected for"}, + }, + { + "version": {"type": "string", "description": "The version of the tool"}, + } + ] ] } @@ -572,7 +578,7 @@ def generate_meta_yml_file(self) -> None: "### TODO nf-core: Add a description of all of the variables used as topics", indent=2 ) - if hasattr(self, "inputs"): + if hasattr(self, "inputs") and len(self.inputs) > 0: inputs_array: list[Union[dict, list[dict]]] = [] for i, (input_name, ontologies) in enumerate(self.inputs.items()): channel_entry: dict[str, dict] = { @@ -621,7 +627,7 @@ def generate_meta_yml_file(self) -> None: meta_yml["input"][0]["bam"]["ontologies"][1].yaml_add_eol_comment("CRAM", "edam") meta_yml["input"][0]["bam"]["ontologies"][2].yaml_add_eol_comment("SAM", "edam") - if hasattr(self, "outputs"): + if hasattr(self, "outputs") and len(self.outputs) > 0: outputs_dict: dict[str, Union[list, dict]] = {} for i, (output_name, ontologies) in enumerate(self.outputs.items()): channel_contents: list[Union[list[dict], dict]] = [] From b1502440ef4d8bde4e71e9e160540d83e99d1bba Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 14 Oct 2025 16:01:32 +0200 Subject: [PATCH 18/20] fix wrongly resolved merge conflict --- nf_core/components/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 2cadc5fdd5..fb0a3fc7f8 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -577,7 +577,7 @@ def generate_meta_yml_file(self) -> None: "### TODO nf-core: Add a description of all of the variables used as topics", indent=2 ) - if hasattr(self, "inputs"): + if hasattr(self, "inputs") and len(self.inputs) > 0: inputs_array: list[dict | list[dict]] = [] for i, (input_name, ontologies) in enumerate(self.inputs.items()): channel_entry: dict[str, dict] = { @@ -626,7 +626,7 @@ def generate_meta_yml_file(self) -> None: meta_yml["input"][0]["bam"]["ontologies"][1].yaml_add_eol_comment("CRAM", "edam") meta_yml["input"][0]["bam"]["ontologies"][2].yaml_add_eol_comment("SAM", "edam") - if hasattr(self, "outputs"): + if hasattr(self, "outputs") and len(self.outputs) > 0: outputs_dict: dict[str, list | dict] = {} for i, (output_name, ontologies) in enumerate(self.outputs.items()): channel_contents: list[list[dict] | dict] = [] From 7c954d6733b66bd6d8385a9c56fdcae01d729ce4 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 14 Oct 2025 16:21:35 +0200 Subject: [PATCH 19/20] fix module create tests + pre-commit --- nf_core/components/create.py | 20 ++--- tests/modules/test_create.py | 164 ++++++++++++----------------------- 2 files changed, 62 insertions(+), 122 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index fb0a3fc7f8..79c4403105 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -528,34 +528,28 @@ def generate_meta_yml_file(self) -> None: with open(self.file_paths["meta.yml"]) as fh: meta_yml: ruamel.yaml.comments.CommentedMap = yaml.load(fh) - versions: dict[str, Union[list, dict]] = { + versions: dict[str, list | dict] = { f"versions_{self.component}": [ [ - { - "${task.process}": {"type": "string", "description": "The name of the process"} - }, - { - f"{self.component}": {"type": "string", "description": "The name of the tool"} - }, + {"${task.process}": {"type": "string", "description": "The name of the process"}}, + {f"{self.component}": {"type": "string", "description": "The name of the tool"}}, { f"{self.component} --version": {"type": "string", "description": "The version of the tool"}, - } + }, ] ] } - versions_topic: dict[str, Union[list, dict]] = { + versions_topic: dict[str, list | dict] = { "versions": [ [ - { - "process": {"type": "string", "description": "The process the versions were collected from"} - }, + {"process": {"type": "string", "description": "The process the versions were collected from"}}, { "tool": {"type": "string", "description": "The tool name the version was collected for"}, }, { "version": {"type": "string", "description": "The version of the tool"}, - } + }, ] ] } diff --git a/tests/modules/test_create.py b/tests/modules/test_create.py index 2e1318d2f9..9e246675c3 100644 --- a/tests/modules/test_create.py +++ b/tests/modules/test_create.py @@ -237,29 +237,20 @@ def test_modules_meta_yml_structure_biotools_meta(self): ] ], "versions_bpipe": [ - { - "${task.process}": {"type": "string", "description": "The name of the process"}, - "bpipe": {"type": "string", "description": "The name of the tool"}, - "bpipe --version": {"type": "string", "description": "The version of the tool"}, - } + [ + {"${task.process}": {"type": "string", "description": "The name of the process"}}, + {"bpipe": {"type": "string", "description": "The name of the tool"}}, + {"bpipe --version": {"type": "string", "description": "The version of the tool"}}, + ] ], }, "topics": { "versions": [ - { - "process": { - "description": "The process the versions were collected from", - "type": "string", - }, - "tool": { - "description": "The tool name the version was collected for", - "type": "string", - }, - "version": { - "description": "The version of the tool", - "type": "string", - }, - } + [ + {"process": {"description": "The process the versions were collected from", "type": "string"}}, + {"tool": {"description": "The tool name the version was collected for", "type": "string"}}, + {"version": {"description": "The version of the tool", "type": "string"}}, + ] ] }, "authors": ["@author"], @@ -328,29 +319,20 @@ def test_modules_meta_yml_structure_biotools_nometa(self): } ], "versions_bpipe": [ - { - "${task.process}": {"type": "string", "description": "The name of the process"}, - "bpipe": {"type": "string", "description": "The name of the tool"}, - "bpipe --version": {"type": "string", "description": "The version of the tool"}, - } + [ + {"${task.process}": {"type": "string", "description": "The name of the process"}}, + {"bpipe": {"type": "string", "description": "The name of the tool"}}, + {"bpipe --version": {"type": "string", "description": "The version of the tool"}}, + ] ], }, "topics": { "versions": [ - { - "process": { - "description": "The process the versions were collected from", - "type": "string", - }, - "tool": { - "description": "The tool name the version was collected for", - "type": "string", - }, - "version": { - "description": "The version of the tool", - "type": "string", - }, - } + [ + {"process": {"description": "The process the versions were collected from", "type": "string"}}, + {"tool": {"description": "The tool name the version was collected for", "type": "string"}}, + {"version": {"description": "The version of the tool", "type": "string"}}, + ] ] }, "authors": ["@author"], @@ -451,20 +433,11 @@ def test_modules_meta_yml_structure_template_meta( }, "topics": { "versions": [ - { - "process": { - "description": "The process the versions were collected from", - "type": "string", - }, - "tool": { - "description": "The tool name the version was collected for", - "type": "string", - }, - "version": { - "description": "The version of the tool", - "type": "string", - }, - } + [ + {"process": {"description": "The process the versions were collected from", "type": "string"}}, + {"tool": {"description": "The tool name the version was collected for", "type": "string"}}, + {"version": {"description": "The version of the tool", "type": "string"}}, + ] ] }, "authors": ["@author"], @@ -540,29 +513,20 @@ def test_modules_meta_yml_structure_template_nometa( } ], "versions_test": [ - { - "${task.process}": {"type": "string", "description": "The name of the process"}, - "test": {"type": "string", "description": "The name of the tool"}, - "test --version": {"type": "string", "description": "The version of the tool"}, - } + [ + {"${task.process}": {"type": "string", "description": "The name of the process"}}, + {"test": {"type": "string", "description": "The name of the tool"}}, + {"test --version": {"type": "string", "description": "The version of the tool"}}, + ] ], }, "topics": { "versions": [ - { - "process": { - "description": "The process the versions were collected from", - "type": "string", - }, - "tool": { - "description": "The tool name the version was collected for", - "type": "string", - }, - "version": { - "description": "The version of the tool", - "type": "string", - }, - } + [ + {"process": {"description": "The process the versions were collected from", "type": "string"}}, + {"tool": {"description": "The tool name the version was collected for", "type": "string"}}, + {"version": {"description": "The version of the tool", "type": "string"}}, + ] ] }, "authors": ["@author"], @@ -632,29 +596,20 @@ def test_modules_meta_yml_structure_empty_meta( ] ], "versions_test": [ - { - "${task.process}": {"type": "string", "description": "The name of the process"}, - "test": {"type": "string", "description": "The name of the tool"}, - "test --version": {"type": "string", "description": "The version of the tool"}, - } + [ + {"${task.process}": {"type": "string", "description": "The name of the process"}}, + {"test": {"type": "string", "description": "The name of the tool"}}, + {"test --version": {"type": "string", "description": "The version of the tool"}}, + ] ], }, "topics": { "versions": [ - { - "process": { - "description": "The process the versions were collected from", - "type": "string", - }, - "tool": { - "description": "The tool name the version was collected for", - "type": "string", - }, - "version": { - "description": "The version of the tool", - "type": "string", - }, - } + [ + {"process": {"description": "The process the versions were collected from", "type": "string"}}, + {"tool": {"description": "The tool name the version was collected for", "type": "string"}}, + {"version": {"description": "The version of the tool", "type": "string"}}, + ] ] }, "authors": ["@author"], @@ -704,29 +659,20 @@ def test_modules_meta_yml_structure_empty_nometa( "output": { "output": [{"*": {"type": "file", "description": "", "pattern": "", "ontologies": [{"edam": ""}]}}], "versions_test": [ - { - "${task.process}": {"type": "string", "description": "The name of the process"}, - "test": {"type": "string", "description": "The name of the tool"}, - "test --version": {"type": "string", "description": "The version of the tool"}, - } + [ + {"${task.process}": {"type": "string", "description": "The name of the process"}}, + {"test": {"type": "string", "description": "The name of the tool"}}, + {"test --version": {"type": "string", "description": "The version of the tool"}}, + ] ], }, "topics": { "versions": [ - { - "process": { - "description": "The process the versions were collected from", - "type": "string", - }, - "tool": { - "description": "The tool name the version was collected for", - "type": "string", - }, - "version": { - "description": "The version of the tool", - "type": "string", - }, - } + [ + {"process": {"description": "The process the versions were collected from", "type": "string"}}, + {"tool": {"description": "The tool name the version was collected for", "type": "string"}}, + {"version": {"description": "The version of the tool", "type": "string"}}, + ] ] }, "authors": ["@author"], From fbe8f635806a019c4adb86bcf5ad78e37a210dd9 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 16 Oct 2025 14:43:57 +0200 Subject: [PATCH 20/20] fix usage of versions string --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 4f0cc49740..94bdae9895 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -68,7 +68,7 @@ workflow {{ short_name|upper }} { } softwareVersionsToYAML(ch_versions.mix(topic_versions.versions_file)) - .mix(topic_versions) + .mix(topic_versions_string) .collectFile( storeDir: "${params.outdir}/pipeline_info", name: {% if is_nfcore %}'nf_core_' + {% endif %} '{{ short_name }}_software_' {% if multiqc %} + 'mqc_' {% endif %} + 'versions.yml',