From 36b80e4d8bbe13579db7e8820a2d77ff98bb05ba Mon Sep 17 00:00:00 2001 From: Toni Hermoso Pulido Date: Wed, 26 Mar 2025 17:33:18 +0100 Subject: [PATCH 1/6] update pending work --- nf_core/modules/lint/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 9a3e7172cc..7ba3671232 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -373,6 +373,7 @@ def update_meta_yml_file(self, mod): break # EDAM ontologies + # TODO: Refactor common code from input and output edam_formats = nf_core.modules.modules_utils.load_edam() if "input" in meta_yml: for i, channel in enumerate(corrected_meta_yml["input"]): @@ -382,6 +383,7 @@ def update_meta_yml_file(self, mod): current_ontologies_i = [] if "pattern" in corrected_meta_yml["input"][i][j][element_name]: pattern = corrected_meta_yml["input"][i][j][element_name]["pattern"] + # TODO: check pattern detection and processing for different cases for extension in re.split(r",|{|}", pattern): if extension in edam_formats: expected_ontologies_i.append((edam_formats[extension][0], extension)) @@ -413,6 +415,7 @@ def update_meta_yml_file(self, mod): current_ontologies_o = [] if "pattern" in corrected_meta_yml["output"][i][ch_name][j][element_name]: pattern = corrected_meta_yml["output"][i][ch_name][j][element_name]["pattern"] + # TODO: check pattern detection and processing for different cases for extension in re.split(r",|{|}", pattern): if extension in edam_formats: expected_ontologies_o.append((edam_formats[extension][0], extension)) From 5eff207a85958e7f319bd7039592919a178a3dcb Mon Sep 17 00:00:00 2001 From: Toni Hermoso Pulido Date: Thu, 27 Mar 2025 17:38:41 +0100 Subject: [PATCH 2/6] handling more pattern cases and likely solving https://github.com/nf-core/tools/issues/3496 --- nf_core/modules/lint/__init__.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 7ba3671232..59d959e990 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -383,10 +383,16 @@ def update_meta_yml_file(self, mod): current_ontologies_i = [] if "pattern" in corrected_meta_yml["input"][i][j][element_name]: pattern = corrected_meta_yml["input"][i][j][element_name]["pattern"] - # TODO: check pattern detection and processing for different cases - for extension in re.split(r",|{|}", pattern): - if extension in edam_formats: - expected_ontologies_i.append((edam_formats[extension][0], extension)) + # Check pattern detection and process for different cases + if re.search(r"{", pattern): + for extension in re.split(r",|{|}", pattern): + if extension in edam_formats: + expected_ontologies_i.append((edam_formats[extension][0], extension)) + else: + if re.search(r"\.\w+$", pattern): + extension = pattern.split(".")[-1] + if extension in edam_formats: + expected_ontologies_i.append((edam_formats[extension][0], extension)) if "ontologies" in corrected_meta_yml["input"][i][j][element_name]: for ontology in corrected_meta_yml["input"][i][j][element_name]["ontologies"]: current_ontologies_i.append(ontology["edam"]) @@ -415,10 +421,16 @@ def update_meta_yml_file(self, mod): current_ontologies_o = [] if "pattern" in corrected_meta_yml["output"][i][ch_name][j][element_name]: pattern = corrected_meta_yml["output"][i][ch_name][j][element_name]["pattern"] - # TODO: check pattern detection and processing for different cases - for extension in re.split(r",|{|}", pattern): - if extension in edam_formats: - expected_ontologies_o.append((edam_formats[extension][0], extension)) + # Check pattern detection and process for different cases + if re.search(r"{", pattern): + for extension in re.split(r",|{|}", pattern): + if extension in edam_formats: + expected_ontologies_o.append((edam_formats[extension][0], extension)) + else: + if re.search(r"\.\w+$", pattern): + extension = pattern.split(".")[-1] + if extension in edam_formats: + expected_ontologies_o.append((edam_formats[extension][0], extension)) if "ontologies" in corrected_meta_yml["output"][i][ch_name][j][element_name]: for ontology in corrected_meta_yml["output"][i][ch_name][j][element_name]["ontologies"]: current_ontologies_o.append(ontology["edam"]) From cb39617266bbf95a79aacc0a8f55922e971a5198 Mon Sep 17 00:00:00 2001 From: Toni Hermoso Pulido Date: Thu, 27 Mar 2025 18:25:50 +0100 Subject: [PATCH 3/6] Refactor input and output in add_edam_ontology function --- nf_core/modules/lint/__init__.py | 103 ++++++++++++------------------- 1 file changed, 38 insertions(+), 65 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 59d959e990..5fac4a9351 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -372,83 +372,56 @@ def update_meta_yml_file(self, mod): ) break + def add_edam_ontologies(section, edam_formats, desc): + expected_ontologies = [] + current_ontologies = [] + if "pattern" in section: + pattern = section["pattern"] + # Check pattern detection and process for different cases + if re.search(r"{", pattern): + for extension in re.split(r",|{|}", pattern): + if extension in edam_formats: + expected_ontologies.append((edam_formats[extension][0], extension)) + else: + if re.search(r"\.\w+$", pattern): + extension = pattern.split(".")[-1] + if extension in edam_formats: + expected_ontologies.append((edam_formats[extension][0], extension)) + if "ontologies" in section: + for ontology in section["ontologies"]: + current_ontologies.append(ontology["edam"]) + elif section["type"] == "file": + section["ontologies"] = [] + log.debug(f"expected ontologies for input: {expected_ontologies}") + log.debug(f"current ontologies for input: {current_ontologies}") + for ontology, ext in expected_ontologies: + if ontology not in current_ontologies: + try: + section["ontologies"].append(ruamel.yaml.comments.CommentedMap({"edam": ontology})) + section[-1].yaml_add_eol_comment(f"{edam_formats[ext][1]}", "edam") + except KeyError: + log.warning(f"Could not add ontologies in {desc}") + # EDAM ontologies - # TODO: Refactor common code from input and output edam_formats = nf_core.modules.modules_utils.load_edam() if "input" in meta_yml: for i, channel in enumerate(corrected_meta_yml["input"]): for j, element in enumerate(channel): element_name = list(element.keys())[0] - expected_ontologies_i = [] - current_ontologies_i = [] - if "pattern" in corrected_meta_yml["input"][i][j][element_name]: - pattern = corrected_meta_yml["input"][i][j][element_name]["pattern"] - # Check pattern detection and process for different cases - if re.search(r"{", pattern): - for extension in re.split(r",|{|}", pattern): - if extension in edam_formats: - expected_ontologies_i.append((edam_formats[extension][0], extension)) - else: - if re.search(r"\.\w+$", pattern): - extension = pattern.split(".")[-1] - if extension in edam_formats: - expected_ontologies_i.append((edam_formats[extension][0], extension)) - if "ontologies" in corrected_meta_yml["input"][i][j][element_name]: - for ontology in corrected_meta_yml["input"][i][j][element_name]["ontologies"]: - current_ontologies_i.append(ontology["edam"]) - elif corrected_meta_yml["input"][i][j][element_name]["type"] == "file": - corrected_meta_yml["input"][i][j][element_name]["ontologies"] = [] - log.debug(f"expected ontologies for input: {expected_ontologies_i}") - log.debug(f"current ontologies for input: {current_ontologies_i}") - for ontology, ext in expected_ontologies_i: - if ontology not in current_ontologies_i: - try: - corrected_meta_yml["input"][i][j][element_name]["ontologies"].append( - ruamel.yaml.comments.CommentedMap({"edam": ontology}) - ) - corrected_meta_yml["input"][i][j][element_name]["ontologies"][-1].yaml_add_eol_comment( - f"{edam_formats[ext][1]}", "edam" - ) - except KeyError: - log.warning(f"Could not add ontologies in input: {element_name}") + add_edam_ontologies( + corrected_meta_yml["input"][i][j][element_name], edam_formats, f"input: {element_name}" + ) if "output" in meta_yml: for i, channel in enumerate(corrected_meta_yml["output"]): ch_name = list(channel.keys())[0] for j, element in enumerate(channel[ch_name]): element_name = list(element.keys())[0] - expected_ontologies_o = [] - current_ontologies_o = [] - if "pattern" in corrected_meta_yml["output"][i][ch_name][j][element_name]: - pattern = corrected_meta_yml["output"][i][ch_name][j][element_name]["pattern"] - # Check pattern detection and process for different cases - if re.search(r"{", pattern): - for extension in re.split(r",|{|}", pattern): - if extension in edam_formats: - expected_ontologies_o.append((edam_formats[extension][0], extension)) - else: - if re.search(r"\.\w+$", pattern): - extension = pattern.split(".")[-1] - if extension in edam_formats: - expected_ontologies_o.append((edam_formats[extension][0], extension)) - if "ontologies" in corrected_meta_yml["output"][i][ch_name][j][element_name]: - for ontology in corrected_meta_yml["output"][i][ch_name][j][element_name]["ontologies"]: - current_ontologies_o.append(ontology["edam"]) - elif corrected_meta_yml["output"][i][ch_name][j][element_name]["type"] == "file": - corrected_meta_yml["output"][i][ch_name][j][element_name]["ontologies"] = [] - log.debug(f"expected ontologies for output: {expected_ontologies_o}") - log.debug(f"current ontologies for output: {current_ontologies_o}") - for ontology, ext in expected_ontologies_o: - if ontology not in current_ontologies_o: - try: - corrected_meta_yml["output"][i][ch_name][j][element_name]["ontologies"].append( - ruamel.yaml.comments.CommentedMap({"edam": ontology}) - ) - corrected_meta_yml["output"][i][ch_name][j][element_name]["ontologies"][ - -1 - ].yaml_add_eol_comment(f"{edam_formats[ext][1]}", key="edam") - except KeyError: - log.warning(f"Could not add ontologies in ouput: {ch_name} - {element_name}") + add_edam_ontologies( + corrected_meta_yml["output"][i][ch_name][j][element_name], + edam_formats, + f"output : {ch_name} - {element_name}", + ) # Add bio.tools identifier for i, tool in enumerate(corrected_meta_yml["tools"]): From 2ace110ae2d143d73a25583c648036975db36f9c Mon Sep 17 00:00:00 2001 From: Toni Hermoso Pulido Date: Thu, 27 Mar 2025 18:34:52 +0100 Subject: [PATCH 4/6] Fix for warnings --- nf_core/modules/lint/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 5fac4a9351..b6f066244c 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -392,8 +392,8 @@ def add_edam_ontologies(section, edam_formats, desc): current_ontologies.append(ontology["edam"]) elif section["type"] == "file": section["ontologies"] = [] - log.debug(f"expected ontologies for input: {expected_ontologies}") - log.debug(f"current ontologies for input: {current_ontologies}") + log.debug(f"expected ontologies for {desc}: {expected_ontologies}") + log.debug(f"current ontologies for {desc}: {current_ontologies}") for ontology, ext in expected_ontologies: if ontology not in current_ontologies: try: @@ -409,7 +409,7 @@ def add_edam_ontologies(section, edam_formats, desc): for j, element in enumerate(channel): element_name = list(element.keys())[0] add_edam_ontologies( - corrected_meta_yml["input"][i][j][element_name], edam_formats, f"input: {element_name}" + corrected_meta_yml["input"][i][j][element_name], edam_formats, f"input - {element_name}" ) if "output" in meta_yml: @@ -420,7 +420,7 @@ def add_edam_ontologies(section, edam_formats, desc): add_edam_ontologies( corrected_meta_yml["output"][i][ch_name][j][element_name], edam_formats, - f"output : {ch_name} - {element_name}", + f"output - {ch_name} - {element_name}", ) # Add bio.tools identifier From 989bf7ef3f60585326fef8d475cde79e72d4b149 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 27 Mar 2025 17:39:45 +0000 Subject: [PATCH 5/6] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73a989e7d9..e40a2f823f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ - Bugfix - add back logo to the README ([#3504](https://github.com/nf-core/tools/pull/3504)) - Update dead link ([#3505](https://github.com/nf-core/tools/pull/3505)) - Changing retrieval of file extension from EDAM ([#3512](https://github.com/nf-core/tools/pull/3512)) +- Refactor adding EDAM ontologies and allowing detect more patterns (e.g., versions.yml) ([#3519](https://github.com/nf-core/tools/pull/3519)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] From e8769c003a958a3ca0c47beb35c669ff9f171db8 Mon Sep 17 00:00:00 2001 From: Toni Hermoso Pulido Date: Fri, 28 Mar 2025 11:28:33 +0100 Subject: [PATCH 6/6] Update nf_core/modules/lint/__init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit missing ontologies key Co-authored-by: Júlia Mir Pedrol --- nf_core/modules/lint/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index b6f066244c..c049fa154f 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -398,7 +398,7 @@ def add_edam_ontologies(section, edam_formats, desc): if ontology not in current_ontologies: try: section["ontologies"].append(ruamel.yaml.comments.CommentedMap({"edam": ontology})) - section[-1].yaml_add_eol_comment(f"{edam_formats[ext][1]}", "edam") + section["ontologies"][-1].yaml_add_eol_comment(f"{edam_formats[ext][1]}", "edam") except KeyError: log.warning(f"Could not add ontologies in {desc}")