From be9d7681a1b483dacf8ceeb9953d33b03cdf7868 Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Tue, 13 May 2025 14:56:49 +0200 Subject: [PATCH 1/2] exit code for unrecoverable snapshots errors When a pipeline process is in the process of being snapshotted via Fusion Snapshots and encounters an unrecoverable error the Fusion Snapshots runtime emits a specific exit code (175) to signal that to Nextflow. This PR makes sure that users creating pipelines out of the template are including the exit code in their retry strategy. --- CHANGELOG.md | 1 + nf_core/pipeline-template/conf/base.config | 2 +- nf_core/pipeline-template/nextflow.config | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42a6faa9df..a847342b6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - run nf-test tests on runsOn runners ([#3525](https://github.com/nf-core/tools/pull/3525)) - Include the centralized nf-core configs also in offline mode, if a local copy is available. ([#3491](https://github.com/nf-core/tools/pull/3491)) - downgrade nf-schema to fix CI tests ([#3544](https://github.com/nf-core/tools/pull/3544)) +- Make jobs automatically resubmit for exit code 175 ([#3564](https://github.com/nf-core/tools/pull/3564)) ### Linting diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index 9f51eb7cd5..9e582457f1 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -15,7 +15,7 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3e0e281073..5871167985 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -83,7 +83,7 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' } From 41550ccb54e52cb341a330ed04620dcfc2f5493e Mon Sep 17 00:00:00 2001 From: Lorenzo Fontana Date: Wed, 14 May 2025 14:12:50 +0200 Subject: [PATCH 2/2] Comment to specify what the 175 exit code stands for. --- nf_core/pipeline-template/nextflow.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 5871167985..4c202d85a0 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -83,6 +83,8 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } + // 175 signals that the Pipeline had an unrecoverable error while + // restoring a Snapshot via Fusion Snapshots. errorStrategy = { task.exitStatus in ((130..145) + 104 + 175) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1'