From 4ca8ad0ee9d530f34508bd2c3654312b4ad737ae Mon Sep 17 00:00:00 2001 From: Nikolay Edigaryev Date: Fri, 9 Aug 2024 22:24:25 +0400 Subject: [PATCH 1/2] tart push: re-try when encountering errors when pushing disk layers --- Sources/tart/OCI/Layerizer/DiskV2.swift | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Sources/tart/OCI/Layerizer/DiskV2.swift b/Sources/tart/OCI/Layerizer/DiskV2.swift index 7c2217ec..d359d8cb 100644 --- a/Sources/tart/OCI/Layerizer/DiskV2.swift +++ b/Sources/tart/OCI/Layerizer/DiskV2.swift @@ -1,6 +1,7 @@ import Foundation import Compression import System +import Retry class DiskV2: Disk { private static let bufferSizeBytes = 4 * 1024 * 1024 @@ -28,8 +29,18 @@ class DiskV2: Disk { let compressedData = try (data as NSData).compressed(using: .lz4) as Data let compressedDataDigest = Digest.hash(compressedData) - if try await !registry.blobExists(compressedDataDigest) { - _ = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb, digest: compressedDataDigest) + try await retry(maxAttempts: 5, backoff: .exponentialWithFullJitter(baseDelay: .seconds(5), maxDelay: .seconds(60))) { + if try await !registry.blobExists(compressedDataDigest) { + _ = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb, digest: compressedDataDigest) + } + } recoverFromFailure: { error in + if error is RuntimeError { + return .throw + } + + print("Error \"\(error.localizedDescription)\" while uploading disk layer \(index), attempting to re-try...") + + return .retry } // Update progress using a relative value From 4191fefe5c340cdade9bc96351387b6d8eee4122 Mon Sep 17 00:00:00 2001 From: Nikolay Edigaryev Date: Sat, 10 Aug 2024 12:06:41 +0400 Subject: [PATCH 2/2] Only re-try on URLError --- Sources/tart/OCI/Layerizer/DiskV2.swift | 11 ++++++----- Sources/tart/VMStorageOCI.swift | 12 ++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Sources/tart/OCI/Layerizer/DiskV2.swift b/Sources/tart/OCI/Layerizer/DiskV2.swift index d359d8cb..7a9e346e 100644 --- a/Sources/tart/OCI/Layerizer/DiskV2.swift +++ b/Sources/tart/OCI/Layerizer/DiskV2.swift @@ -34,13 +34,14 @@ class DiskV2: Disk { _ = try await registry.pushBlob(fromData: compressedData, chunkSizeMb: chunkSizeMb, digest: compressedDataDigest) } } recoverFromFailure: { error in - if error is RuntimeError { - return .throw - } + if error is URLError { + print("Error: \(error.localizedDescription)") + print("Attempting to re-try...") - print("Error \"\(error.localizedDescription)\" while uploading disk layer \(index), attempting to re-try...") + return .retry + } - return .retry + return .throw } // Update progress using a relative value diff --git a/Sources/tart/VMStorageOCI.swift b/Sources/tart/VMStorageOCI.swift index 6be15080..25283d64 100644 --- a/Sources/tart/VMStorageOCI.swift +++ b/Sources/tart/VMStorageOCI.swift @@ -208,14 +208,14 @@ class VMStorageOCI: PrunableStorage { try await tmpVMDir.pullFromRegistry(registry: registry, manifest: manifest, concurrency: concurrency, localLayerCache: localLayerCache) } recoverFromFailure: { error in - if error is RuntimeError { - return .throw - } + if error is Retryable { + print("Error: \(error.localizedDescription)") + print("Attempting to re-try...") - print("Error: \(error.localizedDescription)") - print("Attempting to re-try...") + return .retry + } - return .retry + return .throw } try move(digestName, from: tmpVMDir) transaction.finish()