From 7528351d463b0e210fe8f78126624ba4d2b5b26c Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Wed, 29 Jan 2025 15:37:17 +0200 Subject: [PATCH 01/10] Add backoff and jitter for bfx-api requests in case rate limit --- workers/loc.api/helpers/get-data-from-api.js | 52 ++++++++++++++++++-- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/workers/loc.api/helpers/get-data-from-api.js b/workers/loc.api/helpers/get-data-from-api.js index 0f5f8b00..54ce3299 100644 --- a/workers/loc.api/helpers/get-data-from-api.js +++ b/workers/loc.api/helpers/get-data-from-api.js @@ -12,6 +12,40 @@ const { isAuthError } = require('./api-errors-testers') +const _getRandomInt = (min, max) => { + const minCeiled = Math.ceil(min) + const maxFloored = Math.floor(max) + + return Math.floor(Math.random() * (maxFloored - minCeiled) + minCeiled) +} + +/** + * Decorrelated Jitter implementation + * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + */ +const _calcBackOffAndJitteredDelay = (opts) => { + const { + startingDelayMs = 80 * 1_000, + maxDelayMs = 5 * 60 * 1_000, + timeMultiple = 1.3, + prevBackOffDelayMs = 0, + numOfDelayedAttempts = 1 + } = opts ?? {} + + const startingDelayShifterMs = 5_000 * numOfDelayedAttempts + const _startingDelayMs = startingDelayMs + startingDelayShifterMs + const calcedDelay = prevBackOffDelayMs * timeMultiple + + if (calcedDelay < _startingDelayMs) { + return startingDelayMs + } + + const jitteredDelay = _getRandomInt(_startingDelayMs, calcedDelay) + const limitedDelay = Math.min(maxDelayMs, jitteredDelay) + + return limitedDelay +} + const _delay = (mc = 80000, interrupter) => { if (_isInterrupted(interrupter)) { return Promise.resolve({ isInterrupted: true }) @@ -62,16 +96,16 @@ module.exports = ( middlewareParams, callerName, eNetErrorAttemptsTimeframeMin = 10, // min - eNetErrorAttemptsTimeoutMs = 10000, // ms + eNetErrorAttemptsTimeoutMs = 10_000, // ms shouldNotInterrupt, - interrupter + interrupter, + backOffOpts }) => { const _interrupter = shouldNotInterrupt ? null : interrupter ?? commonInterrupter - const ms = 80000 - + let prevBackOffDelayMs = 0 let countNetError = 0 let countRateLimitError = 0 let countNonceSmallError = 0 @@ -121,7 +155,15 @@ module.exports = ( throw err } - const { isInterrupted } = await _delay(ms, _interrupter) + const delay = _calcBackOffAndJitteredDelay({ + startingDelayMs: 80_000, + maxDelayMs: 3 * 60 * 1_000, + ...backOffOpts, + prevBackOffDelayMs, + numOfDelayedAttempts: countRateLimitError + }) + prevBackOffDelayMs = delay + const { isInterrupted } = await _delay(delay, _interrupter) if (isInterrupted) { return { isInterrupted } From f08ae75e09cdf8072f7068b370fb19f8e63ee19a Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 30 Jan 2025 12:18:14 +0200 Subject: [PATCH 02/10] Add get-random-int helper --- .../helpers/get-data-from-api/helpers/get-random-int.js | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 workers/loc.api/helpers/get-data-from-api/helpers/get-random-int.js diff --git a/workers/loc.api/helpers/get-data-from-api/helpers/get-random-int.js b/workers/loc.api/helpers/get-data-from-api/helpers/get-random-int.js new file mode 100644 index 00000000..697ba2dc --- /dev/null +++ b/workers/loc.api/helpers/get-data-from-api/helpers/get-random-int.js @@ -0,0 +1,8 @@ +'use strict' + +module.exports = (min, max) => { + const minCeiled = Math.ceil(min) + const maxFloored = Math.floor(max) + + return Math.floor(Math.random() * (maxFloored - minCeiled) + minCeiled) +} From f310b1cdf4b0cfe0552059961cb8c7b5743e4f21 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 30 Jan 2025 12:18:45 +0200 Subject: [PATCH 03/10] Add calc-back-off-and-jittered-delay helper --- .../calc-back-off-and-jittered-delay.js | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 workers/loc.api/helpers/get-data-from-api/helpers/calc-back-off-and-jittered-delay.js diff --git a/workers/loc.api/helpers/get-data-from-api/helpers/calc-back-off-and-jittered-delay.js b/workers/loc.api/helpers/get-data-from-api/helpers/calc-back-off-and-jittered-delay.js new file mode 100644 index 00000000..a75b66bc --- /dev/null +++ b/workers/loc.api/helpers/get-data-from-api/helpers/calc-back-off-and-jittered-delay.js @@ -0,0 +1,30 @@ +'use strict' + +const getRandomInt = require('./get-random-int') + +/** + * Decorrelated Jitter implementation + * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + */ +module.exports = (opts) => { + const { + startingDelayMs = 80 * 1_000, + maxDelayMs = 5 * 60 * 1_000, + timeMultiple = 1.3, + prevBackOffDelayMs = 0, + numOfDelayedAttempts = 1 + } = opts ?? {} + + const startingDelayShifterMs = 5_000 * numOfDelayedAttempts + const _startingDelayMs = startingDelayMs + startingDelayShifterMs + const calcedDelay = prevBackOffDelayMs * timeMultiple + + if (calcedDelay < _startingDelayMs) { + return startingDelayMs + } + + const jitteredDelay = getRandomInt(_startingDelayMs, calcedDelay) + const limitedDelay = Math.min(maxDelayMs, jitteredDelay) + + return limitedDelay +} From 1ade54506eb5bb3eafb7fe3b82d6a1fe8a413d57 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 30 Jan 2025 12:19:31 +0200 Subject: [PATCH 04/10] Add is-interrupted helper --- .../get-data-from-api/helpers/is-interrupted.js | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 workers/loc.api/helpers/get-data-from-api/helpers/is-interrupted.js diff --git a/workers/loc.api/helpers/get-data-from-api/helpers/is-interrupted.js b/workers/loc.api/helpers/get-data-from-api/helpers/is-interrupted.js new file mode 100644 index 00000000..01f86a6f --- /dev/null +++ b/workers/loc.api/helpers/get-data-from-api/helpers/is-interrupted.js @@ -0,0 +1,10 @@ +'use strict' + +const Interrupter = require('../../../interrupter') + +module.exports = (interrupter) => { + return ( + interrupter instanceof Interrupter && + interrupter.hasInterrupted() + ) +} From 51064c7d2347f773a25236eb2abefd5e46c09299 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 30 Jan 2025 12:20:01 +0200 Subject: [PATCH 05/10] Add delay helper --- .../get-data-from-api/helpers/delay.js | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 workers/loc.api/helpers/get-data-from-api/helpers/delay.js diff --git a/workers/loc.api/helpers/get-data-from-api/helpers/delay.js b/workers/loc.api/helpers/get-data-from-api/helpers/delay.js new file mode 100644 index 00000000..2df2d3b0 --- /dev/null +++ b/workers/loc.api/helpers/get-data-from-api/helpers/delay.js @@ -0,0 +1,33 @@ +'use strict' + +const Interrupter = require('../../../interrupter') +const isInterrupted = require('./is-interrupted') + +module.exports = (mc = 80000, interrupter) => { + if (isInterrupted(interrupter)) { + return Promise.resolve({ isInterrupted: true }) + } + + return new Promise((resolve) => { + const hasInterrupter = interrupter instanceof Interrupter + const timeout = setTimeout(() => { + if (hasInterrupter) { + interrupter.offInterrupt(onceInterruptHandler) + } + + resolve({ isInterrupted: false }) + }, mc) + const onceInterruptHandler = () => { + if (!timeout.hasRef()) { + return + } + + clearTimeout(timeout) + resolve({ isInterrupted: true }) + } + + if (hasInterrupter) { + interrupter.onceInterrupt(onceInterruptHandler) + } + }) +} From b451b938d06d265ccf21c0506bea22219f5fdf36 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 30 Jan 2025 12:20:33 +0200 Subject: [PATCH 06/10] Add get-empty-arr-res helper --- .../helpers/get-data-from-api/helpers/get-empty-arr-res.js | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 workers/loc.api/helpers/get-data-from-api/helpers/get-empty-arr-res.js diff --git a/workers/loc.api/helpers/get-data-from-api/helpers/get-empty-arr-res.js b/workers/loc.api/helpers/get-data-from-api/helpers/get-empty-arr-res.js new file mode 100644 index 00000000..b72ccd42 --- /dev/null +++ b/workers/loc.api/helpers/get-data-from-api/helpers/get-empty-arr-res.js @@ -0,0 +1,5 @@ +'use strict' + +module.exports = () => { + return { jsonrpc: '2.0', result: [], id: null } +} From 74464178d685cdc9d2377112d415ef8ccbfad814 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 30 Jan 2025 12:21:43 +0200 Subject: [PATCH 07/10] Move get-data-from-api module to decomposited folder --- .../index.js} | 99 +++---------------- 1 file changed, 15 insertions(+), 84 deletions(-) rename workers/loc.api/helpers/{get-data-from-api.js => get-data-from-api/index.js} (57%) diff --git a/workers/loc.api/helpers/get-data-from-api.js b/workers/loc.api/helpers/get-data-from-api/index.js similarity index 57% rename from workers/loc.api/helpers/get-data-from-api.js rename to workers/loc.api/helpers/get-data-from-api/index.js index 54ce3299..4887a752 100644 --- a/workers/loc.api/helpers/get-data-from-api.js +++ b/workers/loc.api/helpers/get-data-from-api/index.js @@ -2,89 +2,20 @@ const { cloneDeep } = require('lib-js-util-base') -const Interrupter = require('../interrupter') -const AbstractWSEventEmitter = require('../abstract.ws.event.emitter') +const AbstractWSEventEmitter = require('../../abstract.ws.event.emitter') const { isRateLimitError, isNonceSmallError, isUserIsNotMerchantError, isENetError, isAuthError -} = require('./api-errors-testers') - -const _getRandomInt = (min, max) => { - const minCeiled = Math.ceil(min) - const maxFloored = Math.floor(max) - - return Math.floor(Math.random() * (maxFloored - minCeiled) + minCeiled) -} - -/** - * Decorrelated Jitter implementation - * https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ - */ -const _calcBackOffAndJitteredDelay = (opts) => { - const { - startingDelayMs = 80 * 1_000, - maxDelayMs = 5 * 60 * 1_000, - timeMultiple = 1.3, - prevBackOffDelayMs = 0, - numOfDelayedAttempts = 1 - } = opts ?? {} - - const startingDelayShifterMs = 5_000 * numOfDelayedAttempts - const _startingDelayMs = startingDelayMs + startingDelayShifterMs - const calcedDelay = prevBackOffDelayMs * timeMultiple - - if (calcedDelay < _startingDelayMs) { - return startingDelayMs - } - - const jitteredDelay = _getRandomInt(_startingDelayMs, calcedDelay) - const limitedDelay = Math.min(maxDelayMs, jitteredDelay) - - return limitedDelay -} - -const _delay = (mc = 80000, interrupter) => { - if (_isInterrupted(interrupter)) { - return Promise.resolve({ isInterrupted: true }) - } - - return new Promise((resolve) => { - const hasInterrupter = interrupter instanceof Interrupter - const timeout = setTimeout(() => { - if (hasInterrupter) { - interrupter.offInterrupt(onceInterruptHandler) - } - - resolve({ isInterrupted: false }) - }, mc) - const onceInterruptHandler = () => { - if (!timeout.hasRef()) { - return - } - - clearTimeout(timeout) - resolve({ isInterrupted: true }) - } - - if (hasInterrupter) { - interrupter.onceInterrupt(onceInterruptHandler) - } - }) -} - -const _isInterrupted = (interrupter) => { - return ( - interrupter instanceof Interrupter && - interrupter.hasInterrupted() - ) -} - -const _getEmptyArrRes = () => { - return { jsonrpc: '2.0', result: [], id: null } -} +} = require('../api-errors-testers') +const { + calcBackOffAndJitteredDelay, + isInterrupted: _isInterrupted, + delay, + getEmptyArrRes +} = require('./helpers') module.exports = ( commonInterrupter, @@ -146,7 +77,7 @@ module.exports = ( break } catch (err) { if (isUserIsNotMerchantError(err)) { - return _getEmptyArrRes() + return getEmptyArrRes() } if (isRateLimitError(err)) { countRateLimitError += 1 @@ -155,15 +86,15 @@ module.exports = ( throw err } - const delay = _calcBackOffAndJitteredDelay({ + const delayMs = calcBackOffAndJitteredDelay({ startingDelayMs: 80_000, - maxDelayMs: 3 * 60 * 1_000, + maxDelayMs: 5 * 60 * 1_000, ...backOffOpts, prevBackOffDelayMs, numOfDelayedAttempts: countRateLimitError }) prevBackOffDelayMs = delay - const { isInterrupted } = await _delay(delay, _interrupter) + const { isInterrupted } = await delay(delayMs, _interrupter) if (isInterrupted) { return { isInterrupted } @@ -178,7 +109,7 @@ module.exports = ( throw err } - const { isInterrupted } = await _delay(1000, _interrupter) + const { isInterrupted } = await delay(1000, _interrupter) if (isInterrupted) { return { isInterrupted } @@ -206,7 +137,7 @@ module.exports = ( const { isInterrupted - } = await _delay(eNetErrorAttemptsTimeoutMs, _interrupter) + } = await delay(eNetErrorAttemptsTimeoutMs, _interrupter) if (isInterrupted) { return { isInterrupted } @@ -225,7 +156,7 @@ module.exports = ( throw err } - const { isInterrupted } = await _delay(10000, _interrupter) + const { isInterrupted } = await delay(10000, _interrupter) if (isInterrupted) { return { isInterrupted } From 629ffb6fc681506af7f877d6b51530b60bd93263 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 30 Jan 2025 12:25:50 +0200 Subject: [PATCH 08/10] Add common entrypoint for helpers --- .../helpers/get-data-from-api/helpers/index.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 workers/loc.api/helpers/get-data-from-api/helpers/index.js diff --git a/workers/loc.api/helpers/get-data-from-api/helpers/index.js b/workers/loc.api/helpers/get-data-from-api/helpers/index.js new file mode 100644 index 00000000..c95de1f2 --- /dev/null +++ b/workers/loc.api/helpers/get-data-from-api/helpers/index.js @@ -0,0 +1,17 @@ +'use strict' + +const getRandomInt = require('./get-random-int') +const calcBackOffAndJitteredDelay = require( + './calc-back-off-and-jittered-delay' +) +const delay = require('./delay') +const isInterrupted = require('./is-interrupted') +const getEmptyArrRes = require('./get-empty-arr-res') + +module.exports = { + getRandomInt, + calcBackOffAndJitteredDelay, + delay, + isInterrupted, + getEmptyArrRes +} From 6d381adec49cef07d6ff6c752b929af5c53450dd Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 30 Jan 2025 13:07:48 +0200 Subject: [PATCH 09/10] Fix backoff delay ms --- workers/loc.api/helpers/get-data-from-api/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/loc.api/helpers/get-data-from-api/index.js b/workers/loc.api/helpers/get-data-from-api/index.js index 4887a752..145e6358 100644 --- a/workers/loc.api/helpers/get-data-from-api/index.js +++ b/workers/loc.api/helpers/get-data-from-api/index.js @@ -93,7 +93,7 @@ module.exports = ( prevBackOffDelayMs, numOfDelayedAttempts: countRateLimitError }) - prevBackOffDelayMs = delay + prevBackOffDelayMs = delayMs const { isInterrupted } = await delay(delayMs, _interrupter) if (isInterrupted) { From 5bf614d7cdfe7bfcbc23b6f159a735a82aa5abb1 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 31 Jan 2025 10:19:13 +0200 Subject: [PATCH 10/10] Optimize back off algorithm --- .../helpers/calc-back-off-and-jittered-delay.js | 17 +++++++---------- .../loc.api/helpers/get-data-from-api/index.js | 3 +-- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/workers/loc.api/helpers/get-data-from-api/helpers/calc-back-off-and-jittered-delay.js b/workers/loc.api/helpers/get-data-from-api/helpers/calc-back-off-and-jittered-delay.js index a75b66bc..9df9790b 100644 --- a/workers/loc.api/helpers/get-data-from-api/helpers/calc-back-off-and-jittered-delay.js +++ b/workers/loc.api/helpers/get-data-from-api/helpers/calc-back-off-and-jittered-delay.js @@ -10,20 +10,17 @@ module.exports = (opts) => { const { startingDelayMs = 80 * 1_000, maxDelayMs = 5 * 60 * 1_000, - timeMultiple = 1.3, - prevBackOffDelayMs = 0, - numOfDelayedAttempts = 1 + startingTimeMultiplier = 1.2, + endingTimeMultiplier = 1.5, + prevBackOffDelayMs = 0 } = opts ?? {} - const startingDelayShifterMs = 5_000 * numOfDelayedAttempts - const _startingDelayMs = startingDelayMs + startingDelayShifterMs - const calcedDelay = prevBackOffDelayMs * timeMultiple + const prevDelayNotLessStarting = Math.max(startingDelayMs, prevBackOffDelayMs) - if (calcedDelay < _startingDelayMs) { - return startingDelayMs - } + const calcedStar = prevDelayNotLessStarting * startingTimeMultiplier + const calcedEnd = prevDelayNotLessStarting * endingTimeMultiplier - const jitteredDelay = getRandomInt(_startingDelayMs, calcedDelay) + const jitteredDelay = getRandomInt(calcedStar, calcedEnd) const limitedDelay = Math.min(maxDelayMs, jitteredDelay) return limitedDelay diff --git a/workers/loc.api/helpers/get-data-from-api/index.js b/workers/loc.api/helpers/get-data-from-api/index.js index 145e6358..61ce7c87 100644 --- a/workers/loc.api/helpers/get-data-from-api/index.js +++ b/workers/loc.api/helpers/get-data-from-api/index.js @@ -90,8 +90,7 @@ module.exports = ( startingDelayMs: 80_000, maxDelayMs: 5 * 60 * 1_000, ...backOffOpts, - prevBackOffDelayMs, - numOfDelayedAttempts: countRateLimitError + prevBackOffDelayMs }) prevBackOffDelayMs = delayMs const { isInterrupted } = await delay(delayMs, _interrupter)