From 0d6c26bcab231ac395c91dc032444b091ee5fc17 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 25 Aug 2023 10:49:05 +0300 Subject: [PATCH 01/10] Add maintenance error tester --- workers/loc.api/helpers/api-errors-testers.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/workers/loc.api/helpers/api-errors-testers.js b/workers/loc.api/helpers/api-errors-testers.js index 2b8e8272..62893f52 100644 --- a/workers/loc.api/helpers/api-errors-testers.js +++ b/workers/loc.api/helpers/api-errors-testers.js @@ -75,6 +75,11 @@ const isForbiddenError = (err) => { return /forbidden/i.test(_getErrorString(err)) } +// https://docs.bitfinex.com/docs/rest-general +const isMaintenanceError = (err) => { + return /maintenance/i.test(_getErrorString(err)) +} + const isENetError = (err) => ( isENetUnreachError(err) || isEConnResetError(err) || @@ -107,5 +112,6 @@ module.exports = { isEProtoError, isTempUnavailableError, isENetError, - isForbiddenError + isForbiddenError, + isMaintenanceError } From 1df8a357df566211b3ea95ef117b6ef26744dbea Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 25 Aug 2023 10:50:19 +0300 Subject: [PATCH 02/10] Add ability to emit maintenance is turned on by ws --- workers/loc.api/abstract.ws.event.emitter/index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/workers/loc.api/abstract.ws.event.emitter/index.js b/workers/loc.api/abstract.ws.event.emitter/index.js index 3dd6b168..37a1ae9f 100644 --- a/workers/loc.api/abstract.ws.event.emitter/index.js +++ b/workers/loc.api/abstract.ws.event.emitter/index.js @@ -35,6 +35,16 @@ class AbstractWSEventEmitter { action ) } + + emitMaintenanceTurnedOn ( + handler = () => {}, + action = 'emitMaintenanceTurnedOn' + ) { + return this.emit( + handler, + action + ) + } } decorateInjectable(AbstractWSEventEmitter) From 8fd69e0ee3314be2c0fdd38ab386868bed05468e Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 25 Aug 2023 10:50:55 +0300 Subject: [PATCH 03/10] Add ability to emit maintenance is turned off by ws --- workers/loc.api/abstract.ws.event.emitter/index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/workers/loc.api/abstract.ws.event.emitter/index.js b/workers/loc.api/abstract.ws.event.emitter/index.js index 37a1ae9f..2845d0f2 100644 --- a/workers/loc.api/abstract.ws.event.emitter/index.js +++ b/workers/loc.api/abstract.ws.event.emitter/index.js @@ -45,6 +45,16 @@ class AbstractWSEventEmitter { action ) } + + emitMaintenanceTurnedOff ( + handler = () => {}, + action = 'emitMaintenanceTurnedOff' + ) { + return this.emit( + handler, + action + ) + } } decorateInjectable(AbstractWSEventEmitter) From c3a5d6ada04833319ff78b1969034136aba93fd4 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 25 Aug 2023 12:46:03 +0300 Subject: [PATCH 04/10] Emit maintenance turned on event if error occurs --- workers/loc.api/responder/index.js | 45 +++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/workers/loc.api/responder/index.js b/workers/loc.api/responder/index.js index 2acb0950..a8e5817b 100644 --- a/workers/loc.api/responder/index.js +++ b/workers/loc.api/responder/index.js @@ -8,7 +8,8 @@ const { isNonceSmallError, isUserIsNotMerchantError, isSymbolInvalidError, - isForbiddenError + isForbiddenError, + isMaintenanceError } = require('../helpers') const { @@ -184,7 +185,6 @@ const _getErrorMetadata = (args, err) => { const _logError = (loggerArgs, err) => { const { logger, - wsEventEmitter, args, name, isInternalRequest @@ -198,8 +198,33 @@ const _logError = (loggerArgs, err) => { error } = _getErrorMetadata(args, err) + _emitEventByWs(loggerArgs, error) + + if ( + code !== 500 || + shouldNotBeLoggedToStdErrorStream + ) { + logger.debug(_prepareErrorData(error, name)) + + return + } + + logger.error(_prepareErrorData(error, name)) +} + +const _emitEventByWs = (emitterArgs, error) => { + const { + logger, + wsEventEmitter, + args, + name + } = emitterArgs ?? {} + + if (!(wsEventEmitter instanceof AbstractWSEventEmitter)) { + return + } + if ( - wsEventEmitter instanceof AbstractWSEventEmitter && args?.auth?.authToken && ( error instanceof AuthError || @@ -217,17 +242,11 @@ const _logError = (loggerArgs, err) => { logger.error(_prepareErrorData(err, name)) }) } - - if ( - code !== 500 || - shouldNotBeLoggedToStdErrorStream - ) { - logger.debug(_prepareErrorData(error, name)) - - return + if (isMaintenanceError(error)) { + wsEventEmitter.emitMaintenanceTurnedOn().then(() => {}, (err) => { + logger.error(_prepareErrorData(err, name)) + }) } - - logger.error(_prepareErrorData(error, name)) } /* From c4c62d5cda7ad45c500256420d8f956b81f67175 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 25 Aug 2023 13:06:14 +0300 Subject: [PATCH 05/10] Export isMaintenanceError helper --- workers/loc.api/helpers/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workers/loc.api/helpers/index.js b/workers/loc.api/helpers/index.js index f8dd8ad3..894a52e0 100644 --- a/workers/loc.api/helpers/index.js +++ b/workers/loc.api/helpers/index.js @@ -37,7 +37,8 @@ const { isENotFoundError, isESocketTimeoutError, isENetError, - isForbiddenError + isForbiddenError, + isMaintenanceError } = require('./api-errors-testers') const { accountCache, @@ -82,6 +83,7 @@ module.exports = { isESocketTimeoutError, isENetError, isForbiddenError, + isMaintenanceError, accountCache, parseFields, parseLoginsExtraDataFields, From 77b4476ef63a6ddaa40c9fc4021868928e9eb745 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 25 Sep 2023 11:35:49 +0300 Subject: [PATCH 06/10] Remove lodash.transform usage --- workers/loc.api/helpers/utils.js | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/workers/loc.api/helpers/utils.js b/workers/loc.api/helpers/utils.js index 26d65a42..543637b5 100644 --- a/workers/loc.api/helpers/utils.js +++ b/workers/loc.api/helpers/utils.js @@ -1,6 +1,5 @@ 'use strict' -const { transform } = require('lodash') const LRU = require('lru') const accountCache = new LRU({ maxAge: 900000, max: 1 }) @@ -8,15 +7,33 @@ const accountCache = new LRU({ maxAge: 900000, max: 1 }) const parseFields = (res, opts) => { const { executed, rate } = opts - return transform(res, (result, obj) => { + if ( + !Array.isArray(res) || + res.length === 0 + ) { + return res + } + + return res.reduce((accum, curr) => { + if ( + !curr || + typeof curr !== 'object' + ) { + accum.push(curr) + + return accum + } + if (executed) { - obj.amountExecuted = obj.amountOrig - obj.amount + curr.amountExecuted = curr.amountOrig - curr.amount } if (rate) { - obj.rate = obj.rate || 'Flash Return Rate' + curr.rate = curr.rate ?? 'Flash Return Rate' } - result.push(obj) + accum.push(curr) + + return accum }, []) } From f15fe58c8495f79e359e1baf64dd71fe744f1414 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 28 Sep 2023 13:03:33 +0300 Subject: [PATCH 07/10] Add stream writer helper for complicated csv --- .../generate-csv/csv-writer/helpers/index.js | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 workers/loc.api/generate-csv/csv-writer/helpers/index.js diff --git a/workers/loc.api/generate-csv/csv-writer/helpers/index.js b/workers/loc.api/generate-csv/csv-writer/helpers/index.js new file mode 100644 index 00000000..91b2cf8f --- /dev/null +++ b/workers/loc.api/generate-csv/csv-writer/helpers/index.js @@ -0,0 +1,42 @@ +'use strict' + +const { pipeline } = require('stream/promises') +const { stringify } = require('csv') + +const streamWriterToOne = async ( + rStream, + wStream, + writeFn, + opts +) => { + const { end = true } = opts ?? {} + const promise = pipeline(rStream, wStream, { end }) + + writeFn(rStream) + rStream.end() + + await promise +} + +const streamWriter = async (wStream, csvStreamDataMap) => { + for (const [i, csvStreamData] of csvStreamDataMap.entries()) { + const isLast = (i + 1) === csvStreamDataMap.length + const { + columnParams, + writeFn + } = csvStreamData + + const stringifier = stringify(columnParams) + await streamWriterToOne( + stringifier, + wStream, + writeFn, + { end: isLast } + ) + } +} + +module.exports = { + streamWriterToOne, + streamWriter +} From b1c394d2a396f2aacf1d819b0ac6b7369b4d7671 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 28 Sep 2023 13:05:24 +0300 Subject: [PATCH 08/10] Rework weighted-averages-report-csv-writer to prevent MaxListenersExceededWarning --- .../weighted-averages-report-csv-writer.js | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js b/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js index 86960164..15134bdf 100644 --- a/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js +++ b/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js @@ -1,13 +1,10 @@ 'use strict' -const { pipeline } = require('stream') -const { stringify } = require('csv') - const { write } = require('../../queue/write-data-to-stream/helpers') -const nope = () => {} +const { streamWriter } = require('./helpers') module.exports = ( rService, @@ -28,50 +25,53 @@ module.exports = ( queue.emit('progress', 0) if (typeof jobData === 'string') { - const stringifier = stringify( - { columns: ['mess'] } + await streamWriter( + wStream, + [{ + columnParams: { columns: ['mess'] }, + writeFn: (stream) => write([{ mess: jobData }], stream) + }] ) - pipeline(stringifier, wStream, nope) - write([{ mess: jobData }], stringifier) queue.emit('progress', 100) - stringifier.end() return } - wStream.setMaxListeners(50) - - const headerStringifier = stringify( - { columns: ['empty', 'buy', 'empty', 'empty', 'sell', 'empty', 'empty', 'cumulative', 'empty'] } - ) - const resStringifier = stringify({ - header: true, - columns: columnsCsv - }) - - pipeline(headerStringifier, wStream, nope) - pipeline(resStringifier, wStream, nope) - const { res } = await getDataFromApi({ getData: rService[name].bind(rService), args, callerName: 'CSV_WRITER' }) - write( - [{ empty: '', buy: 'Buy', sell: 'Sell', cumulative: 'Cumulative' }], - headerStringifier - ) - write( - res, - resStringifier, - formatSettings, - params + wStream.setMaxListeners(50) + + await streamWriter( + wStream, + [ + { + columnParams: { + columns: ['empty', 'buy', 'empty', 'empty', 'sell', 'empty', 'empty', 'cumulative', 'empty'] + }, + writeFn: (stream) => write( + [{ empty: '', buy: 'Buy', sell: 'Sell', cumulative: 'Cumulative' }], + stream + ) + }, + { + columnParams: { + header: true, + columns: columnsCsv + }, + writeFn: (stream) => write( + res, + stream, + formatSettings, + params + ) + } + ] ) queue.emit('progress', 100) - - headerStringifier.end() - resStringifier.end() } From 51674c706e4f01eea32844cb9aa9a45590dad27d Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 28 Sep 2023 13:28:20 +0300 Subject: [PATCH 09/10] Fix params schema for weighted averages report csv --- workers/loc.api/generate-csv/csv.job.data.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/loc.api/generate-csv/csv.job.data.js b/workers/loc.api/generate-csv/csv.job.data.js index 9be0f766..963388bb 100644 --- a/workers/loc.api/generate-csv/csv.job.data.js +++ b/workers/loc.api/generate-csv/csv.job.data.js @@ -1038,7 +1038,7 @@ class CsvJobData { uId, uInfo ) { - checkParams(args, 'paramsSchemaForWeightedAveragesReportApiCsv') + checkParams(args, 'paramsSchemaForWeightedAveragesReportApiCsv', ['symbol']) const { userId, From 2b94889db5fb5d7cb8c9ccbd5abd8a8bfd3465eb Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Wed, 4 Oct 2023 09:40:39 +0300 Subject: [PATCH 10/10] Bump version up to 4.7.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 51a28b9c..3e0baef5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bfx-report", - "version": "4.6.0", + "version": "4.7.0", "description": "Reporting tool", "main": "worker.js", "license": "Apache-2.0",