From a5b89795243e4bad08edfc6cb5aadbade89c4fd2 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 24 Mar 2023 11:59:58 +0200 Subject: [PATCH 01/13] Add params schema for weighted averages report --- workers/loc.api/helpers/schema.js | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/workers/loc.api/helpers/schema.js b/workers/loc.api/helpers/schema.js index 61dc6529..1178cf65 100644 --- a/workers/loc.api/helpers/schema.js +++ b/workers/loc.api/helpers/schema.js @@ -1,5 +1,7 @@ 'use strict' +const { cloneDeep } = require('lodash') + const _publicTradesSymbol = { type: ['string', 'array'], if: { @@ -129,6 +131,21 @@ const paramsSchemaForCandlesApi = { } } +const paramsSchemaForWeightedAveragesReportApi = { + type: 'object', + properties: { + start: { + type: 'integer' + }, + end: { + type: 'integer' + }, + symbol: { + type: ['string', 'array'] + } + } +} + const paramsSchemaForCandlesCsv = { type: 'object', properties: { @@ -321,6 +338,16 @@ const paramsSchemaForOrderTradesCsv = { } } +const paramsSchemaForWeightedAveragesReportApiCsv = { + type: 'object', + properties: { + ...cloneDeep(paramsSchemaForWeightedAveragesReportApi.properties), + timezone, + dateFormat, + language + } +} + module.exports = { paramsSchemaForApi, paramsSchemaForCsv, @@ -339,5 +366,7 @@ module.exports = { paramsSchemaForStatusMessagesApi, paramsSchemaForStatusMessagesCsv, paramsSchemaForCandlesApi, - paramsSchemaForCandlesCsv + paramsSchemaForCandlesCsv, + paramsSchemaForWeightedAveragesReportApi, + paramsSchemaForWeightedAveragesReportApiCsv } From ce1582124eaa7558e28f37d8d019e3bc57db045f Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 24 Mar 2023 12:02:51 +0200 Subject: [PATCH 02/13] Add WeightedAveragesReportCsvWriter service --- .../loc.api/generate-csv/csv-writer/index.js | 9 +++ .../weighted-averages-report-csv-writer.js | 77 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 workers/loc.api/generate-csv/csv-writer/index.js create mode 100644 workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js diff --git a/workers/loc.api/generate-csv/csv-writer/index.js b/workers/loc.api/generate-csv/csv-writer/index.js new file mode 100644 index 00000000..d21f9186 --- /dev/null +++ b/workers/loc.api/generate-csv/csv-writer/index.js @@ -0,0 +1,9 @@ +'use strict' + +const weightedAveragesReportCsvWriter = require( + './weighted-averages-report-csv-writer' +) + +module.exports = { + weightedAveragesReportCsvWriter +} diff --git a/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js b/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js new file mode 100644 index 00000000..3168420e --- /dev/null +++ b/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js @@ -0,0 +1,77 @@ +'use strict' + +const { pipeline } = require('stream') +const { stringify } = require('csv') + +const { + write +} = require('../../queue/write-data-to-stream/helpers') + +const nope = () => {} + +module.exports = ( + rService, + getDataFromApi +) => async ( + wStream, + jobData +) => { + const queue = rService.ctx.lokue_aggregator.q + const { + args, + columnsCsv, + formatSettings, + name + } = jobData ?? {} + const { params } = args ?? {} + + queue.emit('progress', 0) + + if (typeof jobData === 'string') { + const stringifier = stringify( + { columns: ['mess'] } + ) + + pipeline(stringifier, wStream, nope) + write([{ mess: jobData }], stringifier) + queue.emit('progress', 100) + stringifier.end() + + return + } + + wStream.setMaxListeners(50) + + const headerStringifier = stringify( + { columns: ['empty', 'buy', 'empty', 'sell', 'empty', 'cumulative', 'empty'] } + ) + const resStringifier = stringify({ + header: true, + columns: columnsCsv + }) + + pipeline(headerStringifier, wStream, nope) + pipeline(resStringifier, wStream, nope) + + const res = await getDataFromApi({ + getData: rService[name].bind(rService), + args, + callerName: 'CSV_WRITER' + }) + + write( + [{ empty: '', buy: 'Buy', sell: 'Sell', cumulative: 'Cumulative' }], + headerStringifier + ) + write( + res, + resStringifier, + formatSettings, + params + ) + + queue.emit('progress', 100) + + headerStringifier.end() + resStringifier.end() +} From 0b3bf2dee20d944598f53742302c5e7a1153faef Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 24 Mar 2023 12:04:52 +0200 Subject: [PATCH 03/13] Inject WeightedAveragesReportCsvWriter service to di --- workers/loc.api/di/app.deps.js | 13 +++++++++++++ workers/loc.api/di/types.js | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/workers/loc.api/di/app.deps.js b/workers/loc.api/di/app.deps.js index 9d423afe..c0b36622 100644 --- a/workers/loc.api/di/app.deps.js +++ b/workers/loc.api/di/app.deps.js @@ -27,6 +27,9 @@ const generateCsv = require('../generate-csv') const CsvJobData = require('../generate-csv/csv.job.data') const Interrupter = require('../interrupter') const AbstractWSEventEmitter = require('../abstract.ws.event.emitter') +const { + weightedAveragesReportCsvWriter +} = require('../generate-csv/csv-writer') module.exports = ({ rService, @@ -180,5 +183,15 @@ module.exports = ({ .to(Interrupter) bind(TYPES.AbstractWSEventEmitter) .to(AbstractWSEventEmitter) + bind(TYPES.WeightedAveragesReportCsvWriter) + .toConstantValue( + bindDepsToFn( + weightedAveragesReportCsvWriter, + [ + TYPES.RService, + TYPES.GetDataFromApi + ] + ) + ) }) } diff --git a/workers/loc.api/di/types.js b/workers/loc.api/di/types.js index ad0df7cb..63540990 100644 --- a/workers/loc.api/di/types.js +++ b/workers/loc.api/di/types.js @@ -31,5 +31,6 @@ module.exports = { CsvJobData: Symbol.for('CsvJobData'), Interrupter: Symbol.for('Interrupter'), AbstractWSEventEmitter: Symbol.for('AbstractWSEventEmitter'), - GetDataFromApi: Symbol.for('GetDataFromApi') + GetDataFromApi: Symbol.for('GetDataFromApi'), + WeightedAveragesReportCsvWriter: Symbol.for('WeightedAveragesReportCsvWriter') } From e267ec1419bba41bd24e4bde259184dd23d7f71b Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 24 Mar 2023 12:05:48 +0200 Subject: [PATCH 04/13] Add csv job data for weighted averages report --- workers/loc.api/generate-csv/csv.job.data.js | 51 +++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/workers/loc.api/generate-csv/csv.job.data.js b/workers/loc.api/generate-csv/csv.job.data.js index d015e874..5d8ac0e6 100644 --- a/workers/loc.api/generate-csv/csv.job.data.js +++ b/workers/loc.api/generate-csv/csv.job.data.js @@ -15,11 +15,16 @@ const { } = require('../errors') const depsTypes = (TYPES) => [ - TYPES.RService + TYPES.RService, + TYPES.WeightedAveragesReportCsvWriter ] class CsvJobData { - constructor (rService) { + constructor ( + rService, + weightedAveragesReportCsvWriter + ) { this.rService = rService + this.weightedAveragesReportCsvWriter = weightedAveragesReportCsvWriter } async getTradesCsvJobData ( @@ -1027,6 +1032,48 @@ class CsvJobData { jobsData } } + + async getWeightedAveragesReportCsvJobData ( + args, + uId, + uInfo + ) { + checkParams(args, 'paramsSchemaForWeightedAveragesReportApiCsv') + + const { + userId, + userInfo + } = await checkJobAndGetUserData( + this.rService, + uId, + uInfo + ) + + const csvArgs = getCsvArgs(args) + + const jobData = { + userInfo, + userId, + name: 'getWeightedAveragesReport', + fileNamesMap: [['getWeightedAveragesReport', 'weighted-averages-report']], + args: csvArgs, + columnsCsv: { + symbol: 'PAIR', + buyingWeightedPrice: 'WEIGHTED PRICE', + buyingAmount: 'AMOUNT', + sellingWeightedPrice: 'WEIGHTED PRICE', + sellingAmount: 'AMOUNT', + cumulativeWeightedPrice: 'WEIGHTED PRICE', + cumulativeAmount: 'AMOUNT' + }, + formatSettings: { + symbol: 'symbol' + }, + csvCustomWriter: this.weightedAveragesReportCsvWriter + } + + return jobData + } } decorateInjectable(CsvJobData, depsTypes) From 4085b4e99c48287c57416cfa4eff20a90a828cb5 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 24 Mar 2023 12:54:30 +0200 Subject: [PATCH 05/13] Add WeightedAveragesReport service --- .../loc.api/weighted.averages.report/index.js | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 workers/loc.api/weighted.averages.report/index.js diff --git a/workers/loc.api/weighted.averages.report/index.js b/workers/loc.api/weighted.averages.report/index.js new file mode 100644 index 00000000..920eb75c --- /dev/null +++ b/workers/loc.api/weighted.averages.report/index.js @@ -0,0 +1,168 @@ +'use strict' + +const { decorateInjectable } = require('../di/utils') + +const depsTypes = (TYPES) => [ + TYPES.RService +] +class WeightedAveragesReport { + constructor ( + rService + ) { + this.rService = rService + } + + async getWeightedAveragesReport (args = {}) { + const { + auth = {}, + params = {} + } = args ?? {} + + const { + start = 0, + end = Date.now(), + symbol: _symbol = [] + } = params ?? {} + const symbolArr = Array.isArray(_symbol) + ? _symbol + : [_symbol] + const symbol = symbolArr.filter((s) => ( + s && typeof s === 'string' + )) + + const trades = await this._getTrades({ + auth, + start, + end, + symbol + }) + const calcedTrades = this._calcTrades(trades) + + return calcedTrades + } + + async _getTrades (args) { + const { + auth = {}, + start = 0, + end = Date.now(), + symbol = [] + } = args ?? {} + + const symbFilter = ( + Array.isArray(symbol) && + symbol.length !== 0 + ) + ? { $in: { symbol } } + : {} + + // TODO: + return [] + } + + _calcTrades (trades = []) { + const symbResMap = new Map() + + for (const trade of trades) { + const { + symbol, + execAmount, + execPrice + } = trade ?? {} + + if ( + !symbol || + typeof symbol !== 'string' || + !Number.isFinite(execAmount) || + execAmount === 0 || + !Number.isFinite(execPrice) || + execPrice === 0 + ) { + continue + } + + const isBuying = execAmount > 0 + const spent = execAmount * execPrice + + const existedSymbRes = symbResMap.get(symbol) + const { + sumSpent: _sumSpent = 0, + sumAmount: _sumAmount = 0, + sumBuyingSpent: _sumBuyingSpent = 0, + sumBuyingAmount: _sumBuyingAmount = 0, + sumSellingSpent: _sumSellingSpent = 0, + sumSellingAmount: _sumSellingAmount = 0 + } = existedSymbRes ?? {} + + const sumSpent = Number.isFinite(spent) + ? _sumSpent + spent + : _sumSpent + const sumAmount = _sumAmount + execAmount + const sumBuyingSpent = ( + isBuying && + Number.isFinite(spent) + ) + ? _sumBuyingSpent + spent + : _sumBuyingSpent + const sumBuyingAmount = isBuying + ? _sumBuyingAmount + execAmount + : _sumBuyingAmount + const sumSellingSpent = ( + !isBuying && + Number.isFinite(spent) + ) + ? _sumSellingSpent + spent + : _sumSellingSpent + const sumSellingAmount = !isBuying + ? _sumSellingAmount + execAmount + : _sumSellingAmount + + symbResMap.set(symbol, { + sumSpent, + sumAmount, + sumBuyingSpent, + sumBuyingAmount, + sumSellingSpent, + sumSellingAmount, + + buyingWeightedPrice: sumBuyingAmount === 0 + ? 0 + : sumBuyingSpent / sumBuyingAmount, + buyingAmount: sumBuyingAmount, + sellingWeightedPrice: sumSellingAmount === 0 + ? 0 + : sumSellingSpent / sumSellingAmount, + sellingAmount: sumSellingAmount, + cumulativeWeightedPrice: sumAmount === 0 + ? 0 + : sumSpent / sumAmount, + cumulativeAmount: sumAmount + }) + } + + return [...symbResMap].map(([symbol, val]) => { + const { + buyingWeightedPrice = 0, + buyingAmount = 0, + sellingWeightedPrice = 0, + sellingAmount = 0, + cumulativeWeightedPrice = 0, + cumulativeAmount = 0 + } = val ?? {} + + return { + symbol, + buyingWeightedPrice, + buyingAmount, + sellingWeightedPrice, + sellingAmount, + cumulativeWeightedPrice, + cumulativeAmount + } + }) + } +} + +decorateInjectable(WeightedAveragesReport, depsTypes) + +module.exports = WeightedAveragesReport From b29b1739cf90066f0b01ce047fe16d14b7d36ed5 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 24 Mar 2023 12:54:59 +0200 Subject: [PATCH 06/13] Inject WeightedAveragesReport service to di --- workers/loc.api/di/app.deps.js | 6 +++++- workers/loc.api/di/types.js | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/workers/loc.api/di/app.deps.js b/workers/loc.api/di/app.deps.js index c0b36622..b17b65bb 100644 --- a/workers/loc.api/di/app.deps.js +++ b/workers/loc.api/di/app.deps.js @@ -30,6 +30,7 @@ const AbstractWSEventEmitter = require('../abstract.ws.event.emitter') const { weightedAveragesReportCsvWriter } = require('../generate-csv/csv-writer') +const WeightedAveragesReport = require('../weighted.averages.report') module.exports = ({ rService, @@ -48,7 +49,8 @@ module.exports = ({ ['_grcBfxReq', TYPES.GrcBfxReq], ['_prepareApiResponse', TYPES.PrepareApiResponse], ['_generateCsv', TYPES.GenerateCsv], - ['_hasGrcService', TYPES.HasGrcService] + ['_hasGrcService', TYPES.HasGrcService], + ['_weightedAveragesReport', TYPES.WeightedAveragesReport] ]) bind(TYPES.RServiceDepsSchemaAliase) .toDynamicValue((ctx) => { @@ -193,5 +195,7 @@ module.exports = ({ ] ) ) + bind(TYPES.WeightedAveragesReport) + .to(WeightedAveragesReport) }) } diff --git a/workers/loc.api/di/types.js b/workers/loc.api/di/types.js index 63540990..b51deaaa 100644 --- a/workers/loc.api/di/types.js +++ b/workers/loc.api/di/types.js @@ -32,5 +32,6 @@ module.exports = { Interrupter: Symbol.for('Interrupter'), AbstractWSEventEmitter: Symbol.for('AbstractWSEventEmitter'), GetDataFromApi: Symbol.for('GetDataFromApi'), + WeightedAveragesReport: Symbol.for('WeightedAveragesReport'), WeightedAveragesReportCsvWriter: Symbol.for('WeightedAveragesReportCsvWriter') } From 1d778611e48380b0f980ad0cf4e56022d2a0e56e Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 24 Mar 2023 12:55:39 +0200 Subject: [PATCH 07/13] Add getWeightedAveragesReport endpoint --- workers/loc.api/service.report.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/workers/loc.api/service.report.js b/workers/loc.api/service.report.js index 0dbf360c..6fc4edc8 100644 --- a/workers/loc.api/service.report.js +++ b/workers/loc.api/service.report.js @@ -593,6 +593,15 @@ class ReportService extends Api { }, 'getChangeLogs', args, cb) } + getWeightedAveragesReport (space, args, cb) { + return this._responder(async () => { + checkParams(args, 'paramsSchemaForWeightedAveragesReportApi') + + return this._weightedAveragesReport + .getWeightedAveragesReport(args) + }, 'getWeightedAveragesReport', args, cb) + } + getMultipleCsv (space, args, cb) { return this._responder(() => { return this._generateCsv( From ad32ecc905247deca7dda5792a6e7fa89da4f137 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 24 Mar 2023 12:58:04 +0200 Subject: [PATCH 08/13] Add getWeightedAveragesReportCsv endpoint --- workers/loc.api/service.report.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/workers/loc.api/service.report.js b/workers/loc.api/service.report.js index 6fc4edc8..a3ca38b4 100644 --- a/workers/loc.api/service.report.js +++ b/workers/loc.api/service.report.js @@ -799,6 +799,15 @@ class ReportService extends Api { ) }, 'getChangeLogsCsv', args, cb) } + + getWeightedAveragesReportCsv (space, args, cb) { + return this._responder(() => { + return this._generateCsv( + 'getWeightedAveragesReportCsvJobData', + args + ) + }, 'getWeightedAveragesReportCsv', args, cb) + } } module.exports = ReportService From d3cbece94c120dfed06af8f58fb3d5a1396dda0e Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 27 Mar 2023 16:07:48 +0300 Subject: [PATCH 09/13] Add ability to get trades for weighted averages report --- .../loc.api/weighted.averages.report/index.js | 108 +++++++++++++++--- 1 file changed, 92 insertions(+), 16 deletions(-) diff --git a/workers/loc.api/weighted.averages.report/index.js b/workers/loc.api/weighted.averages.report/index.js index 920eb75c..d03e736d 100644 --- a/workers/loc.api/weighted.averages.report/index.js +++ b/workers/loc.api/weighted.averages.report/index.js @@ -3,13 +3,16 @@ const { decorateInjectable } = require('../di/utils') const depsTypes = (TYPES) => [ - TYPES.RService + TYPES.RService, + TYPES.GetDataFromApi ] class WeightedAveragesReport { constructor ( - rService + rService, + getDataFromApi ) { this.rService = rService + this.getDataFromApi = getDataFromApi } async getWeightedAveragesReport (args = {}) { @@ -42,22 +45,95 @@ class WeightedAveragesReport { } async _getTrades (args) { - const { - auth = {}, - start = 0, - end = Date.now(), - symbol = [] - } = args ?? {} - - const symbFilter = ( - Array.isArray(symbol) && - symbol.length !== 0 - ) - ? { $in: { symbol } } + const start = args?.start ?? 0 + const symbol = args?.symbol?.length > 0 + ? { symbol: args.symbol } : {} - // TODO: - return [] + let end = args?.end ?? Date.now() + let prevEnd = end + let serialRequestsCount = 0 + + const trades = [] + + while (true) { + let { + res, + nextPage + } = await this.getDataFromApi({ + getData: this.rService.getTrades.bind(this.rService), + args: { + auth: args?.auth ?? {}, + params: { start, end, ...symbol }, + notThrowError: true + }, + callerName: 'WEIGHTED_AVERAGES', + eNetErrorAttemptsTimeframeMin: 10 / 60, + eNetErrorAttemptsTimeoutMs: 1000, + shouldNotInterrupt: true + }) + + prevEnd = end + end = nextPage + + if ( + Array.isArray(res) && + res.length === 0 && + nextPage && + Number.isInteger(nextPage) && + serialRequestsCount < 1 + ) { + serialRequestsCount += 1 + + continue + } + + serialRequestsCount = 0 + + if ( + !Array.isArray(res) || + res.length === 0 + ) { + break + } + + const lastItem = res[res.length - 1] + const lastMts = lastItem?.mtsCreate + let isAllData = false + + if ( + !lastItem || + typeof lastItem !== 'object' || + !lastMts || + !Number.isInteger(lastMts) + ) { + break + } + + if (start >= lastMts) { + res = res.filter((item) => start <= item?.mtsCreate) + isAllData = true + } + // TODO: + if ( + process.env.NODE_ENV === 'test' && + prevEnd === end + ) { + isAllData = true + } + + trades.push(...res) + + if ( + isAllData || + !end || + !Number.isInteger(end) + ) { + break + } + } + + return trades } _calcTrades (trades = []) { From 98794b74edf2a39a10f9872e0b844e1ddfea9362 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Wed, 29 Mar 2023 07:19:44 +0300 Subject: [PATCH 10/13] Add test case for getWeightedAveragesReport endpoint --- test/1-api.spec.js | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/1-api.spec.js b/test/1-api.spec.js index e16c3e77..1d5188c9 100644 --- a/test/1-api.spec.js +++ b/test/1-api.spec.js @@ -1719,4 +1719,48 @@ describe('API', () => { 'note' ]) }) + + it('it should be successfully performed by the getWeightedAveragesReport method', async function () { + this.timeout(120000) + + const paramsArr = [ + { end, start }, + { + end, + start: end - (10 * 60 * 60 * 1000), + symbol: ['tBTCUSD'] + } + ] + + for (const params of paramsArr) { + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getWeightedAveragesReport', + params, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + + assert.isObject(res.body) + assert.propertyVal(res.body, 'id', 5) + assert.isArray(res.body.result) + + const resItem = res.body.result[0] + + assert.isObject(resItem) + assert.containsAllKeys(resItem, [ + 'symbol', + 'buyingWeightedPrice', + 'buyingAmount', + 'sellingWeightedPrice', + 'sellingAmount', + 'cumulativeWeightedPrice', + 'cumulativeAmount' + ]) + } + }) }) From f4cb7272fe10b52c2dc4316d2aec07e9522ca068 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Wed, 29 Mar 2023 07:20:09 +0300 Subject: [PATCH 11/13] Add test case for getWeightedAveragesReportCsv endpoint --- test/4-queue-base.spec.js | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/4-queue-base.spec.js b/test/4-queue-base.spec.js index 75588588..90314da1 100644 --- a/test/4-queue-base.spec.js +++ b/test/4-queue-base.spec.js @@ -819,6 +819,31 @@ describe('Queue', () => { await testMethodOfGettingCsv(procPromise, aggrPromise, res) }) + it('it should be successfully performed by the getWeightedAveragesReportCsv method', async function () { + this.timeout(60000) + + const procPromise = queueToPromise(processorQueue) + const aggrPromise = queueToPromise(aggregatorQueue) + + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getWeightedAveragesReportCsv', + params: { + end, + start, + email + }, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + + await testMethodOfGettingCsv(procPromise, aggrPromise, res) + }) + it('it should not be successfully auth by the getLedgersCsv method', async function () { this.timeout(60000) From a871b1e96cf5b427cd364dae0a76d026fefd3358 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 3 Apr 2023 14:08:58 +0300 Subject: [PATCH 12/13] Add 2k trades limitation for weighted averages report --- test/4-queue-base.spec.js | 7 +++++-- workers/loc.api/errors/index.js | 11 ++++++++++- workers/loc.api/weighted.averages.report/index.js | 12 +++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/test/4-queue-base.spec.js b/test/4-queue-base.spec.js index 90314da1..db665e5c 100644 --- a/test/4-queue-base.spec.js +++ b/test/4-queue-base.spec.js @@ -42,12 +42,13 @@ const auth = { apiKey: 'fake', apiSecret: 'fake' } +const mockDataAmount = 10000 describe('Queue', () => { before(async function () { this.timeout(20000) - mockRESTv2Srv = createMockRESTv2SrvWithDate(start, end, 10000) + mockRESTv2Srv = createMockRESTv2SrvWithDate(start, end, mockDataAmount) await rmAllFiles(tempDirPath) await rmDB(dbDirPath) @@ -825,6 +826,8 @@ describe('Queue', () => { const procPromise = queueToPromise(processorQueue) const aggrPromise = queueToPromise(aggregatorQueue) + const _start = end - ((end - start) / (mockDataAmount / 2000)) + const res = await agent .post(`${basePath}/json-rpc`) .type('json') @@ -833,7 +836,7 @@ describe('Queue', () => { method: 'getWeightedAveragesReportCsv', params: { end, - start, + start: _start, email }, id: 5 diff --git a/workers/loc.api/errors/index.js b/workers/loc.api/errors/index.js index e1b23107..5b9bf287 100644 --- a/workers/loc.api/errors/index.js +++ b/workers/loc.api/errors/index.js @@ -114,6 +114,14 @@ class MinLimitParamError extends UnprocessableEntityError { } } +class MaxWeightedAveragesReportTradesNumError extends UnprocessableEntityError { + constructor (message = 'ERR_REDUCE_TRADES_NUMBER_IS_NEEDED_FOR_WEIGHTED_AVERAGES_REPORT') { + super(message) + + this.statusMessage = 'To show the weighted averages report, it is required to reduce the number of trades so that it is no more than 2k' + } +} + class QueueJobAddingError extends ConflictError { constructor (message = 'ERR_HAS_JOB_IN_QUEUE') { super(message) @@ -185,5 +193,6 @@ module.exports = { ArgsParamsFilterError, LedgerPaymentFilteringParamsError, GrcSlackAvailabilityError, - ImplementationError + ImplementationError, + MaxWeightedAveragesReportTradesNumError } diff --git a/workers/loc.api/weighted.averages.report/index.js b/workers/loc.api/weighted.averages.report/index.js index d03e736d..d6a48f2e 100644 --- a/workers/loc.api/weighted.averages.report/index.js +++ b/workers/loc.api/weighted.averages.report/index.js @@ -1,5 +1,9 @@ 'use strict' +const { + MaxWeightedAveragesReportTradesNumError +} = require('../errors') + const { decorateInjectable } = require('../di/utils') const depsTypes = (TYPES) => [ @@ -45,6 +49,7 @@ class WeightedAveragesReport { } async _getTrades (args) { + const limit = 2000 const start = args?.start ?? 0 const symbol = args?.symbol?.length > 0 ? { symbol: args.symbol } @@ -53,6 +58,7 @@ class WeightedAveragesReport { let end = args?.end ?? Date.now() let prevEnd = end let serialRequestsCount = 0 + let count = 0 const trades = [] @@ -114,7 +120,10 @@ class WeightedAveragesReport { res = res.filter((item) => start <= item?.mtsCreate) isAllData = true } - // TODO: + // After reducing between start/end timeframe check trades count limitation + if (limit < (count + res.length)) { + throw new MaxWeightedAveragesReportTradesNumError() + } if ( process.env.NODE_ENV === 'test' && prevEnd === end @@ -123,6 +132,7 @@ class WeightedAveragesReport { } trades.push(...res) + count += res.length if ( isAllData || From 851e2b10267fe322e5e68b269a055e6bfbdac225 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Tue, 4 Apr 2023 12:50:07 +0300 Subject: [PATCH 13/13] Return nextPage when catching 2k limit for weighted averages report --- test/1-api.spec.js | 6 +++-- workers/loc.api/errors/index.js | 11 +-------- .../weighted-averages-report-csv-writer.js | 2 +- .../loc.api/weighted.averages.report/index.js | 24 ++++++++++++------- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/test/1-api.spec.js b/test/1-api.spec.js index 1d5188c9..7245c6dd 100644 --- a/test/1-api.spec.js +++ b/test/1-api.spec.js @@ -1747,9 +1747,11 @@ describe('API', () => { assert.isObject(res.body) assert.propertyVal(res.body, 'id', 5) - assert.isArray(res.body.result) + assert.isObject(res.body.result) + assert.isArray(res.body.result.res) + assert.isBoolean(res.body.result.nextPage) - const resItem = res.body.result[0] + const resItem = res.body.result.res[0] assert.isObject(resItem) assert.containsAllKeys(resItem, [ diff --git a/workers/loc.api/errors/index.js b/workers/loc.api/errors/index.js index 5b9bf287..e1b23107 100644 --- a/workers/loc.api/errors/index.js +++ b/workers/loc.api/errors/index.js @@ -114,14 +114,6 @@ class MinLimitParamError extends UnprocessableEntityError { } } -class MaxWeightedAveragesReportTradesNumError extends UnprocessableEntityError { - constructor (message = 'ERR_REDUCE_TRADES_NUMBER_IS_NEEDED_FOR_WEIGHTED_AVERAGES_REPORT') { - super(message) - - this.statusMessage = 'To show the weighted averages report, it is required to reduce the number of trades so that it is no more than 2k' - } -} - class QueueJobAddingError extends ConflictError { constructor (message = 'ERR_HAS_JOB_IN_QUEUE') { super(message) @@ -193,6 +185,5 @@ module.exports = { ArgsParamsFilterError, LedgerPaymentFilteringParamsError, GrcSlackAvailabilityError, - ImplementationError, - MaxWeightedAveragesReportTradesNumError + ImplementationError } diff --git a/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js b/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js index 3168420e..662e8842 100644 --- a/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js +++ b/workers/loc.api/generate-csv/csv-writer/weighted-averages-report-csv-writer.js @@ -53,7 +53,7 @@ module.exports = ( pipeline(headerStringifier, wStream, nope) pipeline(resStringifier, wStream, nope) - const res = await getDataFromApi({ + const { res } = await getDataFromApi({ getData: rService[name].bind(rService), args, callerName: 'CSV_WRITER' diff --git a/workers/loc.api/weighted.averages.report/index.js b/workers/loc.api/weighted.averages.report/index.js index d6a48f2e..186060bc 100644 --- a/workers/loc.api/weighted.averages.report/index.js +++ b/workers/loc.api/weighted.averages.report/index.js @@ -1,9 +1,5 @@ 'use strict' -const { - MaxWeightedAveragesReportTradesNumError -} = require('../errors') - const { decorateInjectable } = require('../di/utils') const depsTypes = (TYPES) => [ @@ -37,7 +33,10 @@ class WeightedAveragesReport { s && typeof s === 'string' )) - const trades = await this._getTrades({ + const { + res: trades, + nextPage + } = await this._getTrades({ auth, start, end, @@ -45,7 +44,10 @@ class WeightedAveragesReport { }) const calcedTrades = this._calcTrades(trades) - return calcedTrades + return { + nextPage, + res: calcedTrades + } } async _getTrades (args) { @@ -61,6 +63,7 @@ class WeightedAveragesReport { let count = 0 const trades = [] + let nextPageRes = false while (true) { let { @@ -122,7 +125,9 @@ class WeightedAveragesReport { } // After reducing between start/end timeframe check trades count limitation if (limit < (count + res.length)) { - throw new MaxWeightedAveragesReportTradesNumError() + res.splice(limit - count) + nextPageRes = res[res.length - 1]?.mtsCreate ?? false + isAllData = true } if ( process.env.NODE_ENV === 'test' && @@ -143,7 +148,10 @@ class WeightedAveragesReport { } } - return trades + return { + nextPage: nextPageRes, + res: trades + } } _calcTrades (trades = []) {