diff --git a/test/3-api-filter-sync-mode-sqlite.spec.js b/test/3-api-filter-sync-mode-sqlite.spec.js index da9841198..e55ae8e86 100644 --- a/test/3-api-filter-sync-mode-sqlite.spec.js +++ b/test/3-api-filter-sync-mode-sqlite.spec.js @@ -20,8 +20,7 @@ const { startEnvironment } = require('./helpers/helpers.boot') const { - emptyDB, - delay + emptyDB } = require('./helpers/helpers.core') const { createMockRESTv2SrvWithDate @@ -31,7 +30,10 @@ process.env.NODE_CONFIG_DIR = path.join(__dirname, 'config') const { app } = require('bfx-report-express') const agent = request.agent(app) -const { signUpTestCase } = require('./test-cases') +const { + signUpTestCase, + getSyncProgressTestCase +} = require('./test-cases') let wrkReportServiceApi = null let processorQueue = null @@ -120,35 +122,7 @@ describe('API filter', () => { ) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the editPublicTradesConf method', async function () { this.timeout(5000) @@ -175,35 +149,7 @@ describe('API filter', () => { assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the editTickersHistoryConf method', async function () { this.timeout(5000) @@ -230,35 +176,7 @@ describe('API filter', () => { assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the api methods', async function () { this.timeout(10000) diff --git a/test/6-update-sub-account.spec.js b/test/6-update-sub-account.spec.js index 30b9a40c4..e3ee1440c 100644 --- a/test/6-update-sub-account.spec.js +++ b/test/6-update-sub-account.spec.js @@ -17,8 +17,7 @@ const { } = require('./helpers/helpers.boot') const { emptyDB, - getRServiceProxy, - delay + getRServiceProxy } = require('./helpers/helpers.core') const { createMockRESTv2SrvWithDate, @@ -31,7 +30,8 @@ const { app } = require('bfx-report-express') const agent = request.agent(app) const { - signUpTestCase + signUpTestCase, + getSyncProgressTestCase } = require('./test-cases') let wrkReportServiceApi = null @@ -289,35 +289,7 @@ describe('Update sub-account', () => { assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth: subAccountAuth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth: subAccountAuth }) it('it should be successfully performed by the getLedgers method, without params', async function () { this.timeout(5000) @@ -517,35 +489,7 @@ describe('Update sub-account', () => { assert.isString(res.body.result.token) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth: subAccountAuth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth: subAccountAuth }) it('it should be successfully performed by the getUsers method', async function () { this.timeout(5000) @@ -587,35 +531,7 @@ describe('Update sub-account', () => { }) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth: subAccountAuth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth: subAccountAuth }) it('it should be successfully performed by the getLedgers method, without params', async function () { this.timeout(5000) diff --git a/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js b/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js index 6cbb6975b..d692479aa 100644 --- a/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js +++ b/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js @@ -10,13 +10,14 @@ const { } = require('bfx-report/test/helpers/helpers.tests') const { - delay, getParamsArrToTestTimeframeGrouping } = require('../helpers/helpers.core') const { testCsvPathHasCommonFolder } = require('../helpers/helpers.tests') +const getSyncProgressTestCase = require('./get-sync-progress-test-case') + module.exports = ( agent, params = {} @@ -123,35 +124,7 @@ module.exports = ( ) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the getBalanceHistory method', async function () { this.timeout(5000) diff --git a/test/test-cases/api-sync-mode-sqlite-test-cases.js b/test/test-cases/api-sync-mode-sqlite-test-cases.js index 5c561f4b2..4e08bf703 100644 --- a/test/test-cases/api-sync-mode-sqlite-test-cases.js +++ b/test/test-cases/api-sync-mode-sqlite-test-cases.js @@ -11,7 +11,7 @@ const { testProcQueue } = require('bfx-report/test/helpers/helpers.tests') -const { delay } = require('../helpers/helpers.core') +const getSyncProgressTestCase = require('./get-sync-progress-test-case') const { getMockData } = require('../helpers/helpers.mock-rest-v2') module.exports = ( @@ -414,35 +414,7 @@ module.exports = ( assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the haveCollsBeenSyncedAtLeastOnce method, returns true', async function () { this.timeout(60000) @@ -488,35 +460,7 @@ module.exports = ( assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the editPublicTradesConf method', async function () { this.timeout(5000) @@ -548,35 +492,7 @@ module.exports = ( assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the getPublicTradesConf method', async function () { this.timeout(5000) @@ -630,35 +546,7 @@ module.exports = ( assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the editTickersHistoryConf method', async function () { this.timeout(5000) @@ -690,35 +578,7 @@ module.exports = ( assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the getTickersHistoryConf method', async function () { this.timeout(5000) @@ -774,35 +634,7 @@ module.exports = ( assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the getStatusMessagesConf method', async function () { this.timeout(5000) @@ -855,35 +687,7 @@ module.exports = ( assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the getCandlesConf method', async function () { this.timeout(5000) @@ -965,35 +769,7 @@ module.exports = ( assert.isOk(res.body.result) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the getAllPublicСollsСonfs method', async function () { this.timeout(5000) @@ -1116,35 +892,7 @@ module.exports = ( ) }) - it('it should be successfully performed by the getSyncProgress method', async function () { - this.timeout(60000) - - while (true) { - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getSyncProgress', - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isNumber(res.body.result) - - if ( - typeof res.body.result !== 'number' || - res.body.result === 100 - ) { - break - } - - await delay() - } - }) + getSyncProgressTestCase(agent, { basePath, auth }) it('it should be successfully performed by the isSyncModeWithDbData method', async function () { this.timeout(5000) @@ -3561,7 +3309,11 @@ module.exports = ( assert.isObject(res.body) assert.propertyVal(res.body, 'id', 5) - assert.isNotOk(res.body.result) + assert.isObject(res.body.result) + assert.isNotOk(res.body.result.progress) + assert.isNull(res.body.result.syncStartedAt) + assert.isNull(res.body.result.spentTime) + assert.isNull(res.body.result.leftTime) }) it('it should be successfully performed by the disableSyncMode method', async function () { diff --git a/test/test-cases/get-sync-progress-test-case.js b/test/test-cases/get-sync-progress-test-case.js new file mode 100644 index 000000000..ff1b7566f --- /dev/null +++ b/test/test-cases/get-sync-progress-test-case.js @@ -0,0 +1,55 @@ +'use strict' + +const { assert } = require('chai') + +const { delay } = require('../helpers/helpers.core') + +module.exports = ( + agent, + params = {} +) => { + const { + basePath, + auth + } = params + + it('it should be successfully performed by the getSyncProgress method', async function () { + this.timeout(60000) + + while (true) { + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getSyncProgress', + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + + assert.isObject(res.body) + assert.propertyVal(res.body, 'id', 5) + assert.isObject(res.body.result) + assert.isNumber(res.body.result.progress) + assert.isNumber(res.body.result.syncStartedAt) + assert.isNumber(res.body.result.spentTime) + + if ( + !Number.isFinite(res.body.result.progress) || + res.body.result.progress <= 0 || + res.body.result.progress > 100 + ) { + assert.isNull(res.body.result.leftTime) + } else { + assert.isNumber(res.body.result.leftTime) + } + + if (res.body.result.progress === 100) { + break + } + + await delay() + } + }) +} diff --git a/test/test-cases/index.js b/test/test-cases/index.js index d76609ef1..4470bccb9 100644 --- a/test/test-cases/index.js +++ b/test/test-cases/index.js @@ -7,9 +7,11 @@ const additionalApiSyncModeSqliteTestCases = require( './additional-api-sync-mode-sqlite-test-cases' ) const signUpTestCase = require('./sign-up-test-case') +const getSyncProgressTestCase = require('./get-sync-progress-test-case') module.exports = { apiSyncModeSqliteTestCases, additionalApiSyncModeSqliteTestCases, - signUpTestCase + signUpTestCase, + getSyncProgressTestCase } diff --git a/workers/loc.api/service.report.framework.js b/workers/loc.api/service.report.framework.js index 72499745a..b8900f965 100644 --- a/workers/loc.api/service.report.framework.js +++ b/workers/loc.api/service.report.framework.js @@ -331,7 +331,12 @@ class FrameworkReportService extends ReportService { isSchedulerEnabled ) ? this._progress.getProgress() - : false + : { + progress: false, + syncStartedAt: null, + spentTime: null, + leftTime: null + } }, 'getSyncProgress', args, cb) } diff --git a/workers/loc.api/sync/data.consistency.checker/index.js b/workers/loc.api/sync/data.consistency.checker/index.js index fedbb2529..d74f15b1f 100644 --- a/workers/loc.api/sync/data.consistency.checker/index.js +++ b/workers/loc.api/sync/data.consistency.checker/index.js @@ -40,7 +40,9 @@ class DataConsistencyChecker { const isValid = await check(auth) if (!isValid) { - const currProgress = await this.progress.getProgress() + const { + progress: currProgress + } = await this.progress.getProgress() const isDBSyncing = currProgress < 100 if (isDBSyncing) { diff --git a/workers/loc.api/sync/index.js b/workers/loc.api/sync/index.js index b4b4e3ed8..4cfc9642d 100644 --- a/workers/loc.api/sync/index.js +++ b/workers/loc.api/sync/index.js @@ -37,8 +37,11 @@ class Sync { let progressForInterrupter = this.syncInterrupter .INITIAL_PROGRESS + this.progress.deactivateSyncTimeEstimate() + if (!error) { try { + this.progress.activateSyncTimeEstimate() progressForInterrupter = await this.syncQueue.process(params) } catch (err) { errorForInterrupter = err @@ -64,7 +67,9 @@ class Sync { ) } - const currProgress = await this.progress.getProgress() + const { + progress: currProgress + } = await this.progress.getProgress() if (this.syncInterrupter.hasInterrupted()) { this.syncInterrupter.emitInterrupted( @@ -99,7 +104,9 @@ class Sync { } const isEnable = await this.rService.isSchedulerEnabled() - const currProgress = await this.progress.getProgress() + const { + progress: currProgress + } = await this.progress.getProgress() if (isEnable) { await this.syncQueue.add({ @@ -112,7 +119,7 @@ class Sync { (currProgress < 100) || !isEnable ) { - return this.progress.getProgress() + return (await this.progress.getProgress())?.progress } await this.rService.pingApi() @@ -142,7 +149,11 @@ class Sync { } async stop () { - const currProgress = await this.progress.getProgress() + const { + progress: currProgress + } = await this.progress + .deactivateSyncTimeEstimate() + .getProgress() if (currProgress < 100) { return this.syncInterrupter.interrupt() diff --git a/workers/loc.api/sync/progress/index.js b/workers/loc.api/sync/progress/index.js index d4323bf94..6503c5deb 100644 --- a/workers/loc.api/sync/progress/index.js +++ b/workers/loc.api/sync/progress/index.js @@ -1,7 +1,6 @@ 'use strict' const EventEmitter = require('events') -const { isEmpty } = require('lodash') const { tryParseJSON @@ -28,6 +27,8 @@ class Progress extends EventEmitter { this.TABLES_NAMES = TABLES_NAMES this.wsEventEmitter = wsEventEmitter this.logger = logger + + this._syncStartedAt = null } async setProgress (progress) { @@ -44,8 +45,12 @@ class Progress extends EventEmitter { this.TABLES_NAMES.PROGRESS, { value: JSON.stringify(_progress) } ) - this.emit(_progress) - await this.wsEventEmitter.emitProgress(() => _progress) + const estimatedSyncTime = this._estimateSyncTime({ + progress: _progress + }) + + this.emit(estimatedSyncTime) + await this.wsEventEmitter.emitProgress(() => estimatedSyncTime) } catch (e) { this.logger.error( `PROGRESS:SYNC:SET: ${e.stack || e}` @@ -60,15 +65,76 @@ class Progress extends EventEmitter { } async getProgress () { - const progress = await this.dao + const progressObj = await this.dao .getElemInCollBy(this.TABLES_NAMES.PROGRESS) - return ( - !isEmpty(progress) && - typeof progress.value === 'string' - ) - ? tryParseJSON(progress.value, true) + const progress = typeof progressObj?.value === 'string' + ? tryParseJSON(progressObj.value, true) : 'SYNCHRONIZATION_HAS_NOT_STARTED_YET' + const estimatedSyncTime = this._estimateSyncTime({ progress }) + + return estimatedSyncTime + } + + activateSyncTimeEstimate () { + this._syncStartedAt = Date.now() + + return this + } + + deactivateSyncTimeEstimate () { + this._syncStartedAt = null + + return this + } + + async _estimateSyncTime (params) { + const { + progress + } = params ?? {} + + const syncStartedAt = this._getSyncStartedAt() + const nowMts = Date.now() + + if ( + !Number.isFinite(syncStartedAt) || + syncStartedAt > nowMts + ) { + return { + progress, + syncStartedAt: null, + spentTime: null, + leftTime: null + } + } + + const spentTime = nowMts - syncStartedAt + + if ( + !Number.isFinite(progress) || + progress <= 0 || + progress > 100 + ) { + return { + progress, + syncStartedAt, + spentTime, + leftTime: null + } + } + + const leftTime = (spentTime / progress) * (100 - progress) + + return { + progress, + syncStartedAt, + spentTime, + leftTime + } + } + + _getSyncStartedAt () { + return this._syncStartedAt ?? null } }