diff --git a/test/helpers/mock-data.js b/test/helpers/mock-data.js index fbac18b57..d6bf87d9c 100644 --- a/test/helpers/mock-data.js +++ b/test/helpers/mock-data.js @@ -73,7 +73,7 @@ module.exports = new Map([ 12.12345, 12345.12345, null, - 'Margin Funding Payment on wallet funding' + 'Trading fees for 0.041154 BTG (BTGEUR) @ 28.818 on BFX (0.2%) on wallet exchange' ], [ 30012345, @@ -95,7 +95,7 @@ module.exports = new Map([ 12.12345, 12345.12345, null, - 'Margin Funding Payment on wallet funding' + 'Used Margin Funding Charge on wallet margin' ] ] ], diff --git a/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js b/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js index d4a3c4ca2..3dabf4789 100644 --- a/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js +++ b/test/test-cases/additional-api-sync-mode-sqlite-test-cases.js @@ -483,6 +483,70 @@ module.exports = ( } }) + it('it should be successfully performed by the getTotalFeesReport method', async function () { + this.timeout(60000) + + const paramsArr = getParamsArrToTestTimeframeGrouping({ + start, + end, + isTradingFees: true, + isFundingFees: true + }) + + for (const params of paramsArr) { + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getTotalFeesReport', + params, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + + assert.isObject(res.body) + assert.propertyVal(res.body, 'id', 5) + assert.isArray(res.body.result) + + const resItem = res.body.result[0] + + assert.isObject(resItem) + assert.containsAllKeys(resItem, [ + 'mts', + 'cumulative', + 'USD' + ]) + } + }) + + it('it should not be successfully performed by the getTotalFeesReport method', async function () { + this.timeout(60000) + + const paramsArr = getParamsArrToTestTimeframeGrouping({ start, end }) + + for (const params of paramsArr) { + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getTotalFeesReport', + params, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(400) + + assert.isObject(res.body) + assert.isObject(res.body.error) + assert.propertyVal(res.body.error, 'code', 400) + assert.propertyVal(res.body.error, 'message', 'Args params is not valid') + assert.propertyVal(res.body, 'id', 5) + } + }) + it('it should be successfully performed by the getPerformingLoan method', async function () { this.timeout(60000) @@ -882,6 +946,33 @@ module.exports = ( await testMethodOfGettingCsv(procPromise, aggrPromise, res) }) + it('it should be successfully performed by the getTotalFeesReportCsv method', async function () { + this.timeout(60000) + + const procPromise = queueToPromise(params.processorQueue) + const aggrPromise = queueToPromise(params.aggregatorQueue) + + const res = await agent + .post(`${basePath}/json-rpc`) + .type('json') + .send({ + auth, + method: 'getTotalFeesReportCsv', + params: { + end, + start, + timeframe: 'day', + email, + isTradingFees: true + }, + id: 5 + }) + .expect('Content-Type', /json/) + .expect(200) + + await testMethodOfGettingCsv(procPromise, aggrPromise, res) + }) + it('it should be successfully performed by the getPerformingLoanCsv method', async function () { this.timeout(60000) diff --git a/workers/loc.api/di/app.deps.js b/workers/loc.api/di/app.deps.js index 90165109d..26fa0a97e 100644 --- a/workers/loc.api/di/app.deps.js +++ b/workers/loc.api/di/app.deps.js @@ -73,6 +73,7 @@ const FullSnapshotReport = require('../sync/full.snapshot.report') const Trades = require('../sync/trades') const TradedVolume = require('../sync/traded.volume') const FeesReport = require('../sync/fees.report') +const TotalFeesReport = require('../sync/total.fees.report') const PerformingLoan = require('../sync/performing.loan') const SubAccountApiData = require('../sync/sub.account.api.data') const PositionsAudit = require('../sync/positions.audit') @@ -133,6 +134,7 @@ module.exports = ({ ['_fullTaxReport', TYPES.FullTaxReport], ['_tradedVolume', TYPES.TradedVolume], ['_feesReport', TYPES.FeesReport], + ['_totalFeesReport', TYPES.TotalFeesReport], ['_performingLoan', TYPES.PerformingLoan], ['_subAccountApiData', TYPES.SubAccountApiData], ['_positionsAudit', TYPES.PositionsAudit], @@ -309,6 +311,8 @@ module.exports = ({ .to(TradedVolume) bind(TYPES.FeesReport) .to(FeesReport) + bind(TYPES.TotalFeesReport) + .to(TotalFeesReport) bind(TYPES.PerformingLoan) .to(PerformingLoan) bind(TYPES.SubAccountApiData) diff --git a/workers/loc.api/di/types.js b/workers/loc.api/di/types.js index b0c3d55d9..23f2b4eba 100644 --- a/workers/loc.api/di/types.js +++ b/workers/loc.api/di/types.js @@ -62,5 +62,6 @@ module.exports = { WinLossVSAccountBalance: Symbol.for('WinLossVSAccountBalance'), DBBackupManager: Symbol.for('DBBackupManager'), ProcessMessageManager: Symbol.for('ProcessMessageManager'), - ProcessMessageManagerFactory: Symbol.for('ProcessMessageManagerFactory') + ProcessMessageManagerFactory: Symbol.for('ProcessMessageManagerFactory'), + TotalFeesReport: Symbol.for('TotalFeesReport') } diff --git a/workers/loc.api/errors/index.js b/workers/loc.api/errors/index.js index 52fb8b1d7..b1a92b932 100644 --- a/workers/loc.api/errors/index.js +++ b/workers/loc.api/errors/index.js @@ -3,7 +3,8 @@ const { BaseError, AuthError, - ConflictError + ConflictError, + ArgsParamsError } = require('bfx-report/workers/loc.api/errors') class CollSyncPermissionError extends BaseError { @@ -180,6 +181,12 @@ class DbRestoringError extends BaseError { } } +class TotalFeesParamsFlagError extends ArgsParamsError { + constructor (message = 'ERR_TOTAL_FEES_REPORT_PARAMS_FLAGS_MUST_HAVE_AT_LEAST_ONCE_TRUE_VALUE') { + super(message) + } +} + module.exports = { BaseError, CollSyncPermissionError, @@ -207,5 +214,6 @@ module.exports = { DataConsistencyError, DataConsistencyWhileSyncingError, ProcessStateSendingError, - DbRestoringError + DbRestoringError, + TotalFeesParamsFlagError } diff --git a/workers/loc.api/generate-csv/csv.job.data.js b/workers/loc.api/generate-csv/csv.job.data.js index 854d07be6..456f0d0b1 100644 --- a/workers/loc.api/generate-csv/csv.job.data.js +++ b/workers/loc.api/generate-csv/csv.job.data.js @@ -585,6 +585,44 @@ class CsvJobData extends BaseCsvJobData { return jobData } + async getTotalFeesReportCsvJobData ( + args, + uId, + uInfo + ) { + checkParams(args, 'paramsSchemaForTotalFeesReportCsv') + + const { + userId, + userInfo + } = await checkJobAndGetUserData( + this.rService, + uId, + uInfo + ) + + const csvArgs = getCsvArgs(args) + + const jobData = { + userInfo, + userId, + name: 'getTotalFeesReport', + fileNamesMap: [['getTotalFeesReport', 'total-fees-report']], + args: csvArgs, + propNameForPagination: null, + columnsCsv: { + USD: 'USD', + cumulative: 'CUMULATIVE USD', + mts: 'DATE' + }, + formatSettings: { + mts: 'date' + } + } + + return jobData + } + async getPerformingLoanCsvJobData ( args, uId, diff --git a/workers/loc.api/helpers/schema.js b/workers/loc.api/helpers/schema.js index 4c11c4b4e..8eb8e4b27 100644 --- a/workers/loc.api/helpers/schema.js +++ b/workers/loc.api/helpers/schema.js @@ -307,6 +307,36 @@ const paramsSchemaForFeesReportApi = { } } +const paramsSchemaForTotalFeesReportApi = { + type: 'object', + properties: { + timeframe: { + type: 'string', + enum: [ + 'day', + 'week', + 'month', + 'year' + ] + }, + start: { + type: 'integer' + }, + end: { + type: 'integer' + }, + symbol: { + type: ['string', 'array'] + }, + isTradingFees: { + type: 'boolean' + }, + isFundingFees: { + type: 'boolean' + } + } +} + const paramsSchemaForPerformingLoanApi = { type: 'object', properties: { @@ -412,6 +442,15 @@ const paramsSchemaForFeesReportCsv = { } } +const paramsSchemaForTotalFeesReportCsv = { + type: 'object', + properties: { + ...cloneDeep(paramsSchemaForTotalFeesReportApi.properties), + timezone, + dateFormat + } +} + const paramsSchemaForPerformingLoanCsv = { type: 'object', properties: { @@ -444,6 +483,7 @@ module.exports = { paramsSchemaForFullTaxReportApi, paramsSchemaForTradedVolumeApi, paramsSchemaForFeesReportApi, + paramsSchemaForTotalFeesReportApi, paramsSchemaForPerformingLoanApi, paramsSchemaForCandlesApi, paramsSchemaForBalanceHistoryCsv, @@ -454,6 +494,7 @@ module.exports = { paramsSchemaForFullTaxReportCsv, paramsSchemaForTradedVolumeCsv, paramsSchemaForFeesReportCsv, + paramsSchemaForTotalFeesReportCsv, paramsSchemaForPerformingLoanCsv, paramsSchemaForCandlesCsv } diff --git a/workers/loc.api/service.report.framework.js b/workers/loc.api/service.report.framework.js index 36f69aaef..ce55e0a6a 100644 --- a/workers/loc.api/service.report.framework.js +++ b/workers/loc.api/service.report.framework.js @@ -1266,6 +1266,9 @@ class FrameworkReportService extends ReportService { }, 'getTradedVolume', args, cb) } + /** + * @deprecated + */ getFeesReport (space, args, cb) { return this._privResponder(async () => { await this._dataConsistencyChecker @@ -1277,6 +1280,17 @@ class FrameworkReportService extends ReportService { }, 'getFeesReport', args, cb) } + getTotalFeesReport (space, args, cb) { + return this._privResponder(async () => { + await this._dataConsistencyChecker + .check(this._CHECKER_NAMES.TOTAL_FEES_REPORT, args) + + checkParams(args, 'paramsSchemaForTotalFeesReportApi') + + return this._totalFeesReport.getTotalFeesReport(args) + }, 'getTotalFeesReport', args, cb) + } + getPerformingLoan (space, args, cb) { return this._privResponder(async () => { await this._dataConsistencyChecker @@ -1366,6 +1380,9 @@ class FrameworkReportService extends ReportService { }, 'getTradedVolumeCsv', args, cb) } + /** + * @deprecated + */ getFeesReportCsv (space, args, cb) { return this._responder(() => { return this._generateCsv( @@ -1375,6 +1392,15 @@ class FrameworkReportService extends ReportService { }, 'getFeesReportCsv', args, cb) } + getTotalFeesReportCsv (space, args, cb) { + return this._responder(() => { + return this._generateCsv( + 'getTotalFeesReportCsvJobData', + args + ) + }, 'getTotalFeesReportCsv', args, cb) + } + getPerformingLoanCsv (space, args, cb) { return this._responder(() => { return this._generateCsv( diff --git a/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v30.js b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v30.js new file mode 100644 index 000000000..d9f94d571 --- /dev/null +++ b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v30.js @@ -0,0 +1,31 @@ +'use strict' + +const AbstractMigration = require('./abstract.migration') + +class MigrationV30 extends AbstractMigration { + /** + * @override + */ + up () { + const sqlArr = [ + `UPDATE ledgers SET _category = 228 + WHERE description LIKE '%margin funding fee%' COLLATE NOCASE` + ] + + this.addSql(sqlArr) + } + + /** + * @override + */ + down () { + const sqlArr = [ + `UPDATE ledgers SET _category = null + WHERE description LIKE '%margin funding fee%' COLLATE NOCASE` + ] + + this.addSql(sqlArr) + } +} + +module.exports = MigrationV30 diff --git a/workers/loc.api/sync/data.consistency.checker/checker.names.js b/workers/loc.api/sync/data.consistency.checker/checker.names.js index 83165f705..c9fd12052 100644 --- a/workers/loc.api/sync/data.consistency.checker/checker.names.js +++ b/workers/loc.api/sync/data.consistency.checker/checker.names.js @@ -9,5 +9,6 @@ module.exports = { FULL_TAX_REPORT: 'getFullTaxReport', TRADED_VOLUME: 'getTradedVolume', FEES_REPORT: 'getFeesReport', + TOTAL_FEES_REPORT: 'getTotalFeesReport', PERFORMING_LOAN: 'getPerformingLoan' } diff --git a/workers/loc.api/sync/data.consistency.checker/checkers.js b/workers/loc.api/sync/data.consistency.checker/checkers.js index c4cce8b98..4cb30eb84 100644 --- a/workers/loc.api/sync/data.consistency.checker/checkers.js +++ b/workers/loc.api/sync/data.consistency.checker/checkers.js @@ -129,6 +129,19 @@ class Checkers { }) } + [CHECKER_NAMES.TOTAL_FEES_REPORT] (auth) { + return this.syncCollsManager + .haveCollsBeenSyncedUpToDate({ + auth, + params: { + schema: [ + this.SYNC_API_METHODS.LEDGERS, + this.SYNC_API_METHODS.CANDLES + ] + } + }) + } + async [CHECKER_NAMES.PERFORMING_LOAN] (auth) { const { _id: userId, diff --git a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js index aa75f2834..35db9c835 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js @@ -143,6 +143,12 @@ const _schema = [ ), category: 226 }, + { + tester: (d) => ( + _includes(d, 'margin funding fee') + ), + category: 228 + }, { tester: (d) => ( _startsWith(d, 'earned fee') || diff --git a/workers/loc.api/sync/schema/models.js b/workers/loc.api/sync/schema/models.js index 4906c61e7..058040802 100644 --- a/workers/loc.api/sync/schema/models.js +++ b/workers/loc.api/sync/schema/models.js @@ -8,7 +8,7 @@ * e.g. `migration.v1.js`, where `v1` is `SUPPORTED_DB_VERSION` */ -const SUPPORTED_DB_VERSION = 29 +const SUPPORTED_DB_VERSION = 30 const TABLES_NAMES = require('./tables-names') const { diff --git a/workers/loc.api/sync/total.fees.report/index.js b/workers/loc.api/sync/total.fees.report/index.js new file mode 100644 index 000000000..4ce7fde96 --- /dev/null +++ b/workers/loc.api/sync/total.fees.report/index.js @@ -0,0 +1,225 @@ +'use strict' + +const { merge } = require('lodash') + +const { + TotalFeesParamsFlagError +} = require('../../errors') +const { + calcGroupedData, + groupByTimeframe +} = require('../helpers') + +const { decorateInjectable } = require('../../di/utils') + +const depsTypes = (TYPES) => [ + TYPES.DAO, + TYPES.ALLOWED_COLLS, + TYPES.SyncSchema, + TYPES.FOREX_SYMBS, + TYPES.Authenticator, + TYPES.SYNC_API_METHODS +] +class TotalFeesReport { + constructor ( + dao, + ALLOWED_COLLS, + syncSchema, + FOREX_SYMBS, + authenticator, + SYNC_API_METHODS + ) { + this.dao = dao + this.ALLOWED_COLLS = ALLOWED_COLLS + this.syncSchema = syncSchema + this.FOREX_SYMBS = FOREX_SYMBS + this.authenticator = authenticator + this.SYNC_API_METHODS = SYNC_API_METHODS + + this.ledgersMethodColl = this.syncSchema.getMethodCollMap() + .get(this.SYNC_API_METHODS.LEDGERS) + this.ledgersModel = this.syncSchema.getModelsMap() + .get(this.ALLOWED_COLLS.LEDGERS) + } + + async getTotalFeesReport (args = {}) { + const { + auth, + params + } = args ?? {} + const { + start = 0, + end = Date.now(), + timeframe = 'day', + symbol: symbs + } = params ?? {} + const _symbol = Array.isArray(symbs) + ? symbs + : [symbs] + const symbol = _symbol.filter((s) => ( + s && typeof s === 'string' + )) + const filter = this._getLedgersFilter(params) + const _args = { + auth, + start, + end, + symbol, + filter + } + + const ledgers = await this._getLedgers(_args) + + const { + dateFieldName: ledgersDateFieldName, + symbolFieldName: ledgersSymbolFieldName + } = this.ledgersMethodColl + + const ledgersGroupedByTimeframe = await groupByTimeframe( + ledgers, + { timeframe, start, end }, + this.FOREX_SYMBS, + ledgersDateFieldName, + ledgersSymbolFieldName, + this._calcLedgers() + ) + + const groupedData = await calcGroupedData( + { ledgersGroupedByTimeframe }, + false, + this._getLedgersByTimeframe(), + true + ) + + return groupedData + } + + async _getLedgers ({ + auth, + start, + end, + symbol, + filter, + projection = this.ledgersModel + }) { + const user = await this.authenticator + .verifyRequestUser({ auth }) + + const symbFilter = ( + Array.isArray(symbol) && + symbol.length !== 0 + ) + ? { $in: { currency: symbol } } + : {} + const filterToSkipNotRecalcedBalance = user.isSubAccount + ? { _isBalanceRecalced: 1 } + : {} + const _filter = merge(filter, symbFilter) + + return this.dao.getElemsInCollBy( + this.ALLOWED_COLLS.LEDGERS, + { + filter: { + ..._filter, + user_id: user._id, + $lte: { mts: end }, + $gte: { mts: start }, + ...filterToSkipNotRecalcedBalance + }, + sort: [['mts', -1]], + projection, + exclude: ['user_id'], + isExcludePrivate: true + } + ) + } + + _getLedgersFilter (params) { + const { + isTradingFees, + isFundingFees + } = params ?? {} + + if ( + !isTradingFees && + !isFundingFees + ) { + throw new TotalFeesParamsFlagError() + } + + const _category = [] + + /* + * Considering 'category' filter + * https://docs.bitfinex.com/reference/rest-auth-ledgers + * workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js + */ + if (isTradingFees) { + _category.push(201) // trading fee + } + if (isFundingFees) { + _category.push( + 27, // position funding cost or interest charged + 226, // used margin funding charge + 228, // unused margin funding fee + 29 // derivatives funding event + ) + } + + return { $in: { _category } } + } + + _calcLedgers () { + return (data = []) => { + const res = data.reduce((accum, curr) => { + const { amountUsd } = curr ?? {} + + if (!Number.isFinite(amountUsd)) { + return accum + } + + const _amountUsd = amountUsd !== 0 + ? amountUsd * -1 + : amountUsd + accum.USD = Number.isFinite(accum.USD) + ? accum.USD + _amountUsd + : _amountUsd + + return accum + }, {}) + + return res + } + } + + _getLedgersByTimeframe () { + let cumulative = 0 + + return ({ ledgersGroupedByTimeframe = {} }) => { + cumulative = this._calcPrevAmount( + ledgersGroupedByTimeframe, + cumulative + ) + + return { + cumulative, + USD: ledgersGroupedByTimeframe.USD ?? 0 + } + } + } + + _calcPrevAmount (usdAmount, cumulative) { + const { USD: amount } = usdAmount ?? {} + const _cumulative = Number.isFinite(cumulative) + ? cumulative + : 0 + + return Number.isFinite(amount) + ? amount + _cumulative + : _cumulative + } +} + +decorateInjectable(TotalFeesReport, depsTypes) + +module.exports = TotalFeesReport