From 1705b43fc82132d5330e01a94e99ac30dc167d0f Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 25 Aug 2025 13:00:46 +0300 Subject: [PATCH 01/14] Remove filter normalizer to speed up requests --- workers/loc.api/generate-report-file/index.js | 29 +---- workers/loc.api/helpers/index.js | 2 - .../helpers/normalize-filter-params.js | 100 ------------------ .../loc.api/helpers/prepare-response/index.js | 6 +- 4 files changed, 7 insertions(+), 130 deletions(-) delete mode 100644 workers/loc.api/helpers/normalize-filter-params.js diff --git a/workers/loc.api/generate-report-file/index.js b/workers/loc.api/generate-report-file/index.js index 3fdda07b..3952cc55 100644 --- a/workers/loc.api/generate-report-file/index.js +++ b/workers/loc.api/generate-report-file/index.js @@ -1,8 +1,7 @@ 'use strict' const { - FILTER_API_METHOD_NAMES, - normalizeFilterParams + FILTER_API_METHOD_NAMES } = require('../helpers') const { getFilterValidationSchemaId @@ -74,26 +73,12 @@ const _filterApiMethodNameMap = Object.values(FILTER_API_METHOD_NAMES) return map }, new Map()) -const _truncateFileNameEnding = (name) => { - if (!name) { - return name - } - - const cleanedName = name - .replace(/^get/i, '') - .replace(/(file)|(csv)$/i, '') - - return `${cleanedName[0].toLowerCase()}${cleanedName.slice(1)}` -} - const _getFilterApiMethodNamesAndArgs = ( name, - reqArgs + args ) => { if (name !== 'getMultipleFileJobData') { const filterApiMethodName = _filterApiMethodNameMap.get(name) - const truncatedName = _truncateFileNameEnding(name) - const args = normalizeFilterParams(truncatedName, reqArgs) return [{ filterApiMethodName, @@ -101,22 +86,18 @@ const _getFilterApiMethodNamesAndArgs = ( }] } - const { params } = { ...reqArgs } - const { multiExport } = { ...params } + const multiExport = args?.params?.multiExport ?? [] const _multiExport = Array.isArray(multiExport) ? multiExport : [] return _multiExport.map((params) => { - const { method } = { ...params } - const name = `${method}JobData` - const truncatedName = _truncateFileNameEnding(method) - const args = normalizeFilterParams(truncatedName, { params }) + const name = `${params?.method}JobData` const filterApiMethodName = _filterApiMethodNameMap.get(name) return { filterApiMethodName, - args + args: { params } } }) } diff --git a/workers/loc.api/helpers/index.js b/workers/loc.api/helpers/index.js index 499cdaf9..9d775494 100644 --- a/workers/loc.api/helpers/index.js +++ b/workers/loc.api/helpers/index.js @@ -58,7 +58,6 @@ const checkJobAndGetUserData = require( ) const grcBfxReq = require('./grc-bfx-req') const filterResponse = require('./filter-response') -const normalizeFilterParams = require('./normalize-filter-params') const FILTER_API_METHOD_NAMES = require('./filter.api.method.names') const FILTER_CONDITIONS = require('./filter.conditions') const getDataFromApi = require('./get-data-from-api') @@ -110,7 +109,6 @@ module.exports = { checkJobAndGetUserData, grcBfxReq, filterResponse, - normalizeFilterParams, FILTER_API_METHOD_NAMES, FILTER_CONDITIONS, getDataFromApi, diff --git a/workers/loc.api/helpers/normalize-filter-params.js b/workers/loc.api/helpers/normalize-filter-params.js deleted file mode 100644 index 451275e8..00000000 --- a/workers/loc.api/helpers/normalize-filter-params.js +++ /dev/null @@ -1,100 +0,0 @@ -'use strict' - -const { cloneDeep } = require('lib-js-util-base') - -const FILTER_API_METHOD_NAMES = require('./filter.api.method.names') -const FILTER_CONDITIONS = require('./filter.conditions') - -const _filterConditionsNames = Object - .values(FILTER_CONDITIONS) - -const _isCondition = (propName) => { - return _filterConditionsNames.some((name) => { - return propName === name - }) -} - -const _normalizeProp = ( - filter, - propName, - propHandler = () => {}, - prevLevel = 0 -) => { - const level = prevLevel + 1 - - if (level > 3) { - return filter - } - - const filterKeys = Object.entries(filter) - - for (const [name, val] of filterKeys) { - if ( - name !== propName && - !_isCondition(name) - ) { - continue - } - if ( - val && - typeof val === 'object' && - !Array.isArray(val) - ) { - _normalizeProp( - val, - propName, - propHandler, - level - ) - - continue - } - if (name === propName) { - filter[name] = propHandler(val) - } - } - - return filter -} - -const _normalizers = { - [FILTER_API_METHOD_NAMES.PUBLIC_TRADES] (filter) { - return _normalizeProp(filter, 'rate', (val) => { - return typeof val === 'string' - ? Number.parseFloat(val) - : val - }) - } -} - -module.exports = ( - methodApi, - args = {} -) => { - const { params } = { ...args } - const { filter } = { ...params } - - if ( - !methodApi || - typeof methodApi !== 'string' || - !filter || - typeof filter !== 'object' || - Array.isArray(filter) || - Object.keys(filter).length === 0 - ) { - return args - } - - const normalizer = _normalizers[methodApi] - const normFilter = typeof normalizer === 'function' - ? normalizer(cloneDeep(filter)) - : filter - - return { - ...args, - params: { - ...params, - filter: normFilter - } - } -} diff --git a/workers/loc.api/helpers/prepare-response/index.js b/workers/loc.api/helpers/prepare-response/index.js index 2c6e6573..1c4a8195 100644 --- a/workers/loc.api/helpers/prepare-response/index.js +++ b/workers/loc.api/helpers/prepare-response/index.js @@ -3,7 +3,6 @@ const { MinLimitParamError } = require('../../errors') const filterResponse = require('../filter-response') -const normalizeFilterParams = require('../normalize-filter-params') const prepareSymbolResponse = require('./prepare-symbol-response') @@ -130,7 +129,7 @@ const prepareApiResponse = ( getREST, dataValidator ) => async ( - reqArgs, + args, apiMethodName, params = {} ) => { @@ -143,8 +142,7 @@ const prepareApiResponse = ( const schemaId = getValidationSchemaId(apiMethodName) const filterSchemaId = getFilterValidationSchemaId(apiMethodName) - await dataValidator.validate(reqArgs, schemaId) - const args = normalizeFilterParams(apiMethodName, reqArgs) + await dataValidator.validate(args, schemaId) await dataValidator.validate( { params: args?.params?.filter }, filterSchemaId From e946efe356b02f22c3173d9f50f572ccce655e20 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 28 Aug 2025 14:30:31 +0300 Subject: [PATCH 02/14] Add ability to fetch symbol info form api when exporting --- workers/loc.api/queue/write-data-to-stream/index.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/workers/loc.api/queue/write-data-to-stream/index.js b/workers/loc.api/queue/write-data-to-stream/index.js index 252c84c1..82336383 100644 --- a/workers/loc.api/queue/write-data-to-stream/index.js +++ b/workers/loc.api/queue/write-data-to-stream/index.js @@ -45,6 +45,13 @@ module.exports = ( const currIterationArgs = cloneDeep(_args) const getData = rService[method].bind(rService) + const getSymbols = rService.getSymbols.bind(rService) + const symbols = await getDataFromApi({ + getSymbols, + args: {}, + callerName: 'REPORT_FILE_WRITER', + shouldNotInterrupt: true + }) let count = 0 let serialRequestsCount = 0 @@ -101,7 +108,7 @@ module.exports = ( res, stream, formatSettings, - { ..._args.params }, + { ..._args.params, symbols }, method ) processorQueue.emit('progress', 100) @@ -142,7 +149,7 @@ module.exports = ( res, stream, formatSettings, - { ..._args.params }, + { ..._args.params, symbols }, method ) From 34e81b77e5af42d3b25653da9eb9cf3827ef630c Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 28 Aug 2025 14:30:53 +0300 Subject: [PATCH 03/14] Implement ccy formatter for tether with network --- .../queue/write-data-to-stream/helpers.js | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/workers/loc.api/queue/write-data-to-stream/helpers.js b/workers/loc.api/queue/write-data-to-stream/helpers.js index 80db494e..91428527 100644 --- a/workers/loc.api/queue/write-data-to-stream/helpers.js +++ b/workers/loc.api/queue/write-data-to-stream/helpers.js @@ -14,6 +14,31 @@ const _validTxtTimeZone = (val, timezone, format) => { } const _formatters = { + prepareCurrency: (ccy, params) => { + const currencies = params?.symbols?.currencies + const currencyName = params?.allDataFields?.currencyName + + if ( + !ccy || + typeof ccy !== 'string' || + !Array.isArray(currencies) || + !currencyName || + typeof currencyName !== 'string' + ) { + return ccy + } + if ( + ccy !== 'USDt' && + ccy !== 'UST' + ) { + return ccy + } + + const ccyId = currencyName.replace('TETHER', '') + const name = currencies.find((item) => item?.name === ccyId) + + return name ?? ccy + }, date: ( val, { @@ -112,6 +137,11 @@ const _dataFormatter = (data, formatSettings, params) => { const objArr = isArray ? clonedData : [clonedData] for (const obj of objArr) { + const formatterParams = { + ...params, + allDataFields: obj + } + for (const [key, val] of Object.entries(formatSettings)) { try { if ( @@ -119,7 +149,7 @@ const _dataFormatter = (data, formatSettings, params) => { val && typeof val === 'object' ) { - obj[key] = _dataFormatter(obj[key], val, params) + obj[key] = _dataFormatter(obj[key], val, formatterParams) continue } @@ -127,7 +157,7 @@ const _dataFormatter = (data, formatSettings, params) => { typeof obj[key] !== 'undefined' && typeof _formatters[val] === 'function' ) { - obj[key] = _formatters[val](obj[key], params) + obj[key] = _formatters[val](obj[key], formatterParams) continue } From a11c76e6ace98e07c1c7e478a33afdc157a7bb58 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Thu, 28 Aug 2025 14:31:13 +0300 Subject: [PATCH 04/14] Add ccy formatter to movement export --- workers/loc.api/generate-report-file/report.file.job.data.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workers/loc.api/generate-report-file/report.file.job.data.js b/workers/loc.api/generate-report-file/report.file.job.data.js index 5616d156..283f3a09 100644 --- a/workers/loc.api/generate-report-file/report.file.job.data.js +++ b/workers/loc.api/generate-report-file/report.file.job.data.js @@ -894,7 +894,8 @@ class ReportFileJobData { note: 'NOTE' }, formatSettings: { - mtsUpdated: 'date' + mtsUpdated: 'date', + currency: 'prepareCurrency' } } From 09b793e4d1a82a2de113dafaa5f257e74187680e Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 29 Aug 2025 11:31:36 +0300 Subject: [PATCH 05/14] Add currency name map to report export job --- workers/loc.api/queue/write-data-to-stream/index.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/workers/loc.api/queue/write-data-to-stream/index.js b/workers/loc.api/queue/write-data-to-stream/index.js index 82336383..a0b1b263 100644 --- a/workers/loc.api/queue/write-data-to-stream/index.js +++ b/workers/loc.api/queue/write-data-to-stream/index.js @@ -46,12 +46,17 @@ module.exports = ( const getData = rService[method].bind(rService) const getSymbols = rService.getSymbols.bind(rService) - const symbols = await getDataFromApi({ - getSymbols, + const symbols = (await getDataFromApi({ + getData: getSymbols, args: {}, callerName: 'REPORT_FILE_WRITER', shouldNotInterrupt: true - }) + })) ?? {} + symbols.currencyNameMap = symbols.currencies.reduce((accum, curr) => { + accum[curr?.id] = curr?.name + + return accum + }, {}) let count = 0 let serialRequestsCount = 0 From be0e8add5f5a339d8b3503291db75ebba918dd4f Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 29 Aug 2025 11:32:05 +0300 Subject: [PATCH 06/14] Speed up movement ccy formatting using name map --- workers/loc.api/queue/write-data-to-stream/helpers.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/workers/loc.api/queue/write-data-to-stream/helpers.js b/workers/loc.api/queue/write-data-to-stream/helpers.js index 91428527..4c7adefe 100644 --- a/workers/loc.api/queue/write-data-to-stream/helpers.js +++ b/workers/loc.api/queue/write-data-to-stream/helpers.js @@ -15,15 +15,16 @@ const _validTxtTimeZone = (val, timezone, format) => { const _formatters = { prepareCurrency: (ccy, params) => { - const currencies = params?.symbols?.currencies const currencyName = params?.allDataFields?.currencyName + const currencyNameMap = params?.symbols?.currencyNameMap if ( !ccy || typeof ccy !== 'string' || - !Array.isArray(currencies) || !currencyName || - typeof currencyName !== 'string' + typeof currencyName !== 'string' || + !currencyNameMap || + typeof currencyNameMap !== 'object' ) { return ccy } @@ -35,7 +36,7 @@ const _formatters = { } const ccyId = currencyName.replace('TETHER', '') - const name = currencies.find((item) => item?.name === ccyId) + const name = currencyNameMap[ccyId] return name ?? ccy }, From f2060c0a0f0e7b65419b15cc2b5c978ae606fa32 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 29 Aug 2025 11:32:26 +0300 Subject: [PATCH 07/14] Adjust mocked symbols for tests --- test/5-queue-load.spec.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/5-queue-load.spec.js b/test/5-queue-load.spec.js index f4f82d3d..929d5817 100644 --- a/test/5-queue-load.spec.js +++ b/test/5-queue-load.spec.js @@ -47,7 +47,14 @@ describe('Queue load', () => { mockRESTv2Srv = createMockRESTv2SrvWithDate(start, end, null, { ledgers: { limit }, - user_info: null + user_info: null, + symbols: null, + map_symbols: null, + inactive_currencies: null, + margin_currencies: null, + inactive_symbols: null, + futures: null, + currencies: null }) await rmAllFiles(tempDirPath) From fd119b28b9bd18d91fbdaca491c5d3d5a23d7adc Mon Sep 17 00:00:00 2001 From: bilalfastian Date: Sat, 6 Sep 2025 01:11:05 +1000 Subject: [PATCH 08/14] Update npmrc --- .npmrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.npmrc b/.npmrc index cffe8cde..37890260 100644 --- a/.npmrc +++ b/.npmrc @@ -1 +1,3 @@ save-exact=true +foreground-scripts=true +ignore-scripts=true From f833f89f171a81654d4b83f40527d5707866923d Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Tue, 16 Sep 2025 09:41:18 +0300 Subject: [PATCH 09/14] Add created-updated mts for movements export --- workers/loc.api/generate-report-file/report.file.job.data.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/workers/loc.api/generate-report-file/report.file.job.data.js b/workers/loc.api/generate-report-file/report.file.job.data.js index 283f3a09..4b6886b0 100644 --- a/workers/loc.api/generate-report-file/report.file.job.data.js +++ b/workers/loc.api/generate-report-file/report.file.job.data.js @@ -884,7 +884,8 @@ class ReportFileJobData { propNameForPagination: 'mtsUpdated', columnsCsv: { id: '#', - mtsUpdated: 'DATE', + mtsStarted: 'STARTED', + mtsUpdated: 'UPDATED', currency: 'CURRENCY', status: 'STATUS', amount: 'AMOUNT', @@ -894,6 +895,7 @@ class ReportFileJobData { note: 'NOTE' }, formatSettings: { + mtsStarted: 'date', mtsUpdated: 'date', currency: 'prepareCurrency' } From c7a4b3aad15d2a91654563db0ee5249f6c42aac9 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 19 Sep 2025 10:17:12 +0300 Subject: [PATCH 10/14] Remove payInvoiceList endpoint support --- .../getPayInvoiceListReqFilter.js | 8 --- .../filter-schemas/helpers/filter-models.js | 15 ------ .../data-validator/filter.schema.names.js | 3 +- .../loc.api/data-validator/schema.names.js | 1 - .../schemas/getPayInvoiceListReq.js | 39 -------------- .../report.file.job.data.js | 52 ------------------- .../helpers/filter.api.method.names.js | 3 +- .../loc.api/helpers/limit-param.helpers.js | 1 - .../get-filter-validation-schema-id.js | 4 +- .../helpers/get-params-map.js | 4 -- .../helpers/get-params-schema-name.js | 19 ------- .../helpers/get-symbol-params.js | 3 -- .../helpers/get-symbols-for-filtering.js | 3 +- .../helpers/get-validation-schema-id.js | 1 - .../helpers/prepare-response/helpers/index.js | 2 - workers/loc.api/service.report.js | 22 -------- 16 files changed, 4 insertions(+), 176 deletions(-) delete mode 100644 workers/loc.api/data-validator/filter-schemas/getPayInvoiceListReqFilter.js delete mode 100644 workers/loc.api/data-validator/schemas/getPayInvoiceListReq.js delete mode 100644 workers/loc.api/helpers/prepare-response/helpers/get-params-schema-name.js diff --git a/workers/loc.api/data-validator/filter-schemas/getPayInvoiceListReqFilter.js b/workers/loc.api/data-validator/filter-schemas/getPayInvoiceListReqFilter.js deleted file mode 100644 index e1c39b40..00000000 --- a/workers/loc.api/data-validator/filter-schemas/getPayInvoiceListReqFilter.js +++ /dev/null @@ -1,8 +0,0 @@ -'use strict' - -const FILTER_SCHEMA_IDS = require('../filter.schema.ids') -const { getFilterSchema } = require('./helpers') - -module.exports = getFilterSchema( - FILTER_SCHEMA_IDS.GET_PAY_INVOICE_LIST_REQ_FILTER -) diff --git a/workers/loc.api/data-validator/filter-schemas/helpers/filter-models.js b/workers/loc.api/data-validator/filter-schemas/helpers/filter-models.js index ca03c41d..35ac9a80 100644 --- a/workers/loc.api/data-validator/filter-schemas/helpers/filter-models.js +++ b/workers/loc.api/data-validator/filter-schemas/helpers/filter-models.js @@ -235,20 +235,5 @@ module.exports = new Map([ low: { type: 'number' }, volume: { type: 'number' } } - ], - [ - FILTER_SCHEMA_IDS.GET_PAY_INVOICE_LIST_REQ_FILTER, - { - id: { type: 'string' }, - t: { type: 'integer' }, - duration: { type: 'number' }, - amount: { type: 'number' }, - currency: { type: 'string' }, - orderId: { type: 'string' }, - webhook: { type: 'string' }, - redirectUrl: { type: 'string' }, - status: { type: 'string' }, - merchantName: { type: 'string' } - } ] ]) diff --git a/workers/loc.api/data-validator/filter.schema.names.js b/workers/loc.api/data-validator/filter.schema.names.js index 46bb8756..d33ea47f 100644 --- a/workers/loc.api/data-validator/filter.schema.names.js +++ b/workers/loc.api/data-validator/filter.schema.names.js @@ -15,6 +15,5 @@ module.exports = { GET_STATUS_MESSAGES_REQ_FILTER: 'getStatusMessagesReqFilter', GET_LOGINS_REQ_FILTER: 'getLoginsReqFilter', GET_CHANGE_LOGS_REQ_FILTER: 'getChangeLogsReqFilter', - GET_CANDLES_REQ_FILTER: 'getCandlesReqFilter', - GET_PAY_INVOICE_LIST_REQ_FILTER: 'getPayInvoiceListReqFilter' + GET_CANDLES_REQ_FILTER: 'getCandlesReqFilter' } diff --git a/workers/loc.api/data-validator/schema.names.js b/workers/loc.api/data-validator/schema.names.js index 9bc39edd..cd1e1e76 100644 --- a/workers/loc.api/data-validator/schema.names.js +++ b/workers/loc.api/data-validator/schema.names.js @@ -5,7 +5,6 @@ module.exports = { COMMON_REQ: 'commonReq', GET_WEIGHTED_AVERAGES_REPORT_REQ: 'getWeightedAveragesReportReq', - GET_PAY_INVOICE_LIST_REQ: 'getPayInvoiceListReq', GET_STATUS_MESSAGES_REQ: 'getStatusMessagesReq', GET_PUBLIC_TRADES_REQ: 'getPublicTradesReq', GET_POSITIONS_AUDIT_REQ: 'getPositionsAuditReq', diff --git a/workers/loc.api/data-validator/schemas/getPayInvoiceListReq.js b/workers/loc.api/data-validator/schemas/getPayInvoiceListReq.js deleted file mode 100644 index 00eedecf..00000000 --- a/workers/loc.api/data-validator/schemas/getPayInvoiceListReq.js +++ /dev/null @@ -1,39 +0,0 @@ -'use strict' - -const SCHEMA_IDS = require('../schema.ids') - -module.exports = { - $id: SCHEMA_IDS.GET_PAY_INVOICE_LIST_REQ, - type: 'object', - additionalProperties: false, - properties: { - start: { - $ref: 'defs#/definitions/start' - }, - end: { - $ref: 'defs#/definitions/end' - }, - limit: { - $ref: 'defs#/definitions/limit' - }, - symbol: { - $ref: 'defs#/definitions/symbolWithMinItem' - }, - id: { - $ref: 'defs#/definitions/strId' - }, - filter: { - $ref: 'defs#/definitions/filter' - }, - - notCheckNextPage: { - $ref: 'defs#/definitions/notCheckNextPage' - }, - notThrowError: { - $ref: 'defs#/definitions/notThrowError' - }, - isSyncRequest: { - $ref: 'defs#/definitions/isSyncRequest' - } - } -} diff --git a/workers/loc.api/generate-report-file/report.file.job.data.js b/workers/loc.api/generate-report-file/report.file.job.data.js index 4b6886b0..5160850a 100644 --- a/workers/loc.api/generate-report-file/report.file.job.data.js +++ b/workers/loc.api/generate-report-file/report.file.job.data.js @@ -632,58 +632,6 @@ class ReportFileJobData { return jobData } - async getPayInvoiceListFileJobData ( - args, - uId, - uInfo - ) { - this.dataValidator.validate( - args, - this.dataValidator.SCHEMA_IDS.COMMON_FILE_REQ - ) - - const { - userId, - userInfo - } = await checkJobAndGetUserData( - this.rService, - uId, - uInfo - ) - - const reportFileArgs = getReportFileArgs( - args, - { - method: 'payInvoiceList', - extraParams: { notThrowError: true } - } - ) - - const jobData = { - userInfo, - userId, - name: 'getPayInvoiceList', - args: reportFileArgs, - propNameForPagination: 't', - columnsCsv: { - id: '#', - t: 'DATE', - duration: 'DURATION', - amount: 'AMOUNT', - currency: 'CURRENCY', - orderId: 'ORDER ID', - payCurrencies: 'PAY CURRENCIES', - status: 'STATUS', - merchantName: 'MERCHANT NAME' - }, - formatSettings: { - t: 'date' - } - } - - return jobData - } - async getOrderTradesFileJobData ( args, uId, diff --git a/workers/loc.api/helpers/filter.api.method.names.js b/workers/loc.api/helpers/filter.api.method.names.js index 04ba4e64..31173d57 100644 --- a/workers/loc.api/helpers/filter.api.method.names.js +++ b/workers/loc.api/helpers/filter.api.method.names.js @@ -26,6 +26,5 @@ module.exports = { STATUS_MESSAGES: 'statusMessages', LOGINS: 'logins', CHANGE_LOGS: 'changeLogs', - CANDLES: 'candles', - PAY_INVOICE_LIST: 'payInvoiceList' + CANDLES: 'candles' } diff --git a/workers/loc.api/helpers/limit-param.helpers.js b/workers/loc.api/helpers/limit-param.helpers.js index e1f4b5fd..b64cfdf1 100644 --- a/workers/loc.api/helpers/limit-param.helpers.js +++ b/workers/loc.api/helpers/limit-param.helpers.js @@ -19,7 +19,6 @@ const getMethodLimit = (sendLimit, method, methodsLimits = {}) => { candles: { default: 500, max: 500, innerMax: 10000 }, changeLogs: { default: 500, max: 500, innerMax: 500 }, positionsSnapshot: { default: 50, max: 500, innerMax: 500 }, - payInvoiceList: { default: 10, max: 100, innerMax: 100 }, ...methodsLimits } diff --git a/workers/loc.api/helpers/prepare-response/helpers/get-filter-validation-schema-id.js b/workers/loc.api/helpers/prepare-response/helpers/get-filter-validation-schema-id.js index c87d0ac3..09b87d5c 100644 --- a/workers/loc.api/helpers/prepare-response/helpers/get-filter-validation-schema-id.js +++ b/workers/loc.api/helpers/prepare-response/helpers/get-filter-validation-schema-id.js @@ -41,9 +41,7 @@ const PARAMS_SCHEMAS_MAP = { [FILTER_API_METHOD_NAMES.CHANGE_LOGS]: FILTER_SCHEMA_IDS .GET_CHANGE_LOGS_REQ_FILTER, [FILTER_API_METHOD_NAMES.CANDLES]: FILTER_SCHEMA_IDS - .GET_CANDLES_REQ_FILTER, - [FILTER_API_METHOD_NAMES.PAY_INVOICE_LIST]: FILTER_SCHEMA_IDS - .GET_PAY_INVOICE_LIST_REQ_FILTER + .GET_CANDLES_REQ_FILTER } module.exports = ( diff --git a/workers/loc.api/helpers/prepare-response/helpers/get-params-map.js b/workers/loc.api/helpers/prepare-response/helpers/get-params-map.js index ef4f4197..f70c446b 100644 --- a/workers/loc.api/helpers/prepare-response/helpers/get-params-map.js +++ b/workers/loc.api/helpers/prepare-response/helpers/get-params-map.js @@ -45,10 +45,6 @@ const PARAMS_MAP = { ...MAIN_PARAMS_MAP, symbol: 'symbols' }, - payInvoiceList: { - ...MAIN_PARAMS_MAP, - id: 'id' - }, trades: { ...DEFAULT_PARAMS_MAP, sort: 'sort' diff --git a/workers/loc.api/helpers/prepare-response/helpers/get-params-schema-name.js b/workers/loc.api/helpers/prepare-response/helpers/get-params-schema-name.js deleted file mode 100644 index d210ea30..00000000 --- a/workers/loc.api/helpers/prepare-response/helpers/get-params-schema-name.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -const PARAMS_SCHEMAS_MAP = { - default: 'paramsSchemaForApi', - - payInvoiceList: 'paramsSchemaForPayInvoiceList', - statusMessages: 'paramsSchemaForStatusMessagesApi', - publicTrades: 'paramsSchemaForPublicTrades', - positionsAudit: 'paramsSchemaForPositionsAudit', - orderTrades: 'paramsSchemaForOrderTradesApi', - candles: 'paramsSchemaForCandlesApi' -} - -module.exports = ( - method, - map = PARAMS_SCHEMAS_MAP -) => { - return map?.[method] ?? map.default -} diff --git a/workers/loc.api/helpers/prepare-response/helpers/get-symbol-params.js b/workers/loc.api/helpers/prepare-response/helpers/get-symbol-params.js index 48a85599..fcb645be 100644 --- a/workers/loc.api/helpers/prepare-response/helpers/get-symbol-params.js +++ b/workers/loc.api/helpers/prepare-response/helpers/get-symbol-params.js @@ -19,9 +19,6 @@ module.exports = ({ category } = params ?? {} - if (apiMethodName === 'payInvoiceList') { - return { symbol: null } - } if ( apiMethodName === 'candles' || apiMethodName === 'publicTrades' diff --git a/workers/loc.api/helpers/prepare-response/helpers/get-symbols-for-filtering.js b/workers/loc.api/helpers/prepare-response/helpers/get-symbols-for-filtering.js index 81cbd146..9a0e9b11 100644 --- a/workers/loc.api/helpers/prepare-response/helpers/get-symbols-for-filtering.js +++ b/workers/loc.api/helpers/prepare-response/helpers/get-symbols-for-filtering.js @@ -18,8 +18,7 @@ module.exports = ({ if ( apiMethodName === 'positionsHistory' || - apiMethodName === 'positionsAudit' || - apiMethodName === 'payInvoiceList' + apiMethodName === 'positionsAudit' ) { return Array.isArray(symbol) ? [...symbol] diff --git a/workers/loc.api/helpers/prepare-response/helpers/get-validation-schema-id.js b/workers/loc.api/helpers/prepare-response/helpers/get-validation-schema-id.js index 7d2012f0..b8628e47 100644 --- a/workers/loc.api/helpers/prepare-response/helpers/get-validation-schema-id.js +++ b/workers/loc.api/helpers/prepare-response/helpers/get-validation-schema-id.js @@ -5,7 +5,6 @@ const { SCHEMA_IDS } = require('../../../data-validator') const PARAMS_SCHEMAS_MAP = { default: SCHEMA_IDS.COMMON_REQ, - payInvoiceList: SCHEMA_IDS.GET_PAY_INVOICE_LIST_REQ, statusMessages: SCHEMA_IDS.GET_STATUS_MESSAGES_REQ, publicTrades: SCHEMA_IDS.GET_PUBLIC_TRADES_REQ, positionsAudit: SCHEMA_IDS.GET_POSITIONS_AUDIT_REQ, diff --git a/workers/loc.api/helpers/prepare-response/helpers/index.js b/workers/loc.api/helpers/prepare-response/helpers/index.js index d63190d3..e24b7416 100644 --- a/workers/loc.api/helpers/prepare-response/helpers/index.js +++ b/workers/loc.api/helpers/prepare-response/helpers/index.js @@ -1,7 +1,6 @@ 'use strict' const getParamsMap = require('./get-params-map') -const getParamsSchemaName = require('./get-params-schema-name') const getValidationSchemaId = require('./get-validation-schema-id') const getFilterValidationSchemaId = require('./get-filter-validation-schema-id') const omitPrivateModelFields = require('./omit-private-model-fields') @@ -13,7 +12,6 @@ const isNotContainedSameMts = require('./is-not-contained-same-mts') module.exports = { getParamsMap, - getParamsSchemaName, getValidationSchemaId, getFilterValidationSchemaId, omitPrivateModelFields, diff --git a/workers/loc.api/service.report.js b/workers/loc.api/service.report.js index 99a4b154..be5fb906 100644 --- a/workers/loc.api/service.report.js +++ b/workers/loc.api/service.report.js @@ -392,19 +392,6 @@ class ReportService extends Api { }, 'getLedgers', args, cb) } - getPayInvoiceList (space, args, cb) { - return this._responder(() => { - return this._prepareApiResponse( - args, - 'payInvoiceList', - { - datePropName: 't', - symbPropName: 'currency' - } - ) - }, 'getInvoiceList', args, cb) - } - getTrades (space, args, cb) { return this._responder(() => { return this._prepareApiResponse( @@ -799,15 +786,6 @@ class ReportService extends Api { }, 'getLedgersFile', args, cb) } - getPayInvoiceListFile (space, args, cb) { - return this._responder(() => { - return this._generateReportFile( - 'getPayInvoiceListFileJobData', - args - ) - }, 'getPayInvoiceListFile', args, cb) - } - getOrderTradesFile (space, args, cb) { return this._responder(() => { return this._generateReportFile( From 410c1f13a2c7e3ff3d71d38fa82fead4b1480aaa Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 19 Sep 2025 10:17:33 +0300 Subject: [PATCH 11/14] Remove payInvoiceList endpoint test coverage --- test/1-api.spec.js | 46 ------------- test/helpers/helpers.mock-rest-v2.js | 9 +-- test/helpers/mock-data.js | 77 ---------------------- test/test-cases/report-file-generations.js | 28 -------- 4 files changed, 1 insertion(+), 159 deletions(-) diff --git a/test/1-api.spec.js b/test/1-api.spec.js index 7fc68770..93f87d24 100644 --- a/test/1-api.spec.js +++ b/test/1-api.spec.js @@ -812,52 +812,6 @@ describe('API', () => { ]) }) - it('it should be successfully performed by the getPayInvoiceList method', async function () { - this.timeout(5000) - - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getPayInvoiceList', - params: { - start: 0, - end, - limit: 2 - }, - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - assert.isObject(res.body) - assert.propertyVal(res.body, 'id', 5) - assert.isObject(res.body.result) - assert.isArray(res.body.result.res) - assert.isNumber(res.body.result.nextPage) - - const resItem = res.body.result.res[0] - - assert.isObject(resItem) - assert.containsAllKeys(resItem, [ - 'id', - 't', - 'duration', - 'amount', - 'currency', - 'orderId', - 'payCurrencies', - 'webhook', - 'redirectUrl', - 'status', - 'customerInfo', - 'invoices', - 'payment', - 'merchantName' - ]) - }) - it('it should be successfully performed by the getTrades method', async function () { this.timeout(5000) diff --git a/test/helpers/helpers.mock-rest-v2.js b/test/helpers/helpers.mock-rest-v2.js index 3807f4e2..f80f4c26 100644 --- a/test/helpers/helpers.mock-rest-v2.js +++ b/test/helpers/helpers.mock-rest-v2.js @@ -51,18 +51,12 @@ const setDataTo = ( const { date = Date.now(), id = 12345, - fee = -0.0001, - strId = uuidv4() + fee = -0.0001 } = { ...params } const _date = Math.round(date) switch (key) { - case 'invoice_list': - dataItem.id = strId - dataItem.t = _date - break - case 'candles': dataItem[0] = _date break @@ -207,7 +201,6 @@ const getMockDataOpts = () => ({ logins_hist: { limit: 250 }, change_log: { limit: 500 }, candles: { limit: 500 }, - invoice_list: { limit: 100 }, user_info: null, symbols: null, map_symbols: null, diff --git a/test/helpers/mock-data.js b/test/helpers/mock-data.js index 1511daac..ae11d980 100644 --- a/test/helpers/mock-data.js +++ b/test/helpers/mock-data.js @@ -706,83 +706,6 @@ module.exports = new Map([ 'pub:api:12ab12ab-12ab-12ab-12ab-12ab12ab12ab-read' ] ], - [ - 'invoice_list', - [ - { - id: '12345678-1234-1234-1234-12345678', - t: _ms, - duration: 900, - amount: 0.1, - currency: 'USD', - orderId: '12', - payCurrencies: [ - 'BTC', - 'ETH', - 'UST-ETH', - 'LNX' - ], - webhook: 'https://bfx-pay-123.321.com', - redirectUrl: 'https://bfx-pay-123.321.com?page_id123', - status: 'EXPIRED', - customerInfo: { - nationality: 'UA', - residCountry: 'UA', - residState: null, - residCity: 'Kharkiv', - residZipCode: '12345', - residStreet: 'street', - residBuildingNo: null, - fullName: 'name user', - email: 'user@mail.com', - tosAccepted: true - }, - invoices: [ - { - amount: 0.00000123, - payCurrency: 'BTC', - poolCurrency: 'BTC', - address: '1234567890qwerty' - }, - { - amount: 0.0000124, - payCurrency: 'ETH', - poolCurrency: 'ETH', - address: '1234567890qwerty' - }, - { - amount: 0.125, - payCurrency: 'UST', - poolCurrency: 'ETH', - address: '1234567890qwerty' - }, - { - amount: 0.00000125, - payCurrency: 'BTC', - poolCurrency: 'LNX', - address: '1234567890qwerty', - ext: { - hash: '1234567890qwerty', - payment_request: '1234567890qwerty', - node_pub: '1234567890qwerty' - } - } - ], - payment: { - txid: '1234567890qwerty', - amount: 0.123123, - currency: 'UST', - method: 'TETHERUSE', - status: 'COMPLETED', - confirmations: 5, - created_at: '2021-09-12 03:03:03', - updated_at: '2021-09-12 03:03:03', - depositId: 1234321 - }, - merchantName: 'Testing' - } - ] - ], [ 'weighted_averages', [ diff --git a/test/test-cases/report-file-generations.js b/test/test-cases/report-file-generations.js index 81b7b467..f5f34262 100644 --- a/test/test-cases/report-file-generations.js +++ b/test/test-cases/report-file-generations.js @@ -350,34 +350,6 @@ module.exports = ( await testMethodOfGettingReportFile(procPromise, aggrPromise, res) }) - it('it should be successfully performed by the getPayInvoiceListFile method', async function () { - this.timeout(60000) - - const procPromise = queueToPromise(params.processorQueue) - const aggrPromise = queueToPromise(params.aggregatorQueue) - - const res = await agent - .post(`${basePath}/json-rpc`) - .type('json') - .send({ - auth, - method: 'getPayInvoiceListFile', - params: { - isPDFRequired, - end, - start, - limit: 100, - timezone: -3, - email - }, - id: 5 - }) - .expect('Content-Type', /json/) - .expect(200) - - await testMethodOfGettingReportFile(procPromise, aggrPromise, res) - }) - it('it should be successfully performed by the getTradesFile method', async function () { this.timeout(60000) From d038c1236a87fb2e53ef9cdbd082e5ea82be3be4 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 19 Sep 2025 12:36:16 +0300 Subject: [PATCH 12/14] Fix getSymbols req auth for fw mode --- workers/loc.api/queue/write-data-to-stream/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/loc.api/queue/write-data-to-stream/index.js b/workers/loc.api/queue/write-data-to-stream/index.js index a0b1b263..4c8061f3 100644 --- a/workers/loc.api/queue/write-data-to-stream/index.js +++ b/workers/loc.api/queue/write-data-to-stream/index.js @@ -48,7 +48,7 @@ module.exports = ( const getSymbols = rService.getSymbols.bind(rService) const symbols = (await getDataFromApi({ getData: getSymbols, - args: {}, + args: { auth: { ...jobData?.args?.auth } }, callerName: 'REPORT_FILE_WRITER', shouldNotInterrupt: true })) ?? {} From bab757ddf3993b63dbdd0cd957e118ce7616b8a8 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Wed, 24 Sep 2025 08:26:57 +0300 Subject: [PATCH 13/14] Bump version up to 4.13.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 44aaff5d..5090fed6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "bfx-report", - "version": "4.12.6", + "version": "4.13.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "bfx-report", - "version": "4.12.6", + "version": "4.13.0", "license": "Apache-2.0", "dependencies": { "ajv": "8.17.1", diff --git a/package.json b/package.json index 695d3e47..cba8cbaf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bfx-report", - "version": "4.12.6", + "version": "4.13.0", "description": "Reporting tool", "main": "worker.js", "license": "Apache-2.0", From a9662c4022b8ed8032f0c84ea1e382a9a2bca25b Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Wed, 24 Sep 2025 08:28:08 +0300 Subject: [PATCH 14/14] Update bfx-report-express dep --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5090fed6..57f68d7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1258,8 +1258,8 @@ } }, "node_modules/bfx-report-express": { - "version": "2.0.8", - "resolved": "git+ssh://git@github.com/bitfinexcom/bfx-report-express.git#96b392dd8cc48a5cee6cc2d6b887686934e4b423", + "version": "2.0.9", + "resolved": "git+ssh://git@github.com/bitfinexcom/bfx-report-express.git#875436acf5ee1309270995f0edc456858e0c49db", "dev": true, "license": "Apache-2.0", "dependencies": {