diff --git a/locales/en/pdf.json b/locales/en/pdf.json index 37fdc566..7e848f05 100644 --- a/locales/en/pdf.json +++ b/locales/en/pdf.json @@ -9,7 +9,7 @@ "email": "Email", "errorMessage": "Your file could not be completed, please try again", "reportGenAt": "Report generated at", - "copyright": "Copyright © 2013-2024 iFinex Inc. All rights reserved.", + "copyright": "Copyright © 2013-{{currYear}} iFinex Inc. All rights reserved.", "page": "Page", "from": "from" } diff --git a/locales/es-EM/pdf.json b/locales/es-EM/pdf.json index 7764d527..551f8475 100644 --- a/locales/es-EM/pdf.json +++ b/locales/es-EM/pdf.json @@ -9,7 +9,7 @@ "email": "Correo", "errorMessage": "Tu archivo no pudo ser completado, por favor inténtalo de nuevo", "reportGenAt": "Reporte generado al", - "copyright": "Copyright © 2013-2024 iFinex Inc. Todos los derechos reservados.", + "copyright": "Copyright © 2013-{{currYear}} iFinex Inc. Todos los derechos reservados.", "page": "Página", "from": "de" } diff --git a/locales/pt-BR/pdf.json b/locales/pt-BR/pdf.json index f33b377e..84c3c04b 100644 --- a/locales/pt-BR/pdf.json +++ b/locales/pt-BR/pdf.json @@ -9,7 +9,7 @@ "email": "e-mail", "errorMessage": "Seu arquivo não pode ser completado, por favor tente novamente", "reportGenAt": "Informe gerado em ", - "copyright": "Copyright © 2013-2024 iFinex Inc. Todos os direitos reservados.", + "copyright": "Copyright © 2013-{{currYear}} iFinex Inc. Todos os direitos reservados.", "page": "Página", "from": "de" } diff --git a/locales/ru/pdf.json b/locales/ru/pdf.json index e04d9f2b..c22e8104 100644 --- a/locales/ru/pdf.json +++ b/locales/ru/pdf.json @@ -9,7 +9,7 @@ "email": "Эл. почта", "errorMessage": "Ваш файл не может быть завершен, пожалуйста, попробуйте еще раз", "reportGenAt": "Отчет сгенерирован в", - "copyright": "Авторское право © 2013-2024 iFinex Inc. Все права защищены.", + "copyright": "Авторское право © 2013-{{currYear}} iFinex Inc. Все права защищены.", "page": "Стр.", "from": "из" } diff --git a/locales/tr/pdf.json b/locales/tr/pdf.json deleted file mode 100644 index 2813bc70..00000000 --- a/locales/tr/pdf.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "template": { - "title": "Rapor", - "statementDetails": "Beyanname Detayları", - "statementDate": "Beyanname Tarihi", - "snapshotAt": "Anlık Görüntü:", - "period": "Dönem", - "username": "Kullanıcı Adı", - "email": "Email", - "errorMessage": "Dosyanız tamamlanamadı, lütfen tekrar deneyin", - "reportGenAt": "Rapor şu tarihte oluşturuldu:", - "copyright": "Telif Hakkı © 2013-2024 iFinex Inc. Tüm hakları saklıdır.", - "page": "Sayfa", - "from": "İtibaren" - } -} diff --git a/locales/zh-CN/pdf.json b/locales/zh-CN/pdf.json index 579aca9b..39d7de1a 100644 --- a/locales/zh-CN/pdf.json +++ b/locales/zh-CN/pdf.json @@ -9,7 +9,7 @@ "email": "电子邮箱", "errorMessage": "无法建立档案，请重试", "reportGenAt": "报告创建时间", - "copyright": "Copyright © 2013-2024 iFinex Inc. All rights reserved.", + "copyright": "Copyright © 2013-{{currYear}} iFinex Inc. All rights reserved.", "page": "页数", "from": "从" } diff --git a/locales/zh-TW/pdf.json b/locales/zh-TW/pdf.json index fae8c1f8..fedbf3b0 100644 --- a/locales/zh-TW/pdf.json +++ b/locales/zh-TW/pdf.json @@ -9,7 +9,7 @@ "email": "電子郵箱", "errorMessage": "無法建立檔案，請重試", "reportGenAt": "報告建立時間：", - "copyright": "Copyright © 2013-2024 iFinex Inc. All rights reserved.", + "copyright": "Copyright © 2013-{{currYear}} iFinex Inc. All rights reserved.", "page": "頁數", "from": "從" } diff --git a/package-lock.json b/package-lock.json index 29cf4a24..692b2189 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "bfx-report", - "version": "4.13.2", + "version": "4.14.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "bfx-report", - "version": "4.13.2", + "version": "4.14.0", "license": "Apache-2.0", "dependencies": { "ajv": "8.17.1", diff --git a/package.json b/package.json index ab865ff9..87de399c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bfx-report", - "version": "4.13.2", + "version": "4.14.0", "description": "Reporting tool", "main": "worker.js", "license": "Apache-2.0", diff --git a/workers/loc.api/generate-report-file/csv-writer/helpers/index.js b/workers/loc.api/generate-report-file/csv-writer/helpers/index.js index 91b2cf8f..27d32494 100644 --- a/workers/loc.api/generate-report-file/csv-writer/helpers/index.js +++ b/workers/loc.api/generate-report-file/csv-writer/helpers/index.js @@ -1,8 +1,11 @@ 'use strict' -const { pipeline } = require('stream/promises') const { stringify } = require('csv') +const { + pipelineStreams +} = require('../../../helpers') + const streamWriterToOne = async ( rStream, wStream, @@ -10,7 +13,7 @@ const streamWriterToOne = async ( opts ) => { const { end = true } = opts ?? {} - const promise = pipeline(rStream, wStream, { end }) + const promise = pipelineStreams(rStream, wStream, { end }) writeFn(rStream) rStream.end() diff --git a/workers/loc.api/generate-report-file/pdf-writer/index.js b/workers/loc.api/generate-report-file/pdf-writer/index.js index 51e6047e..11744ceb 100644 --- a/workers/loc.api/generate-report-file/pdf-writer/index.js +++ b/workers/loc.api/generate-report-file/pdf-writer/index.js @@ -164,6 +164,7 @@ class PdfWriter { isError } = opts ?? {} + const translate = this.getTranslator(language) const template = this.#getTemplate( pdfCustomTemplateName, language @@ -176,6 +177,15 @@ class PdfWriter { jobData.args.params ) const reportColumns = jobData?.columnsPdf ?? jobData?.columnsCsv + const _date = date instanceof Date + ? date + : new Date() + const currYear = _date.getUTCFullYear() + const defaultValue = `Copyright © 2013-${currYear} iFinex Inc. All rights reserved.` + const copyright = translate(defaultValue, { + prop: 'template.copyright', + currYear + }) const html = template({ shouldZoomBeAdjusted: this.shouldZoomBeAdjusted, @@ -191,9 +201,8 @@ class PdfWriter { end: Number.isFinite(jobData?.args?.params?.end) ? new Date(jobData.args.params.end) : new Date(), - date: date instanceof Date - ? date - : new Date() + date: _date, + copyright }) return html diff --git a/workers/loc.api/generate-report-file/pdf-writer/templates/base.pug b/workers/loc.api/generate-report-file/pdf-writer/templates/base.pug index 6ed79b15..235bc9f2 100644 --- a/workers/loc.api/generate-report-file/pdf-writer/templates/base.pug +++ b/workers/loc.api/generate-report-file/pdf-writer/templates/base.pug @@ -138,8 +138,7 @@ html(lang=language) | : #{date.toLocaleString(language, { timeZone: 'UTC' })} div.copyright - :translate(prop='template.copyright') - Copyright © 2013-2024 iFinex Inc. All rights reserved. + | #{copyright} span.footer-page.none-if-no-phantomjs :translate(prop='template.page') diff --git a/workers/loc.api/helpers/index.js b/workers/loc.api/helpers/index.js index 9d775494..7ffcef80 100644 --- a/workers/loc.api/helpers/index.js +++ b/workers/loc.api/helpers/index.js @@ -64,6 +64,7 @@ const getDataFromApi = require('./get-data-from-api') const splitSymbolPairs = require('./split-symbol-pairs') const FOREX_SYMBS = require('./forex.symbs') const getTranslator = require('./get-translator') +const pipelineStreams = require('./pipeline-streams') module.exports = { getREST, @@ -115,5 +116,6 @@ module.exports = { parsePositionsAuditId, splitSymbolPairs, FOREX_SYMBS, - getTranslator + getTranslator, + pipelineStreams } diff --git a/workers/loc.api/helpers/pipeline-streams.js b/workers/loc.api/helpers/pipeline-streams.js new file mode 100644 index 00000000..674245cf --- /dev/null +++ b/workers/loc.api/helpers/pipeline-streams.js @@ -0,0 +1,24 @@ +'use strict' + +const { pipeline } = require('node:stream/promises') + +module.exports = async (stringifier, writable, opts) => { + try { + const end = opts?.end ?? true + + await pipeline(stringifier, writable, { end }) + } catch (err) { + /* + * If an error occurs, eg when receiving data from the BFX API, + * a recording may occur after destruction in the stream + */ + if ( + err.code === 'ERR_STREAM_DESTROYED' || + err.code === 'ERR_STREAM_PREMATURE_CLOSE' + ) { + return + } + + throw err + } +} diff --git a/workers/loc.api/logger/index.js b/workers/loc.api/logger/index.js index c6dcdf07..d8cbd3b6 100644 --- a/workers/loc.api/logger/index.js +++ b/workers/loc.api/logger/index.js @@ -129,7 +129,7 @@ class TransportIPC extends TransportStream { } } -const _getTransports = () => { +const _getTransports = (conf) => { if (!isProdEnv) { return { baseTransports: [ @@ -151,6 +151,29 @@ const _getTransports = () => { } } + if (conf.isHosted) { + return { + baseTransports: [ + new transports.Console({ + level: 'error', + colorize: false + }), + new TransportSlack({ + level: 'error', + colorize: false + }) + ], + exceptionHandlers: [ + new transports.Console({ + colorize: false + }), + new TransportSlack({ + colorize: false + }) + ] + } + } + return { baseTransports: [ new transports.File({ @@ -239,17 +262,17 @@ const _combineFormat = (colorize = !isProdEnv) => { ) } -module.exports = ({ isLoggerDisabled }) => { +module.exports = (conf) => { const { baseTransports, exceptionHandlers - } = _getTransports() + } = _getTransports(conf) return createLogger({ format: _combineFormat(), transports: baseTransports, exceptionHandlers, - silent: isLoggerDisabled || isTestEnv, + silent: conf.isLoggerDisabled || isTestEnv, exitOnError: true }) } diff --git a/workers/loc.api/queue/aggregator.js b/workers/loc.api/queue/aggregator.js index ce3829b2..7ae34ea9 100644 --- a/workers/loc.api/queue/aggregator.js +++ b/workers/loc.api/queue/aggregator.js @@ -18,6 +18,8 @@ module.exports = ( sendMail ) => { return async (job) => { + const streamSet = new Set() + try { const { chunkCommonFolders, @@ -50,7 +52,8 @@ module.exports = ( { ...userInfo, email - } + }, + streamSet ) await sendMail( @@ -128,6 +131,11 @@ module.exports = ( } aggregatorQueue.emit('error:base', err, job) + } finally { + for (const stream of streamSet) { + stream.destroy() + streamSet.delete(stream) + } } } } diff --git a/workers/loc.api/queue/helpers/index.js b/workers/loc.api/queue/helpers/index.js index df574553..c8f5ac10 100644 --- a/workers/loc.api/queue/helpers/index.js +++ b/workers/loc.api/queue/helpers/index.js @@ -3,7 +3,6 @@ const getCompleteFileName = require('./get-complete-file-name') const { moveFileToLocalStorage, - writableToPromise, createUniqueFileName } = require('./utils') const getLocalReportFolderPaths = require( @@ -14,7 +13,6 @@ const getReportContentType = require('./get-report-content-type') module.exports = { moveFileToLocalStorage, - writableToPromise, createUniqueFileName, getCompleteFileName, getLocalReportFolderPaths, diff --git a/workers/loc.api/queue/helpers/utils.js b/workers/loc.api/queue/helpers/utils.js index 9c743e02..568d614f 100644 --- a/workers/loc.api/queue/helpers/utils.js +++ b/workers/loc.api/queue/helpers/utils.js @@ -160,19 +160,7 @@ const createUniqueFileName = async (rootPath, params, count = 0) => { return path.join(tempReportFolderPath, uniqueFileName) } -const writableToPromise = stream => { - return new Promise((resolve, reject) => { - stream.once('finish', () => { - resolve('finish') - }) - stream.once('error', err => { - reject(err) - }) - }) -} - module.exports = { moveFileToLocalStorage, - writableToPromise, createUniqueFileName } diff --git a/workers/loc.api/queue/processor.js b/workers/loc.api/queue/processor.js index 1ac7bda2..28f74268 100644 --- a/workers/loc.api/queue/processor.js +++ b/workers/loc.api/queue/processor.js @@ -4,16 +4,15 @@ const { omit, cloneDeep } = require('lib-js-util-base') -const { promisify } = require('util') -const { pipeline } = require('stream') -const fs = require('fs') +const { createWriteStream } = require('node:fs') +const { unlink } = require('node:fs/promises') const { stringify } = require('csv') -const unlink = promisify(fs.unlink) - const { - createUniqueFileName, - writableToPromise + pipelineStreams +} = require('../helpers') +const { + createUniqueFileName } = require('./helpers') const { isAuthError } = require('../helpers') @@ -21,15 +20,16 @@ const { isAuthError } = require('../helpers') const processReportFile = async (deps, args) => { const { data, - filePath + filePath, + streamSet } = args const write = data?.isUnauth ? 'Your file could not be completed, please try again' : data - const writable = fs.createWriteStream(filePath) - const writablePromise = writableToPromise(writable) + const writable = createWriteStream(filePath) + streamSet.add(writable) if (data?.args?.params?.isPDFRequired) { const pdfStream = await deps.pdfWriter @@ -39,8 +39,9 @@ const processReportFile = async (deps, args) => { language: data?.args?.params.language, isError: data?.isUnauth }) + streamSet.add(pdfStream) - pipeline(pdfStream, writable, () => {}) + const pipelinePromise = pipelineStreams(pdfStream, writable) await deps.writeDataToStream( pdfStream, @@ -49,7 +50,7 @@ const processReportFile = async (deps, args) => { pdfStream.end() - return writablePromise + return await pipelinePromise } if (typeof data?.csvCustomWriter === 'function') { await data.csvCustomWriter( @@ -57,15 +58,16 @@ const processReportFile = async (deps, args) => { write ) - return writablePromise + return } const stringifier = stringify({ header: true, columns: data?.columnsCsv }) + streamSet.add(stringifier) - pipeline(stringifier, writable, () => {}) + const pipelinePromise = pipelineStreams(stringifier, writable) await deps.writeDataToStream( stringifier, @@ -74,7 +76,7 @@ const processReportFile = async (deps, args) => { stringifier.end() - return writablePromise + return await pipelinePromise } module.exports = ( @@ -108,6 +110,7 @@ module.exports = ( }) return async (job) => { + const streamSet = new Set() const filePaths = [] const chunkCommonFolders = [] const subParamsArr = [] @@ -161,7 +164,8 @@ module.exports = ( }, { data, - filePath + filePath, + streamSet } ) } @@ -195,6 +199,11 @@ module.exports = ( } processorQueue.emit('error:base', err, job) + } finally { + for (const stream of streamSet) { + stream.destroy() + streamSet.delete(stream) + } } } } diff --git a/workers/loc.api/queue/upload-to-s3/index.js b/workers/loc.api/queue/upload-to-s3/index.js index 7fd7e3ab..0a109443 100644 --- a/workers/loc.api/queue/upload-to-s3/index.js +++ b/workers/loc.api/queue/upload-to-s3/index.js @@ -69,7 +69,8 @@ module.exports = ( queueName, subParamsArr, isSignatureRequired, - userInfo + userInfo, + streamSet ) => { const isMultiExport = ( queueName === 'getMultiple' || @@ -92,8 +93,11 @@ module.exports = ( const isSignReq = isSignatureRequired && isUppedPGPService const streams = filePaths.map((filePath, i) => { + const stream = fs.createReadStream(filePath) + streamSet.add(stream) + return { - stream: fs.createReadStream(filePath), + stream, data: { name: getCompleteFileName( subParamsArr[i].name, diff --git a/workers/loc.api/queue/write-data-to-stream/helpers.js b/workers/loc.api/queue/write-data-to-stream/helpers.js index 4c7adefe..8853ba21 100644 --- a/workers/loc.api/queue/write-data-to-stream/helpers.js +++ b/workers/loc.api/queue/write-data-to-stream/helpers.js @@ -184,6 +184,10 @@ const write = ( const _item = dataNormalizer(item, method, params) const res = _dataFormatter(_item, formatSettings, params) + if (!stream.writable) { + return + } + stream.write(res) } } diff --git a/workers/loc.api/queue/write-data-to-stream/index.js b/workers/loc.api/queue/write-data-to-stream/index.js index 4c8061f3..1ffb8db7 100644 --- a/workers/loc.api/queue/write-data-to-stream/index.js +++ b/workers/loc.api/queue/write-data-to-stream/index.js @@ -58,12 +58,12 @@ module.exports = ( return accum }, {}) + processorQueue.emit('progress', 0) + let count = 0 let serialRequestsCount = 0 while (true) { - processorQueue.emit('progress', 0) - const _res = await getDataFromApi({ getData, args: currIterationArgs, @@ -113,7 +113,7 @@ module.exports = ( res, stream, formatSettings, - { ..._args.params, symbols }, + { ...jobData?.args?.params, symbols }, method ) processorQueue.emit('progress', 100) @@ -154,7 +154,7 @@ module.exports = ( res, stream, formatSettings, - { ..._args.params, symbols }, + { ...jobData?.args?.params, symbols }, method )