diff --git a/package.json b/package.json index fd2512834..2f0554943 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "lodash": "^4.17.11", "moment": "^2.24.0", "scrypt-js": "^3.0.0", - "sqlite3": "4.1.1", + "sqlite3": "5.0.0", "uuid": "^8.0.0", "yargs": "^13.2.4" }, diff --git a/worker.js b/worker.js index aec5c49b6..1286b658e 100644 --- a/worker.js +++ b/worker.js @@ -1,5 +1,3 @@ 'use strict' -process.versions.electron = process.env.ELECTRON_VERSION - module.exports = require('bfx-svc-boot-js') diff --git a/workers/loc.api/sync/currency.converter/index.js b/workers/loc.api/sync/currency.converter/index.js index 246895af7..407208692 100644 --- a/workers/loc.api/sync/currency.converter/index.js +++ b/workers/loc.api/sync/currency.converter/index.js @@ -11,14 +11,14 @@ const { FindMethodError } = require('bfx-report/workers/loc.api/errors') const { - getDataFromApi + getDataFromApi, + splitSymbolPairs } = require('bfx-report/workers/loc.api/helpers') const { CurrencyConversionDataFindingError } = require('../../errors') const { - splitSymbolPairs, isForexSymb } = require('../helpers') const { tryParseJSON } = require('../../helpers') diff --git a/workers/loc.api/sync/dao/dao.sqlite.js b/workers/loc.api/sync/dao/dao.sqlite.js index ef7a46013..0bdae0ae1 100644 --- a/workers/loc.api/sync/dao/dao.sqlite.js +++ b/workers/loc.api/sync/dao/dao.sqlite.js @@ -7,7 +7,8 @@ const { } = require('inversify') const { getLimitNotMoreThan, - checkFilterParams + checkFilterParams, + normalizeFilterParams } = require('bfx-report/workers/loc.api/helpers') const { AuthError @@ -44,6 +45,12 @@ const { } = require('../../errors') class SqliteDAO extends DAO { + constructor (...args) { + super(...args) + + this._transactionPromise = Promise.resolve() + } + _run (sql, params = []) { return new Promise((resolve, reject) => { this.db.run(sql, params, function (err) { @@ -94,14 +101,16 @@ class SqliteDAO extends DAO { return this._run('ROLLBACK') } - _beginTrans ( + async _beginTrans ( asyncExecQuery, { beforeTransFn, afterTransFn } = {} ) { - return new Promise((resolve, reject) => { + await this._transactionPromise + + const promise = new Promise((resolve, reject) => { this.db.serialize(async () => { let isTransBegun = false @@ -139,6 +148,10 @@ class SqliteDAO extends DAO { } }) }) + + this._transactionPromise = promise + + return promise } async _createTablesIfNotExists () { @@ -636,7 +649,7 @@ class SqliteDAO extends DAO { */ async findInCollBy ( method, - args, + reqArgs, { isPrepareResponse = false, isPublic = false, @@ -648,6 +661,7 @@ class SqliteDAO extends DAO { ) { const filterModelName = filterModelNameMap.get(method) + const args = normalizeFilterParams(method, reqArgs) checkFilterParams(filterModelName, args) const { auth: user } = { ...args } diff --git a/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v14.js b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v14.js new file mode 100644 index 000000000..a358a2169 --- /dev/null +++ b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v14.js @@ -0,0 +1,73 @@ +'use strict' + +const AbstractMigration = require('./abstract.migration') +const { getSqlArrToModifyColumns } = require('./helpers') + +class MigrationV14 extends AbstractMigration { + /** + * @override + */ + up () { + const sqlArr = [ + 'ALTER TABLE ledgers ADD COLUMN _category INT', + + /* + Delete all data from ledgers to allow + resync from scratch for adding categories + */ + 'DELETE FROM ledgers' + ] + + this.addSql(sqlArr) + } + + /** + * @override + */ + beforeDown () { return this.dao.disableForeignKeys() } + + /** + * @override + */ + down () { + const sqlArr = [ + ...getSqlArrToModifyColumns( + 'ledgers', + { + _id: 'INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT', + id: 'BIGINT', + currency: 'VARCHAR(255)', + mts: 'BIGINT', + amount: 'DECIMAL(22,12)', + amountUsd: 'DECIMAL(22,12)', + balance: 'DECIMAL(22,12)', + _nativeBalance: 'DECIMAL(22,12)', + balanceUsd: 'DECIMAL(22,12)', + _nativeBalanceUsd: 'DECIMAL(22,12)', + description: 'TEXT', + wallet: 'VARCHAR(255)', + _isMarginFundingPayment: 'INT', + _isAffiliateRebate: 'INT', + _isStakingPayments: 'INT', + _isBalanceRecalced: 'INT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + __constraints__: `CONSTRAINT #{tableName}_fk_user_id + FOREIGN KEY (user_id) + REFERENCES users(_id) + ON UPDATE CASCADE + ON DELETE CASCADE` + } + ) + ] + + this.addSql(sqlArr) + } + + /** + * @override + */ + afterDown () { return this.dao.enableForeignKeys() } +} + +module.exports = MigrationV14 diff --git a/workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js b/workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js index 228f5b70c..6b2e77670 100644 --- a/workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js +++ b/workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js @@ -17,13 +17,20 @@ const getFieldsFilters = ( const field = _params[fieldName] if ( - typeof field === 'boolean' && Object.keys(model) .some(key => key === modelFieldName) ) { - return { - ...accum, - [modelFieldName]: Number(field) + if (typeof field === 'boolean') { + return { + ...accum, + [modelFieldName]: Number(field) + } + } + if (typeof field === 'number') { + return { + ...accum, + [modelFieldName]: field + } } } @@ -59,7 +66,8 @@ module.exports = ( [ 'isMarginFundingPayment', 'isAffiliateRebate', - 'isStakingPayments' + 'isStakingPayments', + 'category' ], params, model diff --git a/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js b/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js index 040215422..0dafff719 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js @@ -10,7 +10,8 @@ const TYPES = require('../../../di/types') const SYNC_API_METHODS = require('../../schema/sync.api.methods') const { addPropsToResIfExist, - getFlagsFromLedgerDescription + getFlagsFromLedgerDescription, + getCategoryFromDescription } = require('./helpers') class ApiMiddlewareHandlerAfter { @@ -124,6 +125,10 @@ class ApiMiddlewareHandlerAfter { { fieldName: '_isStakingPayments', pattern: 'Staking Payments' + }, + { + fieldName: '_category', + handler: getCategoryFromDescription } ] ), diff --git a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js new file mode 100644 index 000000000..3d6196b72 --- /dev/null +++ b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js @@ -0,0 +1,225 @@ +'use strict' + +const _startsWith = (d, pattern) => { + return d.toLowerCase().startsWith(pattern) +} + +const _includes = (d, pattern) => { + return d.toLowerCase().includes(pattern) +} + +const _test = (d, pattern) => { + return pattern.test(d.toLowerCase()) +} + +const _schema = [ + { + tester: (d) => ( + _startsWith(d, 'exchange') + ), + category: 5 + }, + { + tester: (d) => ( + _startsWith(d, 'position modified') || + _startsWith(d, 'position closed') || + _test(d, /position #[0-9]* close/) || + _test(d, /position #[0-9]* liquidate/) + ), + category: 22 + }, + { + tester: (d) => ( + _startsWith(d, 'position claim') || + _test(d, /position #[0-9]* claim/) + ), + category: 23 + }, + { + tester: (d) => ( + _test(d, /position #[0-9]* transfer/) + ), + category: 25 + }, + { + tester: (d) => ( + _test(d, /position #[0-9]* swap/) + ), + category: 26 + }, + { + tester: (d) => ( + _startsWith(d, 'position funding cost') || + _startsWith(d, 'interest charge') || + _test(d, /position #[0-9]* funding cost/) + ), + category: 27 + }, + { + tester: (d) => ( + _startsWith(d, 'margin funding payment') || + _startsWith(d, 'swap payment') || + _startsWith(d, 'interest payment') + ), + category: 28 + }, + { + tester: (d) => ( + _includes(d, 'settlement') + ), + category: 31 + }, + { + tester: (d) => ( + _startsWith(d, 'transfer') + ), + category: 51 + }, + { + tester: (d) => ( + _startsWith(d, 'deposit (') || + _startsWith(d, 'deposit on wallet') + ), + category: 101 + }, + { + tester: (d) => ( + _includes(d, ' withdrawal #') + ), + category: 104 + }, + { + tester: (d) => ( + _startsWith(d, 'canceled withdrawal') + ), + category: 105 + }, + { + tester: (d) => ( + _startsWith(d, 'trading fee') + ), + category: 201 + }, + { + tester: (d) => ( + _includes(d, 'trading rebate') + ), + category: 202 + }, + { + tester: (d) => ( + _startsWith(d, 'hidden order fee') + ), + category: 204 + }, + { + tester: (d) => ( + _startsWith(d, 'otc trade fee') + ), + category: 207 + }, + { + tester: (d) => ( + _startsWith(d, 'swap fee') + ), + category: 222 + }, + { + tester: (d) => ( + _startsWith(d, 'claiming fee') + ), + category: 224 + }, + { + tester: (d) => ( + _includes(d, 'margin funding charge') + ), + category: 226 + }, + { + tester: (d) => ( + _startsWith(d, 'earned fee') || + _startsWith(d, 'affiliate rebate') + ), + category: 241 + }, + { + tester: (d) => ( + _startsWith(d, 'ETHFX loyalty fee') + ), + category: 243 + }, + { + tester: (d) => ( + _includes(d, 'deposit fee') + ), + category: 251 + }, + { + tester: (d) => ( + _includes(d, 'withdrawal fee') + ), + category: 254 + }, + { + tester: (d) => ( + _includes(d, 'withdrawal express fee') + ), + category: 255 + }, + { + tester: (d) => ( + _startsWith(d, 'miner fee') + ), + category: 258 + }, + { + tester: (d) => ( + _startsWith(d, 'staking reward') + ), + category: 262 + }, + { + tester: (d) => ( + _startsWith(d, 'adjustment') + ), + category: 401 + }, + { + tester: (d) => ( + _startsWith(d, 'expense') + ), + category: 501 + }, + { + tester: (d) => ( + _startsWith(d, 'currency conversion') || + _startsWith(d, 'computation fee') + ), + category: 905 + }, + { + tester: (d) => ( + _startsWith(d, 'monthly profit payment') + ), + category: 907 + }, + { + tester: (d) => ( + _startsWith(d, 'losses') + ), + category: 911 + } +] + +module.exports = (description) => { + const d = description.toLowerCase() + + for (const { tester, category } of _schema) { + if (tester(d)) { + return category + } + } + + return null +} diff --git a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js index a3699b6d7..f2f5ad520 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js @@ -16,8 +16,17 @@ module.exports = ( const { fieldName, pattern, + handler, isCaseSensitivity } = { ...schema } + + if (typeof handler === 'function') { + return { + ...accum, + [fieldName]: handler(ledger.description) + } + } + const regExp = new RegExp( pattern, isCaseSensitivity ? '' : 'i' diff --git a/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js b/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js index 9471ff211..9204b06fb 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js @@ -6,8 +6,12 @@ const addPropsToResIfExist = require( const getFlagsFromLedgerDescription = require( './get-flags-from-ledger-description' ) +const getCategoryFromDescription = require( + './get-category-from-description' +) module.exports = { addPropsToResIfExist, - getFlagsFromLedgerDescription + getFlagsFromLedgerDescription, + getCategoryFromDescription } diff --git a/workers/loc.api/sync/helpers/__test__/split-symbol-pairs.spec.js b/workers/loc.api/sync/helpers/__test__/split-symbol-pairs.spec.js deleted file mode 100644 index 1f083cf02..000000000 --- a/workers/loc.api/sync/helpers/__test__/split-symbol-pairs.spec.js +++ /dev/null @@ -1,106 +0,0 @@ -'use strict' - -const { assert } = require('chai') - -const splitSymbolPairs = require('../split-symbol-pairs') - -describe('splitSymbolPairs helper', () => { - it('BTC:CNHT pair', function () { - this.timeout(1000) - - const res = splitSymbolPairs('BTC:CNHT') - - assert.isArray(res) - assert.lengthOf(res, 2) - assert.strictEqual(res[0], 'BTC') - assert.strictEqual(res[1], 'CNHT') - }) - - it('DUSK:USD pair', function () { - this.timeout(1000) - - const res = splitSymbolPairs('DUSK:USD') - - assert.isArray(res) - assert.lengthOf(res, 2) - assert.strictEqual(res[0], 'DUSK') - assert.strictEqual(res[1], 'USD') - }) - - it('EURUSD pair', function () { - this.timeout(1000) - - const res = splitSymbolPairs('EURUSD') - - assert.isArray(res) - assert.lengthOf(res, 2) - assert.strictEqual(res[0], 'EUR') - assert.strictEqual(res[1], 'USD') - }) - - it('tXAUT:USD pair', function () { - this.timeout(1000) - - const res = splitSymbolPairs('tXAUT:USD') - - assert.isArray(res) - assert.lengthOf(res, 2) - assert.strictEqual(res[0], 'XAUT') - assert.strictEqual(res[1], 'USD') - }) - - it('fBTCUSD pair', function () { - this.timeout(1000) - - const res = splitSymbolPairs('fBTCUSD') - - assert.isArray(res) - assert.lengthOf(res, 2) - assert.strictEqual(res[0], 'BTC') - assert.strictEqual(res[1], 'USD') - }) - - it('tBTCF0:USD pair', function () { - this.timeout(1000) - - const res = splitSymbolPairs('tBTCF0:USD') - - assert.isArray(res) - assert.lengthOf(res, 2) - assert.strictEqual(res[0], 'BTCF0') - assert.strictEqual(res[1], 'USD') - }) - - it('tBTCF0USD pair, without separator', function () { - this.timeout(1000) - - const res = splitSymbolPairs('tBTCF0USD') - - assert.isArray(res) - assert.lengthOf(res, 2) - assert.strictEqual(res[0], 'BTCF0') - assert.strictEqual(res[1], 'USD') - }) - - it('tXAUTUSD pair, without separator', function () { - this.timeout(1000) - - const res = splitSymbolPairs('tXAUTUSD') - - assert.isArray(res) - assert.lengthOf(res, 2) - assert.strictEqual(res[0], 'XAUT') - assert.strictEqual(res[1], 'USD') - }) - - it('tEUR:USD pair, with separator', function () { - this.timeout(1000) - - const res = splitSymbolPairs('tEUR:USD') - - assert.isArray(res) - assert.lengthOf(res, 2) - assert.strictEqual(res[0], 'EUR') - assert.strictEqual(res[1], 'USD') - }) -}) diff --git a/workers/loc.api/sync/helpers/index.js b/workers/loc.api/sync/helpers/index.js index abbc387b6..9e9f001b5 100644 --- a/workers/loc.api/sync/helpers/index.js +++ b/workers/loc.api/sync/helpers/index.js @@ -8,7 +8,6 @@ const FOREX_SYMBS = require('./forex-symbs') const isForexSymb = require('./is-forex-symb') const redirectRequestsToApi = require('./redirect-requests-to-api') const checkCollPermission = require('./check-coll-permission') -const splitSymbolPairs = require('./split-symbol-pairs') const setMtsToStartingAndEndingFrames = require( './set-mts-to-starting-and-ending-frames' ) @@ -26,7 +25,6 @@ module.exports = { isForexSymb, redirectRequestsToApi, checkCollPermission, - splitSymbolPairs, delay, setMtsToStartingAndEndingFrames, getBackIterable diff --git a/workers/loc.api/sync/helpers/split-symbol-pairs.js b/workers/loc.api/sync/helpers/split-symbol-pairs.js deleted file mode 100644 index 786534779..000000000 --- a/workers/loc.api/sync/helpers/split-symbol-pairs.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict' - -module.exports = (symbol) => { - const str = ( - symbol[0] === 't' || - symbol[0] === 'f' - ) - ? symbol.slice(1) - : symbol - - if ( - str.length > 5 && - /.+[:].+/.test(str) - ) { - return str.split(':') - } - if (str.length < 6) { - return [str] - } - - return [str.slice(0, -3), str.slice(-3)] -} diff --git a/workers/loc.api/sync/positions.snapshot/index.js b/workers/loc.api/sync/positions.snapshot/index.js index 71feb0156..5b93212cb 100644 --- a/workers/loc.api/sync/positions.snapshot/index.js +++ b/workers/loc.api/sync/positions.snapshot/index.js @@ -7,8 +7,10 @@ const { injectable, inject } = require('inversify') +const { + splitSymbolPairs +} = require('bfx-report/workers/loc.api/helpers') -const { splitSymbolPairs } = require('../helpers') const TYPES = require('../../di/types') class PositionsSnapshot { diff --git a/workers/loc.api/sync/schema/index.js b/workers/loc.api/sync/schema/index.js index 4ce422bae..5c33977bf 100644 --- a/workers/loc.api/sync/schema/index.js +++ b/workers/loc.api/sync/schema/index.js @@ -7,7 +7,7 @@ * in the `workers/loc.api/sync/dao/db-migrations/sqlite-migrations` folder, * e.g. `migration.v1.js`, where `v1` is `SUPPORTED_DB_VERSION` */ -const SUPPORTED_DB_VERSION = 13 +const SUPPORTED_DB_VERSION = 14 const TABLES_NAMES = require('./tables-names') const ALLOWED_COLLS = require('./allowed.colls') @@ -97,6 +97,7 @@ const _models = new Map([ _nativeBalanceUsd: 'DECIMAL(22,12)', description: 'TEXT', wallet: 'VARCHAR(255)', + _category: 'INT', _isMarginFundingPayment: 'INT', _isAffiliateRebate: 'INT', _isStakingPayments: 'INT', diff --git a/workers/loc.api/sync/trades/index.js b/workers/loc.api/sync/trades/index.js index 6abc95ba9..fdecbc816 100644 --- a/workers/loc.api/sync/trades/index.js +++ b/workers/loc.api/sync/trades/index.js @@ -5,12 +5,14 @@ const { injectable, inject } = require('inversify') +const { + splitSymbolPairs +} = require('bfx-report/workers/loc.api/helpers') const TYPES = require('../../di/types') const { calcGroupedData, groupByTimeframe, - splitSymbolPairs, getMtsGroupedByTimeframe } = require('../helpers')