diff --git a/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v14.js b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v14.js new file mode 100644 index 000000000..a358a2169 --- /dev/null +++ b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v14.js @@ -0,0 +1,73 @@ +'use strict' + +const AbstractMigration = require('./abstract.migration') +const { getSqlArrToModifyColumns } = require('./helpers') + +class MigrationV14 extends AbstractMigration { + /** + * @override + */ + up () { + const sqlArr = [ + 'ALTER TABLE ledgers ADD COLUMN _category INT', + + /* + Delete all data from ledgers to allow + resync from scratch for adding categories + */ + 'DELETE FROM ledgers' + ] + + this.addSql(sqlArr) + } + + /** + * @override + */ + beforeDown () { return this.dao.disableForeignKeys() } + + /** + * @override + */ + down () { + const sqlArr = [ + ...getSqlArrToModifyColumns( + 'ledgers', + { + _id: 'INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT', + id: 'BIGINT', + currency: 'VARCHAR(255)', + mts: 'BIGINT', + amount: 'DECIMAL(22,12)', + amountUsd: 'DECIMAL(22,12)', + balance: 'DECIMAL(22,12)', + _nativeBalance: 'DECIMAL(22,12)', + balanceUsd: 'DECIMAL(22,12)', + _nativeBalanceUsd: 'DECIMAL(22,12)', + description: 'TEXT', + wallet: 'VARCHAR(255)', + _isMarginFundingPayment: 'INT', + _isAffiliateRebate: 'INT', + _isStakingPayments: 'INT', + _isBalanceRecalced: 'INT', + subUserId: 'INT', + user_id: 'INT NOT NULL', + __constraints__: `CONSTRAINT #{tableName}_fk_user_id + FOREIGN KEY (user_id) + REFERENCES users(_id) + ON UPDATE CASCADE + ON DELETE CASCADE` + } + ) + ] + + this.addSql(sqlArr) + } + + /** + * @override + */ + afterDown () { return this.dao.enableForeignKeys() } +} + +module.exports = MigrationV14 diff --git a/workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js b/workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js index 228f5b70c..6b2e77670 100644 --- a/workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js +++ b/workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js @@ -17,13 +17,20 @@ const getFieldsFilters = ( const field = _params[fieldName] if ( - typeof field === 'boolean' && Object.keys(model) .some(key => key === modelFieldName) ) { - return { - ...accum, - [modelFieldName]: Number(field) + if (typeof field === 'boolean') { + return { + ...accum, + [modelFieldName]: Number(field) + } + } + if (typeof field === 'number') { + return { + ...accum, + [modelFieldName]: field + } } } @@ -59,7 +66,8 @@ module.exports = ( [ 'isMarginFundingPayment', 'isAffiliateRebate', - 'isStakingPayments' + 'isStakingPayments', + 'category' ], params, model diff --git a/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js b/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js index 040215422..0dafff719 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js @@ -10,7 +10,8 @@ const TYPES = require('../../../di/types') const SYNC_API_METHODS = require('../../schema/sync.api.methods') const { addPropsToResIfExist, - getFlagsFromLedgerDescription + getFlagsFromLedgerDescription, + getCategoryFromDescription } = require('./helpers') class ApiMiddlewareHandlerAfter { @@ -124,6 +125,10 @@ class ApiMiddlewareHandlerAfter { { fieldName: '_isStakingPayments', pattern: 'Staking Payments' + }, + { + fieldName: '_category', + handler: getCategoryFromDescription } ] ), diff --git a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js new file mode 100644 index 000000000..3d6196b72 --- /dev/null +++ b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js @@ -0,0 +1,225 @@ +'use strict' + +const _startsWith = (d, pattern) => { + return d.toLowerCase().startsWith(pattern) +} + +const _includes = (d, pattern) => { + return d.toLowerCase().includes(pattern) +} + +const _test = (d, pattern) => { + return pattern.test(d.toLowerCase()) +} + +const _schema = [ + { + tester: (d) => ( + _startsWith(d, 'exchange') + ), + category: 5 + }, + { + tester: (d) => ( + _startsWith(d, 'position modified') || + _startsWith(d, 'position closed') || + _test(d, /position #[0-9]* close/) || + _test(d, /position #[0-9]* liquidate/) + ), + category: 22 + }, + { + tester: (d) => ( + _startsWith(d, 'position claim') || + _test(d, /position #[0-9]* claim/) + ), + category: 23 + }, + { + tester: (d) => ( + _test(d, /position #[0-9]* transfer/) + ), + category: 25 + }, + { + tester: (d) => ( + _test(d, /position #[0-9]* swap/) + ), + category: 26 + }, + { + tester: (d) => ( + _startsWith(d, 'position funding cost') || + _startsWith(d, 'interest charge') || + _test(d, /position #[0-9]* funding cost/) + ), + category: 27 + }, + { + tester: (d) => ( + _startsWith(d, 'margin funding payment') || + _startsWith(d, 'swap payment') || + _startsWith(d, 'interest payment') + ), + category: 28 + }, + { + tester: (d) => ( + _includes(d, 'settlement') + ), + category: 31 + }, + { + tester: (d) => ( + _startsWith(d, 'transfer') + ), + category: 51 + }, + { + tester: (d) => ( + _startsWith(d, 'deposit (') || + _startsWith(d, 'deposit on wallet') + ), + category: 101 + }, + { + tester: (d) => ( + _includes(d, ' withdrawal #') + ), + category: 104 + }, + { + tester: (d) => ( + _startsWith(d, 'canceled withdrawal') + ), + category: 105 + }, + { + tester: (d) => ( + _startsWith(d, 'trading fee') + ), + category: 201 + }, + { + tester: (d) => ( + _includes(d, 'trading rebate') + ), + category: 202 + }, + { + tester: (d) => ( + _startsWith(d, 'hidden order fee') + ), + category: 204 + }, + { + tester: (d) => ( + _startsWith(d, 'otc trade fee') + ), + category: 207 + }, + { + tester: (d) => ( + _startsWith(d, 'swap fee') + ), + category: 222 + }, + { + tester: (d) => ( + _startsWith(d, 'claiming fee') + ), + category: 224 + }, + { + tester: (d) => ( + _includes(d, 'margin funding charge') + ), + category: 226 + }, + { + tester: (d) => ( + _startsWith(d, 'earned fee') || + _startsWith(d, 'affiliate rebate') + ), + category: 241 + }, + { + tester: (d) => ( + _startsWith(d, 'ETHFX loyalty fee') + ), + category: 243 + }, + { + tester: (d) => ( + _includes(d, 'deposit fee') + ), + category: 251 + }, + { + tester: (d) => ( + _includes(d, 'withdrawal fee') + ), + category: 254 + }, + { + tester: (d) => ( + _includes(d, 'withdrawal express fee') + ), + category: 255 + }, + { + tester: (d) => ( + _startsWith(d, 'miner fee') + ), + category: 258 + }, + { + tester: (d) => ( + _startsWith(d, 'staking reward') + ), + category: 262 + }, + { + tester: (d) => ( + _startsWith(d, 'adjustment') + ), + category: 401 + }, + { + tester: (d) => ( + _startsWith(d, 'expense') + ), + category: 501 + }, + { + tester: (d) => ( + _startsWith(d, 'currency conversion') || + _startsWith(d, 'computation fee') + ), + category: 905 + }, + { + tester: (d) => ( + _startsWith(d, 'monthly profit payment') + ), + category: 907 + }, + { + tester: (d) => ( + _startsWith(d, 'losses') + ), + category: 911 + } +] + +module.exports = (description) => { + const d = description.toLowerCase() + + for (const { tester, category } of _schema) { + if (tester(d)) { + return category + } + } + + return null +} diff --git a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js index a3699b6d7..f2f5ad520 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js @@ -16,8 +16,17 @@ module.exports = ( const { fieldName, pattern, + handler, isCaseSensitivity } = { ...schema } + + if (typeof handler === 'function') { + return { + ...accum, + [fieldName]: handler(ledger.description) + } + } + const regExp = new RegExp( pattern, isCaseSensitivity ? '' : 'i' diff --git a/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js b/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js index 9471ff211..9204b06fb 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js @@ -6,8 +6,12 @@ const addPropsToResIfExist = require( const getFlagsFromLedgerDescription = require( './get-flags-from-ledger-description' ) +const getCategoryFromDescription = require( + './get-category-from-description' +) module.exports = { addPropsToResIfExist, - getFlagsFromLedgerDescription + getFlagsFromLedgerDescription, + getCategoryFromDescription } diff --git a/workers/loc.api/sync/schema/index.js b/workers/loc.api/sync/schema/index.js index 4ce422bae..5c33977bf 100644 --- a/workers/loc.api/sync/schema/index.js +++ b/workers/loc.api/sync/schema/index.js @@ -7,7 +7,7 @@ * in the `workers/loc.api/sync/dao/db-migrations/sqlite-migrations` folder, * e.g. `migration.v1.js`, where `v1` is `SUPPORTED_DB_VERSION` */ -const SUPPORTED_DB_VERSION = 13 +const SUPPORTED_DB_VERSION = 14 const TABLES_NAMES = require('./tables-names') const ALLOWED_COLLS = require('./allowed.colls') @@ -97,6 +97,7 @@ const _models = new Map([ _nativeBalanceUsd: 'DECIMAL(22,12)', description: 'TEXT', wallet: 'VARCHAR(255)', + _category: 'INT', _isMarginFundingPayment: 'INT', _isAffiliateRebate: 'INT', _isStakingPayments: 'INT',