Merge pull request #120 from ZIMkaRU/feature/add-filter-by-category-f…

…or-ledgers

Add filter by category for ledgers to the sync mode

workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v14.js

@@ -0,0 +1,73 @@
+'use strict'
+
+const AbstractMigration = require('./abstract.migration')
+const { getSqlArrToModifyColumns } = require('./helpers')
+
+class MigrationV14 extends AbstractMigration {
+  /**
+   * @override
+   */
+  up () {
+    const sqlArr = [
+      'ALTER TABLE ledgers ADD COLUMN _category INT',
+
+      /*
+        Delete all data from ledgers to allow
+        resync from scratch for adding categories
+      */
+      'DELETE FROM ledgers'
+    ]
+
+    this.addSql(sqlArr)
+  }
+
+  /**
+   * @override
+   */
+  beforeDown () { return this.dao.disableForeignKeys() }
+
+  /**
+   * @override
+   */
+  down () {
+    const sqlArr = [
+      ...getSqlArrToModifyColumns(
+        'ledgers',
+        {
+          _id: 'INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT',
+          id: 'BIGINT',
+          currency: 'VARCHAR(255)',
+          mts: 'BIGINT',
+          amount: 'DECIMAL(22,12)',
+          amountUsd: 'DECIMAL(22,12)',
+          balance: 'DECIMAL(22,12)',
+          _nativeBalance: 'DECIMAL(22,12)',
+          balanceUsd: 'DECIMAL(22,12)',
+          _nativeBalanceUsd: 'DECIMAL(22,12)',
+          description: 'TEXT',
+          wallet: 'VARCHAR(255)',
+          _isMarginFundingPayment: 'INT',
+          _isAffiliateRebate: 'INT',
+          _isStakingPayments: 'INT',
+          _isBalanceRecalced: 'INT',
+          subUserId: 'INT',
+          user_id: 'INT NOT NULL',
+          __constraints__: `CONSTRAINT #{tableName}_fk_user_id
+            FOREIGN KEY (user_id)
+            REFERENCES users(_id)
+            ON UPDATE CASCADE
+            ON DELETE CASCADE`
+        }
+      )
+    ]
+
+    this.addSql(sqlArr)
+  }
+
+  /**
+   * @override
+   */
+  afterDown () { return this.dao.enableForeignKeys() }
+}
+
+module.exports = MigrationV14

workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js

@@ -17,13 +17,20 @@ const getFieldsFilters = (
     const field = _params[fieldName]
 
     if (
-      typeof field === 'boolean' &&
       Object.keys(model)
         .some(key => key === modelFieldName)
     ) {
-      return {
-        ...accum,
-        [modelFieldName]: Number(field)
+      if (typeof field === 'boolean') {
+        return {
+          ...accum,
+          [modelFieldName]: Number(field)
+        }
+      }
+      if (typeof field === 'number') {
+        return {
+          ...accum,
+          [modelFieldName]: field
+        }
       }
     }
 
@@ -59,7 +66,8 @@ module.exports = (
     [
       'isMarginFundingPayment',
       'isAffiliateRebate',
-      'isStakingPayments'
+      'isStakingPayments',
+      'category'
     ],
     params,
     model

workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js

@@ -10,7 +10,8 @@ const TYPES = require('../../../di/types')
 const SYNC_API_METHODS = require('../../schema/sync.api.methods')
 const {
   addPropsToResIfExist,
-  getFlagsFromLedgerDescription
+  getFlagsFromLedgerDescription,
+  getCategoryFromDescription
 } = require('./helpers')
 
 class ApiMiddlewareHandlerAfter {
@@ -124,6 +125,10 @@ class ApiMiddlewareHandlerAfter {
             {
               fieldName: '_isStakingPayments',
               pattern: 'Staking Payments'
+            },
+            {
+              fieldName: '_category',
+              handler: getCategoryFromDescription
             }
           ]
         ),

workers/loc.api/sync/data.inserter/api.middleware/helpers/get-category-from-description.js

@@ -0,0 +1,225 @@
+'use strict'
+
+const _startsWith = (d, pattern) => {
+  return d.toLowerCase().startsWith(pattern)
+}
+
+const _includes = (d, pattern) => {
+  return d.toLowerCase().includes(pattern)
+}
+
+const _test = (d, pattern) => {
+  return pattern.test(d.toLowerCase())
+}
+
+const _schema = [
+  {
+    tester: (d) => (
+      _startsWith(d, 'exchange')
+    ),
+    category: 5
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'position modified') ||
+      _startsWith(d, 'position closed') ||
+      _test(d, /position #[0-9]* close/) ||
+      _test(d, /position #[0-9]* liquidate/)
+    ),
+    category: 22
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'position claim') ||
+      _test(d, /position #[0-9]* claim/)
+    ),
+    category: 23
+  },
+  {
+    tester: (d) => (
+      _test(d, /position #[0-9]* transfer/)
+    ),
+    category: 25
+  },
+  {
+    tester: (d) => (
+      _test(d, /position #[0-9]* swap/)
+    ),
+    category: 26
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'position funding cost') ||
+      _startsWith(d, 'interest charge') ||
+      _test(d, /position #[0-9]* funding cost/)
+    ),
+    category: 27
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'margin funding payment') ||
+      _startsWith(d, 'swap payment') ||
+      _startsWith(d, 'interest payment')
+    ),
+    category: 28
+  },
+  {
+    tester: (d) => (
+      _includes(d, 'settlement')
+    ),
+    category: 31
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'transfer')
+    ),
+    category: 51
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'deposit (') ||
+      _startsWith(d, 'deposit on wallet')
+    ),
+    category: 101
+  },
+  {
+    tester: (d) => (
+      _includes(d, ' withdrawal #')
+    ),
+    category: 104
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'canceled withdrawal')
+    ),
+    category: 105
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'trading fee')
+    ),
+    category: 201
+  },
+  {
+    tester: (d) => (
+      _includes(d, 'trading rebate')
+    ),
+    category: 202
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'hidden order fee')
+    ),
+    category: 204
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'otc trade fee')
+    ),
+    category: 207
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'swap fee')
+    ),
+    category: 222
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'claiming fee')
+    ),
+    category: 224
+  },
+  {
+    tester: (d) => (
+      _includes(d, 'margin funding charge')
+    ),
+    category: 226
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'earned fee') ||
+      _startsWith(d, 'affiliate rebate')
+    ),
+    category: 241
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'ETHFX loyalty fee')
+    ),
+    category: 243
+  },
+  {
+    tester: (d) => (
+      _includes(d, 'deposit fee')
+    ),
+    category: 251
+  },
+  {
+    tester: (d) => (
+      _includes(d, 'withdrawal fee')
+    ),
+    category: 254
+  },
+  {
+    tester: (d) => (
+      _includes(d, 'withdrawal express fee')
+    ),
+    category: 255
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'miner fee')
+    ),
+    category: 258
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'staking reward')
+    ),
+    category: 262
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'adjustment')
+    ),
+    category: 401
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'expense')
+    ),
+    category: 501
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'currency conversion') ||
+      _startsWith(d, 'computation fee')
+    ),
+    category: 905
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'monthly profit payment')
+    ),
+    category: 907
+  },
+  {
+    tester: (d) => (
+      _startsWith(d, 'losses')
+    ),
+    category: 911
+  }
+]
+
+module.exports = (description) => {
+  const d = description.toLowerCase()
+
+  for (const { tester, category } of _schema) {
+    if (tester(d)) {
+      return category
+    }
+  }
+
+  return null
+}

workers/loc.api/sync/data.inserter/api.middleware/helpers/get-flags-from-ledger-description.js

@@ -16,8 +16,17 @@ module.exports = (
     const {
       fieldName,
       pattern,
+      handler,
       isCaseSensitivity
     } = { ...schema }
+
+    if (typeof handler === 'function') {
+      return {
+        ...accum,
+        [fieldName]: handler(ledger.description)
+      }
+    }
+
     const regExp = new RegExp(
       pattern,
       isCaseSensitivity ? '' : 'i'

workers/loc.api/sync/data.inserter/api.middleware/helpers/index.js

@@ -6,8 +6,12 @@ const addPropsToResIfExist = require(
 const getFlagsFromLedgerDescription = require(
   './get-flags-from-ledger-description'
 )
+const getCategoryFromDescription = require(
+  './get-category-from-description'
+)
 
 module.exports = {
   addPropsToResIfExist,
-  getFlagsFromLedgerDescription
+  getFlagsFromLedgerDescription,
+  getCategoryFromDescription
 }