Merge pull request #20 from CAFECA-IO/feature/aiTrade

Luphia · web-flow · commit 689322b0a069 · 2024-04-17T19:02:58.000+08:00
Feature/ai trade
diff --git a/.github/workflows/autotrade.yml b/.github/workflows/autotrade.yml
@@ -21,7 +21,7 @@ jobs:
         with:
           node-version: 18
       - name: Install dependencies
-        run: npm ci
+        run: npm i
       - name: Build nestjs app
         run: npm run build
       - name: Run tests
diff --git a/.gitignore b/.gitignore
@@ -34,4 +34,6 @@ lerna-debug.log*
 !.vscode/launch.json
 !.vscode/extensions.json
 request.http
-package-lock.json
+
+# package-lock.json
+package-lock.json
diff --git a/README.md b/README.md
@@ -176,7 +176,7 @@ Parameters
 
 ```typescript
 
-PUT /tradebot?id=tradebotId
+GET /tradebot?id=tradebotId
 
 ```
 
diff --git a/dqn/model.json b/dqn/model.json
diff --git a/dqn/weights.bin b/dqn/weights.bin
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "auto-trade",
-  "version": "0.0.1+1",
+  "version": "0.0.1+2",
   "description": "",
   "author": "",
   "private": true,
diff --git a/src/price_ticker/price_ticker.controller.ts b/src/price_ticker/price_ticker.controller.ts
@@ -4,5 +4,4 @@ import { PriceTickerService } from './price_ticker.service';
 @Controller('price-ticker')
 export class PriceTickerController {
   constructor(private readonly priceTickerService: PriceTickerService) {}
-
 }
diff --git a/src/price_ticker/price_ticker.service.ts b/src/price_ticker/price_ticker.service.ts
@@ -46,7 +46,7 @@ export class PriceTickerService {
   async getCandlesticks(
     instId: string = 'ETH-USDT',
     timeSpan: string = '5m',
-    limit: number = 300,
+    limit: number = 100,
   ): Promise<number[]> {
     const { data } = await firstValueFrom(
       this.httpService.get<CandlestickDto>(
diff --git a/src/strategies/strategies.controller.ts b/src/strategies/strategies.controller.ts
@@ -7,6 +7,7 @@ export class StrategiesController {
 
   @Get()
   async trainDqn() {
-    this.strategiesService.trainDqn();
+    await this.strategiesService.trainDqn();
+    return 'Training DQN finished';
   }
 }
diff --git a/src/strategies/strategies.service.spec.ts b/src/strategies/strategies.service.spec.ts
@@ -116,58 +116,22 @@ describe('StrategiesService', () => {
     // Info: (20240320 - Jacky) this is aim to sum the profit of all trades
     const profitArray = tradeArray.map((trade) => trade.profit);
     const sum = profitArray.reduce((total, profit) => total + profit, 0);
-    expect(sum).toBe(-1069.9531315100012);
+    expect(sum).toBe(-1060.9868554650009);
   });
 
   it('should backtest use api', async () => {
     console.log = () => {};
-    const ETHPriceArray = [
-      3250.92, 3227.38, 3264, 3252.43, 3277.55, 3264.01, 3280.58, 3260.4,
-      3258.96, 3275.94, 3292.73, 3281.7, 3273.62, 3259.7, 3222.2, 3228.48,
-      3225.21, 3229.59, 3228, 3230.22, 3245.07, 3244.62, 3227.1, 3213.34,
-      3205.78, 3218.81, 3212.59, 3216.6, 3243.82, 3254.6, 3260.18, 3276.6,
-      3273.45, 3276.47, 3270, 3289.41, 3278.61, 3257.55, 3272.34, 3264.35,
-      3278.03, 3292.56, 3303.6, 3295.36, 3305.09, 3295.91, 3282.62, 3279.36,
-      3279.6, 3273.79, 3296.25, 3306.6, 3285.4, 3295.09, 3279.6, 3272.93,
-      3277.75, 3292.59, 3260, 3242.66, 3241.78, 3237.8, 3222.6, 3234.3, 3238.34,
-      3252.28, 3267.21, 3264.54, 3258.5, 3259.2, 3250.44, 3248.98, 3235.4,
-      3245.99, 3257.86, 3251.08, 3277.35, 3295.41, 3312.28, 3323.59, 3308.63,
-      3302.98, 3306.66, 3317.32, 3303.59, 3299.27, 3296.26, 3299, 3302.28,
-      3315.59, 3324.42, 3323.63, 3332.26, 3327.73, 3338.18, 3332.01, 3341.54,
-      3335.91, 3342.99, 3350.09, 3331.74, 3322.51, 3317.57, 3312.87, 3312.96,
-      3314.18, 3307.64, 3295.61, 3287, 3304.74, 3301.41, 3306.23, 3311.47,
-      3312.52, 3329.5, 3314.01, 3312.61, 3316.38, 3306.79, 3329.9, 3332.61,
-      3342.5, 3356, 3344.52, 3334.21, 3342.99, 3339.01, 3332.32, 3327.49,
-      3333.23, 3312.47, 3297.72, 3297.61, 3287.31, 3272.79, 3276.98, 3293.86,
-      3295.18, 3278.81, 3291.58, 3279.67, 3279.21, 3283.9, 3272.82, 3275.05,
-      3282.55, 3262.38, 3273.59, 3273.32, 3280.56, 3284.08, 3298.75, 3294.55,
-      3291.02, 3285.03, 3278.96, 3252.62, 3246.18, 3233.95, 3226.14, 3243.15,
-      3229.47, 3211.81, 3216.67, 3201.42, 3211.38, 3184.59, 3177.5, 3170.81,
-      3174.33, 3155.81, 3159.2, 3173.39, 3162.93, 3162.81, 3175.59, 3163.63,
-      3158.4, 3158.05, 3173.82, 3168.86, 3193.39, 3201.67, 3206.98, 3199.53,
-      3213.39, 3215.19, 3213.22, 3220.43, 3235.22, 3220.58, 3212.89, 3211.78,
-      3188.78, 3168.61, 3188.41, 3174.71, 3175.2, 3164.39, 3179.45, 3187.09,
-      3175.39, 3187.27, 3195.52, 3203.25, 3214.48, 3210.22, 3218.77, 3216.85,
-      3221, 3221.37, 3225.99, 3216.23, 3224, 3230.16, 3227.6, 3224.2, 3240,
-      3251.13, 3263.32, 3253.84, 3248.38, 3232.99, 3210.99, 3197.31, 3160.85,
-      3174.81, 3193.03, 3191.8, 3187.52, 3162.31, 3160.3, 3121.62, 3130,
-      3119.86, 3133, 3128.61, 3128.84, 3125.61, 3114.78, 3102.2, 3104.79,
-      3069.41, 3074, 3087.97, 3096.29, 3098.59, 3092.15, 3095.56, 3110.15,
-      3113.2, 3119.41, 3128.17, 3124.95, 3136.15, 3142.86, 3140, 3137.91,
-      3144.45, 3143.04, 3146.88, 3140.86, 3148.23, 3154.01, 3160.57, 3165.67,
-      3174.26, 3184.2, 3184.82, 3181.76, 3210.49, 3205.48, 3218.07, 3217.91,
-      3210.38, 3221.28, 3220.03, 3219.03, 3220.01, 3224.01, 3228.11, 3212.89,
-      3211.79, 3224.31, 3216.85, 3207.8, 3224.44, 3220.49, 3212.7, 3219.26,
-    ];
+    const ethArrFile = fs.readFileSync('src/strategies/etharr.txt', 'utf8');
+    const etharr = JSON.parse(ethArrFile);
     const { tradeArray } = await strategiesService.backTesting(
       'autoArima',
       'autoArima',
       'autoArima',
-      ETHPriceArray,
+      etharr,
     );
     // Info: (20240320 - Jacky) this is aim to sum the profit of all trades
     const profitArray = tradeArray.map((trade) => trade.profit);
     const sum = profitArray.reduce((total, profit) => total + profit, 0);
-    expect(sum).toBe(-673.9419000000021);
+    expect(sum).toBe(-7023.318874999997);
   });
 });
diff --git a/src/strategies/strategies.service.ts b/src/strategies/strategies.service.ts
@@ -1,8 +1,8 @@
 import { Injectable } from '@nestjs/common';
 import { Environment } from './strategy/rl-dqn/environment';
-import * as fs from 'fs';
 import { TradeAgent } from './strategy/rl-dqn/tradeAgent';
 import { train } from './strategy/rl-dqn/train';
+import * as fs from 'fs';
 
 @Injectable()
 export class StrategiesService {
@@ -99,18 +99,9 @@ export class StrategiesService {
   }
 
   async trainDqn() {
-    // const csvContent = fs.readFileSync('src/strategies/ETH-USD.csv', 'utf8');
-    const etharr = fs.readFileSync('src/strategies/etharr.txt', 'utf8');
-    console.log("🚀 ~ StrategiesService ~ trainDqn ~ etharr:", etharr)
-    const rows = etharr.split('\n');
-    // Define the index of the column you want to read
-    const closeIndex = 4;
-    // Extract the data from the specified column
-    const priceData = rows.map((row) => {
-      const columns = row.split(','); // Split by comma (adjust delimiter as necessary)
-      return parseFloat(columns[closeIndex]);
-    });
-    const env = new Environment(priceData.slice(1));
+    const ethArrFile = fs.readFileSync('src/strategies/etharr.txt', 'utf8');
+    const etharr = JSON.parse(ethArrFile);
+    const env = new Environment(etharr);
     const tradeAgent = new TradeAgent(env);
     await train(tradeAgent);
   }
diff --git a/src/strategies/strategy/rl-dqn/dqn.ts b/src/strategies/strategy/rl-dqn/dqn.ts
@@ -20,7 +20,7 @@ import * as tf from '@tensorflow/tfjs-node';
 export function createDeepQNetwork(numActions) {
   const model = tf.sequential();
   model.add(
-    tf.layers.dense({ units: 32, activation: 'relu', inputShape: [52] }),
+    tf.layers.dense({ units: 32, activation: 'relu', inputShape: [104] }),
   );
   model.add(tf.layers.dense({ units: 64, activation: 'relu' }));
   model.add(tf.layers.dense({ units: 128, activation: 'relu' }));
diff --git a/src/strategies/strategy/rl-dqn/environment.ts b/src/strategies/strategy/rl-dqn/environment.ts
@@ -10,7 +10,8 @@ export class Environment {
   openPrice: number;
   currentPrice: number;
   profitLog: number[];
-  actionLog: any[] = [];
+  actionLog: any[];
+  rewardLog: any[];
 
   constructor(priceArray) {
     this.current_step = 48;
@@ -23,6 +24,7 @@ export class Environment {
     this.holdingStatus = 0;
     this.profitLog = [];
     this.actionLog = [];
+    this.rewardLog = [];
   }
 
   reset() {
@@ -47,7 +49,6 @@ export class Environment {
     let profit = 0;
     let reward = 0;
     this.currentPrice = this.priceArray[this.current_step];
-    // console.log("🚀 ~ Environment ~ step ~ currentPrice:", this.currentPrice)
     this.spreadFee = 0.005 * this.currentPrice;
     if (action === 3) {
       if (this.holdingStatus === 0) {
@@ -71,29 +72,29 @@ export class Environment {
         const currentPriceStr = JSON.stringify(this.currentPrice);
         this.openPrice = JSON.parse(currentPriceStr);
         this.holdingStatus = 1;
-        reward = 20;
+        reward = 25;
         if (this.openPrice < this.priceArray[this.current_step + 10]) {
-          reward += 15;
+          reward += 24;
         } else {
-          reward += -15;
+          reward += -14;
         }
       } else {
-        reward = -20;
+        reward = -35;
       }
     }
     if (action === 2) {
       if (this.holdingStatus === 0) {
         const currentPriceStr = JSON.stringify(this.currentPrice);
         this.openPrice = JSON.parse(currentPriceStr);
         this.holdingStatus = 2;
-        reward = 20;
+        reward = 25;
         if (this.openPrice > this.priceArray[this.current_step + 10]) {
-          reward += 15;
+          reward += 20;
         } else {
-          reward += -15;
+          reward += -20;
         }
       } else {
-        reward = -20;
+        reward = -35;
       }
     }
     this.current_step += 1;
@@ -105,13 +106,8 @@ export class Environment {
         reward = -30;
       }
     }
-    this.actionLog.push(action);
-    this.profitLog.push(profit);
     return {
-      previousPrice: this.priceArray.slice(
-        this.current_step - 48,
-        this.current_step,
-      ),
+      previousPrice: this.priceArray.slice(-100),
       currentPrice: this.currentPrice,
       openPrice: this.openPrice,
       spreadFee: this.spreadFee,
@@ -121,7 +117,7 @@ export class Environment {
   }
   getState() {
     return [
-      this.priceArray.slice(this.current_step - 48, this.current_step),
+      this.priceArray.slice(-100),
       this.currentPrice,
       this.openPrice,
       this.spreadFee,
diff --git a/src/strategies/strategy/rl-dqn/models/dqn/model.json b/src/strategies/strategy/rl-dqn/models/dqn/model.json
@@ -0,0 +1 @@
+{"modelTopology":{"class_name":"Sequential","config":{"name":"sequential_1","layers":[{"class_name":"Dense","config":{"units":32,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense1","trainable":true,"batch_input_shape":[null,104],"dtype":"float32"}},{"class_name":"Dense","config":{"units":64,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense2","trainable":true}},{"class_name":"Dense","config":{"units":128,"activation":"relu","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense3","trainable":true}},{"class_name":"Dropout","config":{"rate":0.25,"noise_shape":null,"seed":null,"name":"dropout_Dropout1","trainable":true}},{"class_name":"Dense","config":{"units":4,"activation":"sigmoid","use_bias":true,"kernel_initializer":{"class_name":"VarianceScaling","config":{"scale":1,"mode":"fan_avg","distribution":"normal","seed":null}},"bias_initializer":{"class_name":"Zeros","config":{}},"kernel_regularizer":null,"bias_regularizer":null,"activity_regularizer":null,"kernel_constraint":null,"bias_constraint":null,"name":"dense_Dense4","trainable":true}}]},"keras_version":"tfjs-layers 4.17.0","backend":"tensor_flow.js"},"weightsManifest":[{"paths":["weights.bin"],"weights":[{"name":"dense_Dense1/kernel","shape":[104,32],"dtype":"float32"},{"name":"dense_Dense1/bias","shape":[32],"dtype":"float32"},{"name":"dense_Dense2/kernel","shape":[32,64],"dtype":"float32"},{"name":"dense_Dense2/bias","shape":[64],"dtype":"float32"},{"name":"dense_Dense3/kernel","shape":[64,128],"dtype":"float32"},{"name":"dense_Dense3/bias","shape":[128],"dtype":"float32"},{"name":"dense_Dense4/kernel","shape":[128,4],"dtype":"float32"},{"name":"dense_Dense4/bias","shape":[4],"dtype":"float32"}]}],"format":"layers-model","generatedBy":"TensorFlow.js tfjs-layers v4.17.0","convertedBy":null}
diff --git a/src/strategies/strategy/rl-dqn/models/dqn/weights.bin b/src/strategies/strategy/rl-dqn/models/dqn/weights.bin
diff --git a/src/strategies/strategy/rl-dqn/tradeAgent.ts b/src/strategies/strategy/rl-dqn/tradeAgent.ts
@@ -55,7 +55,7 @@ export class TradeAgent {
    *   - `learningRate` {number} The learning rate to use during training.
    */
   constructor(environment) {
-    this.NUM_ACTIONS = 300;
+    this.NUM_ACTIONS = 4;
     this.env = environment;
     this.epsilonInit = 1;
     this.epsilonFinal = 0.1;
diff --git a/src/strategies/strategy/rl-dqn/train.ts b/src/strategies/strategy/rl-dqn/train.ts
@@ -1,5 +1,6 @@
 import * as tf from '@tensorflow/tfjs-node';
 import { TradeAgent } from './tradeAgent';
+import { copyWeights } from './dqn';
 
 class MovingAverager {
   buffer: any[];
@@ -22,7 +23,7 @@ class MovingAverager {
 
 export async function train(
   agent: TradeAgent,
-  batchSize = 32,
+  batchSize = 48,
   gamma = 0.99,
   learningRate = 1e-3,
   savePath = '/models',
@@ -36,9 +37,10 @@ export async function train(
 
   const optimizer = tf.train.adam(learningRate);
   let averageReward100Best = -Infinity;
+  let syncCount = 0;
   while (true) {
     agent.trainOnReplayBatch(batchSize, gamma, optimizer);
-    const { action, cumulativeReward, done } = agent.playStep();
+    const { cumulativeReward, done } = agent.playStep();
     if (done) {
       rewardAverager100.append(cumulativeReward);
       const averageReward100 = rewardAverager100.average();
@@ -50,7 +52,12 @@ export async function train(
           );
         }
       }
+      syncCount++;
       return agent.cumulativeReward_;
     }
+    if (syncCount === 300) {
+      syncCount = 0;
+      copyWeights(agent.targetNetwork, agent.onlineNetwork);
+    }
   }
 }
diff --git a/src/tradebot/tradebot.controller.ts b/src/tradebot/tradebot.controller.ts
@@ -92,6 +92,10 @@ export class TradebotController {
         const depositCommand = this.tradebotService.receiveDeposit(tradebot);
         return depositCommand;
       }
+      if (data.command === 'aiTrade') {
+        const runAiCommand = this.tradebotService.aiTrade(tradebot);
+        return runAiCommand;
+      }
       return data.command + 'is invalid command';
     } catch (error) {
       return error.message;
diff --git a/src/tradebot/tradebot.service.spec.ts b/src/tradebot/tradebot.service.spec.ts
@@ -127,11 +127,11 @@ describe('TradebotService', () => {
       const tradebot = new Tradebot();
       const instId = 'ETH-USDT';
       jest
-        .spyOn(service, 'executeStrategy')
+        .spyOn(service, 'executeTrade')
         .mockResolvedValue(
           'Tradebot ' + tradebot.id + ' sucessfully close position',
         );
-      const result = await service.executeStrategy(tradebot, instId);
+      const result = await service.executeTrade(tradebot, instId, 'CLOSE');
       // Assert
       expect(result).toBe(
         'Tradebot ' + tradebot.id + ' sucessfully close position',
@@ -144,16 +144,24 @@ describe('TradebotService', () => {
       const instId = 'ETH-USDT';
       // Act
       jest
-        .spyOn(service, 'executeStrategy')
+        .spyOn(service, 'executeTrade')
         .mockResolvedValue(
           'Tradebot ' + tradebot.id + ' failed to close position',
         );
-      const result = await service.executeStrategy(tradebot, instId);
+      const result = await service.executeTrade(tradebot, instId, 'CLOSE');
 
       // Assert
       expect(result).toBe(
         'Tradebot ' + tradebot.id + ' failed to close position',
       );
     });
   });
+  it('should convert holding status to number', () => {
+    // Arrange
+    const holdingStatus = 'WAIT';
+    // Act
+    const result = service.convertHoldingStatus(holdingStatus);
+    // Assert
+    expect(result).toBe(0);
+  });
 });
diff --git a/src/tradebot/tradebot.service.ts b/src/tradebot/tradebot.service.ts

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "auto-trade",`
`3`		`- "version": "0.0.1+1",`
	`3`	`+ "version": "0.0.1+2",`
`4`	`4`	`"description": "",`
`5`	`5`	`"author": "",`
`6`	`6`	`"private": true,`
Original file line number	Diff line number	Diff line change
`@@ -4,5 +4,4 @@ import { PriceTickerService } from './price_ticker.service';`
`4`	`4`	`@Controller('price-ticker')`
`5`	`5`	`export class PriceTickerController {`
`6`	`6`	`constructor(private readonly priceTickerService: PriceTickerService) {}`
`7`		`-`
`8`	`7`	`}`
Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@ export class StrategiesController {`
`7`	`7`
`8`	`8`	`@Get()`
`9`	`9`	`async trainDqn() {`
`10`		`- this.strategiesService.trainDqn();`
	`10`	`+ await this.strategiesService.trainDqn();`
	`11`	`+ return 'Training DQN finished';`
`11`	`12`	`}`
`12`	`13`	`}`