Merge pull request #212 from boostcampwm-2024/dev-back

Dev back merge to main

Author: hyo-limilimee

backend/console-server/src/log/rank/dto/get-elapsed-time-rank-response.dto.ts

@@ -25,15 +25,15 @@ export class GetElapsedTimeRankResponseDto {
         example: [
             {
                 projectName: 'test059',
-                elapsedTime: 100,
+                value: 100,
             },
             {
                 projectName: 'test007',
-                elapsedTime: 110,
+                value: 110,
             },
             {
                 projectName: 'test079',
-                elapsedTime: 120,
+                value: 120,
             },
         ],
     })

backend/name-server/src/database/query/dau-recorder.ts

@@ -8,8 +8,9 @@ export class DAURecorder implements DAURecorderInterface {
     private clickhouseClient = ClickhouseDatabase.getInstance();
 
     public async recordAccess(domain: string): Promise<void> {
-        const dateString = new Date().toLocaleDateString();
-        const values = [{ domain: domain.toLowerCase(), date: dateString, access: 1 }];
+        const values = [
+            { domain: domain.toLowerCase(), date: this.formatDate(new Date()), access: 1 },
+        ];
         try {
             await this.clickhouseClient.insert({
                 table: 'dau',
@@ -20,4 +21,12 @@ export class DAURecorder implements DAURecorderInterface {
             console.error('ClickHouse Error:', error);
         }
     }
+
+    private formatDate(date: Date): string {
+        const year = date.getFullYear();
+        const month = String(date.getMonth() + 1).padStart(2, '0');
+        const day = String(date.getDate()).padStart(2, '0');
+
+        return `${year}-${month}-${day}`;
+    }
 }

backend/proxy-server/package.json

@@ -7,6 +7,7 @@
   "scripts": {
     "test": "jest --config jest.config.js -ts-config=tsconfig.test.json",
     "build": "tsc -p tsconfig.json && tsc-alias -p tsconfig.json",
+    "start": "NODE_ENV=production tsx src/index.ts",
     "dev": "NODE_ENV=development tsx src/index.ts",
     "lint": "eslint src/**/*.ts",
     "lint:fix": "eslint src/**/*.ts --fix"

backend/proxy-server/src/app.ts

@@ -29,7 +29,10 @@ export class Application {
             const projectCacheRepository = new ProjectCacheRepositoryRedis();
             const projectService = new ProjectService(projectRepository, projectCacheRepository);
             const proxyService = new ProxyService(projectService);
-            const logRepository = new LogRepositoryClickhouse();
+            const logRepository = new LogRepositoryClickhouse({
+                maxSize: 1000,
+                flushIntervalSecond: 5,
+            });
             const logService = new LogService(logRepository);
 
             const errorLogRepository = new ErrorLogRepository();

backend/proxy-server/src/database/query/log.repository.clickhouse.ts

@@ -4,35 +4,80 @@ import { LogRepository } from '../../domain/log/log.repository';
 import { HttpLogEntity } from '../../domain/log/http-log.entity';
 import { ClickHouseClient } from '@clickhouse/client';
 import { formatDateTime } from '../../common/utils/date.util';
+import { LogBufferConfig } from 'domain/config/log-buffer.config';
+import { FastifyLogger } from 'common/logger/fastify.logger';
+
+type httpLogRecord = {
+    method: string;
+    path: string;
+    host: string;
+    status_code: number;
+    elapsed_time: number;
+    timestamp: string;
+};
 
 export class LogRepositoryClickhouse implements LogRepository {
     private readonly clickhouse: ClickHouseClient;
+    private readonly config: LogBufferConfig;
+    private logBuffer: httpLogRecord[] = [];
+    private flushTimer: NodeJS.Timeout | null = null;
+    private isProcessing: boolean = false;
 
-    constructor() {
+    constructor(config: LogBufferConfig) {
         this.clickhouse = ClickhouseDatabase.getInstance();
+        this.config = config;
+        this.startFlushTimer();
     }
 
-    public async insertHttpLog(log: HttpLogEntity): Promise<void> {
-        const values = [
-            {
-                method: log.method,
-                path: log.path || '',
-                host: log.host,
-                status_code: log.statusCode,
-                elapsed_time: Math.round(log.responseTime),
-                timestamp: formatDateTime(new Date()),
-            },
-        ];
+    private startFlushTimer(): void {
+        if (this.flushTimer) {
+            clearInterval(this.flushTimer);
+        }
+
+        this.flushTimer = setInterval(async () => {
+            if (this.logBuffer.length > 0 && !this.isProcessing) {
+                await this.flush();
+            }
+        }, this.config.flushIntervalSecond * 1000);
+    }
+
+    private async flush(): Promise<void> {
+        if (this.isProcessing || this.logBuffer.length === 0) {
+            return;
+        }
+        let batchToFlush = [...this.logBuffer];
 
+        this.logBuffer = [];
         try {
+            this.isProcessing = true;
+
             await this.clickhouse.insert({
                 table: 'http_log',
-                values: values,
+                values: batchToFlush,
                 format: 'JSONEachRow',
             });
         } catch (error) {
-            console.error('ClickHouse Error:', error);
+            this.logBuffer = [...this.logBuffer, ...batchToFlush];
             throw new DatabaseQueryError(error as Error);
+        } finally {
+            this.isProcessing = false;
+        }
+    }
+
+    public async insertHttpLog(log: HttpLogEntity): Promise<void> {
+        const httpLogRecord: httpLogRecord = {
+            method: log.method,
+            path: log.path || '',
+            host: log.host,
+            status_code: log.statusCode,
+            elapsed_time: Math.round(log.responseTime),
+            timestamp: formatDateTime(new Date()),
+        };
+
+        this.logBuffer.push(httpLogRecord);
+
+        if (this.logBuffer.length >= this.config.maxSize) {
+            this.flush();
         }
     }
 }

backend/proxy-server/src/domain/config/log-buffer.config.ts

@@ -0,0 +1,4 @@
+export type LogBufferConfig = {
+    maxSize: number;
+    flushIntervalSecond: number;
+};

backend/proxy-server/src/domain/proxy/proxy.service.ts

@@ -34,8 +34,6 @@ export class ProxyService {
 
             return ip;
         } catch (error) {
-            console.log('error: ', error);
-
             if (error instanceof ProxyError) {
                 throw error;
             }

backend/proxy-server/src/server/config/fastify.config.ts

@@ -10,7 +10,7 @@ export const fastifyConfig = {
         process.env.NODE_ENV === 'development'
             ? true
             : {
-                  level: process.env.LOG_LEVEL || 'info',
+                  level: process.env.LOG_LEVEL || 'warning',
                   serializers: {
                       req(
                           request: FastifyRequest<