Merge pull request #2 from Nizernizer/main

1

.github/workflows/code-check.yml

@@ -1,7 +1,7 @@
 # This workflow will build a Java project with Maven
 # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
 
-name: CodeCheck For Develop
+name: DongTai Agent Java CodeCheck、Unit Test And Vulns Test
 
 on:
   push:
@@ -14,22 +14,70 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.6
+      - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
-          java-version: 1.6
+          java-version: 1.8
+
+      - uses: dbelyaev/action-checkstyle@master
+        with:
+          github_token: ${{ secrets.github_token }}
+          reporter: github-pr-review
+          level: warning
 
       - name: Set up Maven
         uses: stCarolas/setup-maven@v4
         with:
           maven-version: 3.2.5
 
-      - name: maven-settings-xml-action
-        uses: whelk-io/maven-settings-xml-action@v14
-        with:
-          mirrors: '[{ "id": "aliyunmaven", "mirrorOf": "*", "url": "https://maven.aliyun.com/repository/public" }]'
-
       - name: Build DongTai-Java-Agent With Maven
         run: |
           mvn -version
           mvn -B package -Dmaven.test.skip=true
+      
+      - uses: shogo82148/actions-setup-mysql@v1
+        with:
+          mysql-version: '8.0'
+          auto-start: true
+          root-password: yuhjnbGYUI
+          user: test
+          password: test
+
+      - name: OpenRASP-Vulns-Test
+        run: |
+          cd ..
+
+          echo "download apache-tomcat-8.5.0..."
+          wget https://github.com/exexute/github_action_samples/releases/download/1.0.0/apache-tomcat-8.5.40.zip 1>/dev/null
+          unzip apache-tomcat-8.5.40.zip
+
+          echo "copy dongtai agent java to tomcat..."
+          cp DongTai-agent-java/release/iast-agent.jar apache-tomcat-8.5.40/iast/agent.jar
+          cp DongTai-agent-java/release/lib/iast-core.jar apache-tomcat-8.5.40/temp/iast-core.jar
+          cp DongTai-agent-java/release/lib/iast-inject.jar apache-tomcat-8.5.40/temp/iast-inject.jar
+          cd apache-tomcat-8.5.40
+
+          echo "init mysql"
+          cat init.sql
+          mysql -uroot -pyuhjnbGYUI -h127.0.0.1 < init.sql
+          mysql -uroot -pyuhjnbGYUI -h127.0.0.1 -e 'show DATABASES;'
+
+          echo "start catalina and waitting 30s..."
+          ./bin/startup.sh 2>/dev/null
+          sleep 30
+
+          echo "start vulns spider..."
+          ./spider.sh
+
+          echo "waitting for send vuls data"
+          sleep 600
+      
+      - name: Benchmark-Vulns-Test
+        run: |
+          pwd
+          ls -l
+          java -version
+          mvn -version
+          # download Benchmark
+          # Copy Agent To Benchmark
+          # Run

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,77 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ develop ]
+  schedule:
+    - cron: '28 6 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+
+    - name: Set up Maven
+      uses: stCarolas/setup-maven@v4
+      with:
+        maven-version: 3.2.5
+
+    - name: Build DongTai-Java-Agent With Maven
+      run: |
+        mvn -version
+        mvn -B package -Dmaven.test.skip=true
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

README.md

@@ -28,7 +28,7 @@ Dongtai-agent-java consists of `agent.jar`, `iast-core-jar`, `iast-inject. Jar`
 
 ## Quick Start
 
-Please refer to the [Quick Start](https://hxsecurity.github.io/DongTaiDoc/#/doc/tutorial/quickstart).
+Please refer to the [Quick Start](https://hxsecurity.github.io/DongTai-Doc/#/doc/tutorial/quickstart).
 
 ## Quick Development
 
@@ -72,4 +72,4 @@ Please refer to the [Quick Start](https://hxsecurity.github.io/DongTaiDoc/#/doc/
 ### Supported Java versions and middleware
 
 - Java 1.6+
-- Tomcat, Jetty, WebLogic, WebSphere, SpringBoot and Mainstream software and middleware.
+- Tomcat, Jetty, WebLogic, WebSphere, SpringBoot and Mainstream software and middleware.

README_CN.md

@@ -31,7 +31,7 @@ DongTai-agent-java 由`agent.jar`、`iast-core.jar `、`iast-inject.jar`、`dong
 
 ### 快速使用
 
-请参考：[快速开始](https://hxsecurity.github.io/DongTaiDoc/#/doc/tutorial/quickstart)
+请参考：[快速开始](https://hxsecurity.github.io/DongTai-Doc/#/doc/tutorial/quickstart)
 
 ### 快速开发
 

iast-agent/src/main/resources/iast.properties

@@ -3,7 +3,7 @@ iast.version=1.0.0
 iast.response.name=dongtai
 iast.response.value=1.0.0
 iast.server.url=http://openapi.iast.huoxian.cn:8000
-iast.server.token=88d2f0096662335d42580cbd03d8ddea745fdfab
+iast.server.token=79798299b48839c84886d728958a8f708e119868
 iast.allhook.enable=false
 iast.dump.class.enable=false
 iast.dump.class.path=/tmp/iast-class-dump/

iast-core/src/main/java/com/secnium/iast/core/AbstractThread.java

@@ -22,7 +22,6 @@ public void run() {
             EngineManager.turnOffLingzhi();
         }
         try {
-            Thread.sleep(properties.getReportInterval());
             send();
         } catch (IOException e) {
             logger.error("report error reason: ", e);

iast-core/src/main/java/com/secnium/iast/core/EngineManager.java

@@ -1,5 +1,6 @@
 package com.secnium.iast.core;
 
+import com.secnium.iast.core.handler.models.IastReplayModel;
 import com.secnium.iast.core.handler.models.MethodEvent;
 import com.secnium.iast.core.middlewarerecognition.IastServer;
 import com.secnium.iast.core.middlewarerecognition.ServerDetect;
@@ -37,6 +38,7 @@ public class EngineManager {
 
     private static final ArrayBlockingQueue<String> REPORTS = new ArrayBlockingQueue<String>(2048);
     private static final ArrayBlockingQueue<String> METHOD_REPORT = new ArrayBlockingQueue<String>(2048);
+    private final static ArrayBlockingQueue<IastReplayModel> REPLAY_QUEUE = new ArrayBlockingQueue<IastReplayModel>(256);
 
     private static boolean logined = false;
     private static int reqCounts = 0;
@@ -152,6 +154,26 @@ public static boolean hasNewReport() {
         return !REPORTS.isEmpty();
     }
 
+    public static int getReportQueueSize() {
+        return REPORTS.size();
+    }
+
+    public static boolean hasReplayData() {
+        return !REPLAY_QUEUE.isEmpty();
+    }
+
+    public static IastReplayModel getReplayModel() {
+        return REPLAY_QUEUE.poll();
+    }
+
+    public static void sendReplayModel(IastReplayModel replayModel) {
+        REPLAY_QUEUE.offer(replayModel);
+    }
+
+    public static int getReplayQueueSize() {
+        return REPLAY_QUEUE.size();
+    }
+
     public static void sendMethodReport(String report) {
         METHOD_REPORT.offer(report);
     }
@@ -164,6 +186,10 @@ public static boolean hasMethodReport() {
         return !METHOD_REPORT.isEmpty();
     }
 
+    public static int getMethodReportQueueSize() {
+        return METHOD_REPORT.size();
+    }
+
     public static boolean getIsLoginLogic() {
         return LOGIN_LOGIC_WEIGHT.get() != null && LOGIN_LOGIC_WEIGHT.get().equals(2);
     }

iast-core/src/main/java/com/secnium/iast/core/PropertyUtils.java

@@ -22,6 +22,7 @@ public class PropertyUtils {
     private String iastDumpPath;
     private Long heartBeatInterval = -1L;
     private Long reportInterval = -1L;
+    private Long replayInterval = -1L;
     private String serverUrl;
     private String namespace;
     private String engineName;
@@ -165,10 +166,10 @@ public boolean isEnableDumpClass() {
     }
 
     public long getReplayInterval() {
-        if (heartBeatInterval == -1L) {
-            heartBeatInterval = Long.valueOf(System.getProperty("iast.service.replay.interval", cfg.getProperty("iast.service.replay.interval", "300000")));
+        if (replayInterval == -1L) {
+            replayInterval = Long.valueOf(System.getProperty("iast.service.replay.interval", cfg.getProperty("iast.service.replay.interval", "5000")));
         }
-        return heartBeatInterval;
+        return replayInterval;
     }
 
     public long getReportInterval() {
@@ -178,6 +179,13 @@ public long getReportInterval() {
         return reportInterval;
     }
 
+    public long getHeartBeatInterval() {
+        if (heartBeatInterval == -1L) {
+            heartBeatInterval = Long.valueOf(System.getProperty("iast.service.heartbeat.interval", cfg.getProperty("iast.service.heartbeat.interval", "5")));
+        }
+        return heartBeatInterval;
+    }
+
     public String getBaseUrl() {
         if (null == serverUrl) {
             serverUrl = System.getProperty("iast.server.url", cfg.getProperty("iast.server.url"));

iast-core/src/main/java/com/secnium/iast/core/ServiceFactory.java

@@ -2,6 +2,7 @@
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.secnium.iast.core.replay.HttpRequestReplay;
+import com.secnium.iast.core.report.HeartBeatSender;
 import com.secnium.iast.core.report.MethodReportSender;
 import com.secnium.iast.core.report.ReportSender;
 
@@ -16,11 +17,14 @@ public class ServiceFactory {
     private static ServiceFactory INSTANCE;
     private final long replayInterval;
     private final long reportInterval;
-    private final ScheduledExecutorService executorService;
+    private final long heartBeatInterval;
+    private final ScheduledExecutorService heartBeatService;
+    private final ScheduledExecutorService reportService;
 
     ReportSender report = null;
     HttpRequestReplay requestReplay = null;
     MethodReportSender methodReportSender = null;
+    HeartBeatSender heartBeatSender = null;
 
     public static ServiceFactory getInstance() {
         if (null == INSTANCE) {
@@ -37,19 +41,23 @@ public ServiceFactory() {
         PropertyUtils propertiesUtils = PropertyUtils.getInstance();
         this.replayInterval = propertiesUtils.getReplayInterval();
         this.reportInterval = propertiesUtils.getReportInterval();
-        this.executorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("dongtai-engine-report").build());
+        this.heartBeatInterval = propertiesUtils.getHeartBeatInterval();
+        this.heartBeatService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("dongtai-heartbeat").build());
+        this.reportService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("dongtai-report").build());
     }
 
     public void init() {
+        heartBeatSender = new HeartBeatSender();
         methodReportSender = new MethodReportSender();
         report = new ReportSender();
         requestReplay = new HttpRequestReplay();
     }
 
     public void start() {
-        executorService.scheduleWithFixedDelay(methodReportSender, 0, reportInterval, TimeUnit.MILLISECONDS);
-        executorService.scheduleWithFixedDelay(report, 0, reportInterval, TimeUnit.MILLISECONDS);
-        executorService.scheduleWithFixedDelay(requestReplay, 0, replayInterval, TimeUnit.MILLISECONDS);
+        heartBeatService.scheduleWithFixedDelay(heartBeatSender, 0, heartBeatInterval, TimeUnit.SECONDS);
+        reportService.scheduleWithFixedDelay(methodReportSender, 0, reportInterval, TimeUnit.MILLISECONDS);
+        reportService.scheduleWithFixedDelay(report, 0, reportInterval, TimeUnit.MILLISECONDS);
+        reportService.scheduleWithFixedDelay(requestReplay, 0, replayInterval, TimeUnit.MILLISECONDS);
     }
 
     public void stop() {

iast-core/src/main/java/com/secnium/iast/core/handler/vulscan/ReportConstant.java

@@ -100,4 +100,8 @@ public class ReportConstant {
     public static final String API_DATA_FILE = "file";
     public static final String API_DATA_CONTROLLER = "controller";
     public static final String API_DATA_DESCRIPTION = "description";
+
+    public static final String REPORT_QUEUE = "report_queue";
+    public static final String METHOD_QUEUE = "method_queue";
+    public static final String REPLAY_QUEUE = "replay_queue";
 }