Merge pull request #43 from HXSecurity/feature/issue-80

Feature/issue 80

Author: exexute

.github/workflows/release_engine.yml

@@ -0,0 +1,74 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Release DongTai Engine
+
+on:
+  push:
+    branches: [ "release-*" ]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.7]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      - name: start-build
+        uses: joelwmale/webhook-action@master
+        with:
+          url: ${{ secrets.WEBHOOK_URL }}
+          body: '{"msg_type": "interactive","card": {"config": {"wide_screen_mode": true,"enable_forward": true},"elements": [{"tag": "div","text": {"content": "状态：项目${{github.repository}}构建开始\n分支：${{github.ref}}\n流程：${{github.workflow}}\n构建编号：${{github.run_number}}\n触发事件：${{github.event_name}}\n提交人：${{github.actor}}\nSHA-1：${{github.sha}}\n","tag": "lark_md"}}]}}'
+
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - name: Set up Python 3.7
+        uses: actions/checkout@v1
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-prod.txt
+
+      - name: Lint with flake8
+        run: |
+          pip install flake8
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.ALIYUN_REGISTRY }}
+          username: ${{ secrets.ALIYUN_DOCKERHUB_USER }}
+          password: ${{ secrets.ALIYUN_DOCKERHUB_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: "registry.cn-beijing.aliyuncs.com/huoxian_pub/dongtai-engine:1.0.0,"
+
+      - name: finish build
+        uses: joelwmale/webhook-action@master
+        with:
+          url: ${{ secrets.WEBHOOK_URL }}
+          body: '{"msg_type": "interactive","card": {"config": {"wide_screen_mode": true,"enable_forward": true},"elements": [{"tag": "div","text": {"content": "状态：项目${{github.repository}}构建成功\n分支：${{github.ref}}\n流程：${{github.workflow}}\n构建编号：${{github.run_number}}\n触发事件：${{github.event_name}}\n提交人：${{github.actor}}\nSHA-1：${{github.sha}}\n","tag": "lark_md"}}]}}'

conf/uwsgi.ini

@@ -3,8 +3,6 @@
 #uwsgi监听的socket，可以为socket文件或ip地址+端口号，用nginx的时候就配socket , 直接运行的时候配 http, http-socket = 127.0.0.1:8080
 http = :8000
 
-listen = 1024
-
 #指定项目的目录，在app加载前切换到当前目录
 chdir = /opt/dongtai/engine
 

core/tasks.py

@@ -399,7 +399,7 @@ def update_sca():
 
 
 def is_alive(agent_id, timestamp):
-    return IastHeartbeat.objects.values('id').filter(agent__id=agent_id, dt__gt=(timestamp - 600)).exists()
+    return IastHeartbeat.objects.values('id').filter(agent__id=agent_id, dt__gt=(timestamp - 60 * 20)).exists()
 
 
 @shared_task(queue='dongtai-periodic-task')

deploy/deploy-eks-iast-saas-engine-prod.yml

@@ -1,24 +1,24 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: iast-engine
+  name: dongtai-engine
   namespace: iast-prod
   annotations:
-    kubesphere.io/description: iast-engine
+    kubesphere.io/description: dongtai-engine
   labels:
-    app: iast-engine
+    app: dongtai-engine
 spec:
   replicas: 5
   selector:
     matchLabels:
-      app: iast-engine
+      app: dongtai-engine
   template:
     metadata:
       labels:
-        app: iast-engine
+        app: dongtai-engine
     spec:
       containers:
-        - name: iast-engine-container
+        - name: dongtai-engine-container
           image: registry.cn-beijing.aliyuncs.com/secnium/iast-saas-engine:VERSION
           imagePullPolicy: Always
           volumeMounts:
@@ -38,6 +38,6 @@ spec:
       volumes:
         - name: configfile
           configMap:
-            name: iast-test-config.ini
+            name: dongtai-iast-config-prod.ini
       imagePullSecrets:
         - name: aliyun-registry-secret

deploy/deploy-eks-iast-saas-engine-task-prod.yml

@@ -1,24 +1,24 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: iast-engine-task
+  name: dongtai-engine-task
   namespace: iast-prod
   annotations:
-    kubesphere.io/description: iast-engine-task
+    kubesphere.io/description: dongtai-engine-task
   labels:
-    app: iast-engine-task
+    app: dongtai-engine-task
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: iast-engine-task
+      app: dongtai-engine-task
   template:
     metadata:
       labels:
-        app: iast-engine-task
+        app: dongtai-engine-task
     spec:
       containers:
-        - name: iast-engine-task-container
+        - name: dongtai-engine-task-container
           image: registry.cn-beijing.aliyuncs.com/secnium/iast-saas-engine:VERSION
           command: ["/bin/bash","/opt/dongtai/engine/docker/entrypoint.sh"]
           args: ["task"]
@@ -40,6 +40,6 @@ spec:
       volumes:
         - name: configfile
           configMap:
-            name: iast-test-config.ini
+            name: dongtai-iast-config-prod.ini
       imagePullSecrets:
         - name: aliyun-registry-secret

lingzhi_engine/celery.py

@@ -25,22 +25,23 @@
 #   should have a `CELERY_` prefix.
 
 configs["CELERY_QUEUES"] = [
-    Queue("dongtai-method-pool-scan", Exchange("method_pool"), routing_key="method_pool"),
-    Queue("dongtai-replay-vul-scan", Exchange("method_pool"), routing_key="method_pool"),
-    Queue("dongtai-strategy-scan", Exchange("strategy"), routing_key="strategy"),
-    Queue("dongtai-search-scan", Exchange("search"), routing_key="search"),
-    Queue("dongtai-periodic-task", Exchange("periodic_task"), routing_key="periodic_task"),
-    Queue("dongtai-replay-task", Exchange("replay_task"), routing_key="replay_task"),
+    Queue("dongtai-method-pool-scan", Exchange("dongtai-method-pool-scan"), routing_key="dongtai-method-pool-scan"),
+    Queue("dongtai-replay-vul-scan", Exchange("dongtai-replay-vul-scan"), routing_key="dongtai-replay-vul-scan"),
+    Queue("dongtai-strategy-scan", Exchange("dongtai-strategy-scan"), routing_key="dongtai-strategy-scan"),
+    Queue("dongtai-search-scan", Exchange("dongtai-search-scan"), routing_key="dongtai-search-scan"),
+    Queue("dongtai-periodic-task", Exchange("dongtai-periodic-task"), routing_key="dongtai-periodic-task"),
+    Queue("dongtai-replay-task", Exchange("dongtai-replay-task"), routing_key="dongtai-replay-task"),
 ]
 configs["CELERY_ROUTES"] = {
-    "core.tasks.search_vul_from_method_pool": {'exchange': 'method_pool', 'routing_key': 'method_pool'},
-    "core.tasks.search_vul_from_strategy": {'exchange': 'strategy', 'routing_key': 'strategy'},
-    "core.tasks.search_sink_from_method_pool": {'exchange': 'search', 'routing_key': 'search'},
-    "core.tasks.update_sca": {'exchange': 'periodic_task', 'routing_key': 'periodic_task'},
-    "core.tasks.update_agent_status": {'exchange': 'periodic_task', 'routing_key': 'periodic_task'},
-    "core.tasks.heartbeat": {'exchange': 'periodic_task', 'routing_key': 'periodic_task'},
-    "core.tasks.clear_error_log": {'exchange': 'periodic_task', 'routing_key': 'periodic_task'},
-    "core.tasks.vul_recheck": {'exchange': 'replay_task', 'routing_key': 'replay_task'},
+    "core.tasks.search_vul_from_method_pool": {'exchange': 'dongtai-method-pool-scan', 'routing_key': 'dongtai-method-pool-scan'},
+    "core.tasks.search_vul_from_strategy": {'exchange': 'dongtai-strategy-scan', 'routing_key': 'dongtai-strategy-scan'},
+    "core.tasks.search_vul_from_replay_method_pool": {'exchange': 'dongtai-replay-vul-scan', 'routing_key': 'dongtai-replay-vul-scan'},
+    "core.tasks.search_sink_from_method_pool": {'exchange': 'dongtai-search-scan', 'routing_key': 'dongtai-search-scan'},
+    "core.tasks.update_sca": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'},
+    "core.tasks.update_agent_status": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'},
+    "core.tasks.heartbeat": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'},
+    "core.tasks.clear_error_log": {'exchange': 'dongtai-periodic-task', 'routing_key': 'dongtai-periodic-task'},
+    "core.tasks.vul_recheck": {'exchange': 'dongtai-replay-task', 'routing_key': 'dongtai-replay-task'},
 }
 configs["CELERY_ENABLE_UTC"] = False
 configs["CELERY_TIMEZONE"] = settings.TIME_ZONE

signals/handlers/vul_handler.py

@@ -230,11 +230,10 @@ def save_vul(vul_meta, vul_level, vul_name, vul_stack, top_stack, bottom_stack,
 
     vul = IastVulnerabilityModel.objects.filter(
         type=vul_name,  # 指定漏洞类型
-        url=vul_meta.url,
+        uri=vul_meta.uri,
         http_method=vul_meta.http_method,
-        taint_position=taint_position,
-        param_name=param_name,
-        agent=vul_meta.agent
+        agent=vul_meta.agent,
+        method_pool_id=vul_meta.id
     ).first()
     if vul:
         vul.req_header = vul_meta.req_header

test/core/tasks.py

@@ -17,7 +17,7 @@ def test_search_vul_from_replay_method_pool(self):
         search_vul_from_replay_method_pool(method_id)
 
     def test_search_vul_from_method_pool(self):
-        method_pool_id = 585
+        method_pool_id = 2311521
         from core.tasks import search_vul_from_method_pool
         search_vul_from_method_pool(method_pool_id)
 

test/task.py

@@ -1,6 +1,6 @@
 import unittest
 
-from core.tasks import maven_spider, heartbeat
+from core.tasks import heartbeat, search_vul_from_method_pool
 from test import DongTaiTestCase
 
 
@@ -56,9 +56,6 @@ def test_agent_status_update(self):
         from core.tasks import update_agent_status
         update_agent_status()
 
-    def test_maven_spider(self):
-        maven_spider()
-
     def test_heart_beat(self):
         import os
         os.environ.setdefault("DJANGO_SETTINGS_MODULE", "lingzhi_engine.settings")