Merge remote-tracking branch 'origin/main'

client/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.7.12-slim
 
-ARG EXTRA_TOOLS="curl wget python3-dev git git-lfs vim-tiny gcc locales subversion telnet procps openssh-client"
+ARG EXTRA_TOOLS="curl wget python3-dev git git-lfs vim-tiny gcc locales subversion telnet procps openssh-client libreadline-dev"
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends $EXTRA_TOOLS \
@@ -10,6 +10,7 @@ RUN apt-get update \
     && echo "LANG=zh_CN.UTF-8" > /etc/locale.conf \
     && locale-gen \
     && ln -sf /usr/share/zoneinfo/Asia/Hong_Kong /etc/localtime \
+    && cd /lib/x86_64-linux-gnu && ln -sf libreadline.so.8.1 libreadline.so.6 \
     && rm -rf /var/cache/apt/* /root/.cache
 
 WORKDIR /workspace/client

client/tool/luacheck.py

@@ -12,6 +12,7 @@
 
 import sys
 import os
+import threading
 
 try:
     import xml.etree.cElementTree as ET
@@ -24,6 +25,7 @@
 from util.subprocc import SubProcController
 from util.pathfilter import FilterPathUtil
 from util.logutil import LogPrinter
+from util.scanlang.callback_queue import CallbackQueue
 
 logger = LogPrinter
 
@@ -40,12 +42,11 @@ def analyze(self, params):
         incr_scan = params["incr_scan"]
         path_exclude = params.path_filters.get("wildcard_exclusion", [])
         path_include = params.path_filters.get("wildcard_inclusion", [])
-        error_output = os.path.join(work_dir, "luacheck_result.xml")
         LUACHECK_HOME = os.environ.get("LUACHECK_HOME")
         pos = len(source_dir) + 1
         path_mgr = PathMgr()
         want_suffix = ".lua"
-
+        issues = []
         # update_task_progress(request, 'LuaCheck工具开始分析', 40)
         # luacheck在win下支持分析目录，在mac和linux下需要安装luafilesystem才能支持分析目录.暂不使用 --cache
         # 1. --read-globals coroutine._yield 是为了消除luacheck对coroutine._yield的误报，详细如下：
@@ -80,7 +81,10 @@ def analyze(self, params):
                 scan_cmd.append("--exclude-files")
                 for path in path_exclude:
                     scan_cmd.append(os.path.join(source_dir, path))
+            error_output = os.path.join(work_dir, "luacheck_result.xml")
+            issues.extend(LuaCheckRunner().run_luacheck(scan_cmd, error_output, pos, rules))
         else:
+            # 单文件分析
             toscans = []
             if incr_scan:
                 diffs = SCMMgr(params).get_scm_diff()
@@ -92,19 +96,34 @@ def analyze(self, params):
             else:
                 toscans = path_mgr.get_dir_files(source_dir, want_suffix)
             # filter include and exclude path
-            relpos = len(source_dir) + 1
-            toscans = FilterPathUtil(params).get_include_files(toscans, relpos)
+            toscans = FilterPathUtil(params).get_include_files(toscans, pos)
             if not toscans:
                 logger.debug("To-be-scanned files is empty ")
                 return []
             # 在文件前后加上双引号，变成字符串，防止文件路径中有特殊字符比如(，luacheck无法分析
             toscans = [self.handle_path_with_space(path) for path in toscans]
-            scan_cmd.extend(toscans)
-        scan_cmd = PathMgr().format_cmd_arg_list(scan_cmd)
-        self.print_log(scan_cmd)
-        SubProcController(scan_cmd, stdout_filepath=error_output, stderr_line_callback=self.print_log).wait()
+            # 多线程分析
+            issues = ThreadRunner(scan_cmd, work_dir, pos, rules).run(toscans)
+        logger.info(issues)
+        return issues
+
+    def handle_path_with_space(self, path):
+        """
+        处理文件路径中包含特殊字符比如(的情况
+        :param path:
+        :return:
+        """
+        return '"' + path + '"'
+
+class LuaCheckRunner(object):
+    def __init__(self):
+        pass
 
-        # update_task_progress(request, '分析结果处理', 60)
+    def run_luacheck(self, scan_cmd, error_output, pos, rules):
+        """
+        执行luacheck并处理结果
+        """
+        SubProcController(scan_cmd, stdout_filepath=error_output, stderr_line_callback=logger.info).wait()
         if sys.platform == "win32":
             xmlContent = open(error_output, "r", encoding="gbk").read()
             open(error_output, "w", encoding="utf-8").write(xmlContent)
@@ -121,16 +140,40 @@ def analyze(self, params):
                 line = int(msg.split(":")[1])
                 column = int(msg.split(":")[2])
                 issues.append({"path": path, "rule": rule, "msg": msg, "line": line, "column": column})
-        logger.debug(issues)
         return issues
 
-    def handle_path_with_space(self, path):
-        """
-        处理文件路径中包含特殊字符比如(的情况
-        :param path:
-        :return:
-        """
-        return '"' + path + '"'
+
+class ThreadRunner(object):
+    # 多线程执行 luacheck 类
+    def __init__(self, scan_cmd, work_dir, pos, rules):
+        self.scan_cmd = scan_cmd
+        self.work_dir = work_dir
+        self.issues = []
+        self.pos = pos
+        self.rules = rules
+        self.mutex = threading.Lock()  # 线程锁
+
+    def __run_luacheck_on_file(self, index, file_path):
+        scan_cmd = self.scan_cmd.copy()
+        scan_cmd.append(file_path)
+        scan_cmd = PathMgr().format_cmd_arg_list(scan_cmd)
+        error_output = os.path.join(self.work_dir, str(index) + "luacheck_result.xml")
+        return LuaCheckRunner().run_luacheck(scan_cmd, error_output, self.pos, self.rules)
+
+    def __scan_file_callback_(self, index, file_path):
+        file_result = self.__run_luacheck_on_file(index, file_path)
+        self.mutex.acquire()  # 上锁
+        self.issues.extend(file_result)
+        self.mutex.release()  # 解锁
+
+    def run(self, toscans):
+        # 多线程执行类
+        callback_queue = CallbackQueue(min_threads=20, max_threads=1000)
+        for index, path in enumerate(toscans):
+            LogPrinter.info("path: %s" % path)
+            callback_queue.append(self.__scan_file_callback_, index, path)
+        callback_queue.wait_for_all_callbacks_to_be_execute_and_destroy()
+        return self.issues
 
 
 tool = Luacheck

doc/en/quickStarted/enhanceDeploy.md

@@ -22,46 +22,8 @@ The TCA enhanced analysis module requires users to additionally deploy the Licen
 
 **Continuously updating...**
 
-## TCA Trial Version Enhanced Capability Application
-If you usually use the [TCA trial version public service](https://tca.tencent.com/) and want to experience the enhanced analysis module's analysis capabilities on the trial version, you can apply and configure as follows:
-
-1. Apply for the Client License by email to the TCA team
-
-- Recipient email:
-```
-[email protected]
-[email protected]
-[email protected]
-[email protected]
-```
-
-- The application email format is as follows:
-
-Subject: **TCA Independent Tool Client License Application**
-
-Content:
-| | | 
-|  :----:  | :----:  |
-| Applicant's Name  | xxx |
-| Name of the Organization the Applicant Belongs to | xxx |
-| Type of Organization the Applicant Belongs to | Optional: Company/School/Individual |
-| Applicant's Email | xxx |
-| Applicant's Mobile Number | xxx |
-| Is it Privately Deployed | No |
-| Purpose of Experience Application | xxx |
-
-2. After receiving the reply email from the TCA team, configure the CLS microservice information in the `config.ini` of the TCA Client directory, for example
-
-```ini
-[LICENSE_CONFIG]
-; [optional] Fill in when using independent tools, no need by default
-; Domain name and port of the License service
-URL=https://tca.tencent.com
-BASE_PATH=server/license
-LICENSE=<Client License>
-```
-
-After the configuration is complete, you can normally use the enhanced analysis module capabilities on the TCA trial version.
+## TCA official website version enhanced capability application
+If users usually use [TCA official website version public service](https://tca.tencent.com/) and want to experience the analysis capabilities of the enhanced analysis module on the official website version, they can follow [this help document](https://tca.tencent.com/document/zh/guide/%E5%AE%A2%E6%88%B7%E7%AB%AF%E6%8E%A5%E5%85%A5/License%E9%85%8D%E7%BD%AE.html) to apply for configuration.
 
 ## TCA Private Deployment Enhanced Capability Application
 If you are using TCA in an enterprise intranet environment and want to experience the enhanced analysis module capabilities of TCA on the intranet, you can apply as follows.