Merge branch 'feature/tools' of https://github.com/yalechen-cyw/CodeAnalysis into feature/tools

Author: cyw3

LICENSE LICENSE.txtLICENSE renamed to LICENSE.txt

@@ -10,6 +10,10 @@
 
 English | [简体中文](README_ZH.md)
 
+## TCA Official Website
+
+[https://cloud.tencent.com/product/tcap](https://cloud.tencent.com/product/tcap)
+
 ## TCA Github Pages
 
 [https://tencent.github.io/CodeAnalysis/](https://tencent.github.io/CodeAnalysis/)
@@ -45,12 +49,9 @@ Using TCA can help team find normative, structural, security vulnerabilities and
 
 ## Getting Started
 
-- [How to get start](https://tencent.github.io/CodeAnalysis/)
+- [How to deploy](https://tencent.github.io/CodeAnalysis/en/quickStarted/)
 - [How to use TCA Action](https://github.com/TCATools/TCA-action/blob/main/README.md)
-- [How to deploy server and web](https://tencent.github.io/CodeAnalysis/zh/quickStarted/deploySever.html#通过源代码)
-- [How to deploy server and web with docker-compose](https://tencent.github.io/CodeAnalysis/zh/quickStarted/deploySever.html#通过docker-compose)
-- [How to use client](https://tencent.github.io/CodeAnalysis/zh/quickStarted/deployClient.html)
-- [Deploy Q&A](https://tencent.github.io/CodeAnalysis/zh/quickStarted/FAQ.html)
+- [How to use client](https://tencent.github.io/CodeAnalysis/en/guide/客户端/本地分析.html)
 
 ## Community
 

README.md

@@ -8,13 +8,17 @@
 
 [![license](https://img.shields.io/badge/License-MIT-brightgreen.svg?style=flat)](LICENSE) [![docs](https://img.shields.io/badge/docs-read-brightgreen.svg?style=flat)](https://tencent.github.io/CodeAnalysis/)
 
-## TCA Github Pages
+## TCA-官方网址
+
+[https://cloud.tencent.com/product/tcap](https://cloud.tencent.com/product/tcap)
+
+## TCA-Github代码库
 
 [https://tencent.github.io/CodeAnalysis/](https://tencent.github.io/CodeAnalysis/)
 
 [https://github.com/TCATools](https://github.com/TCATools)
 
-## 工蜂代码库镜像
+## TCA-国内镜像工蜂代码库
 
 [https://git.code.tencent.com/Tencent_Open_Source/CodeAnalysis.git](https://git.code.tencent.com/Tencent_Open_Source/CodeAnalysis.git)
 
@@ -43,12 +47,9 @@
 
 ## 快速入门
 
-- [快速入门](https://tencent.github.io/CodeAnalysis/)
+- [快速部署](https://tencent.github.io/CodeAnalysis/zh/quickStarted/)
 - [如何使用TCA Action快速体验](https://github.com/TCATools/TCA-action/blob/main/README.md)
-- [如何在本地部署Server与Web](https://tencent.github.io/CodeAnalysis/zh/quickStarted/deploySever.html#通过源代码)
-- [如何通过Docker-Compose部署Server与Web](https://tencent.github.io/CodeAnalysis/zh/quickStarted/deploySever.html#通过docker-compose)
-- [如何使用客户端](https://tencent.github.io/CodeAnalysis/zh/quickStarted/deployClient.html)
-- [部署常见问题与解决方式](https://tencent.github.io/CodeAnalysis/zh/quickStarted/FAQ.html)
+- [如何使用客户端](https://tencent.github.io/CodeAnalysis/zh/guide/客户端/本地分析.html)
 
 ## 社区
 

README_ZH.md

@@ -1,6 +1,6 @@
 FROM python:3.7.12-slim
 
-ARG EXTRA_TOOLS="curl wget python3-dev git git-lfs vim-tiny gcc locales subversion telnet procps"
+ARG EXTRA_TOOLS="curl wget python3-dev git git-lfs vim-tiny gcc locales subversion telnet procps openssh-client"
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends $EXTRA_TOOLS \

client/Dockerfile

@@ -40,7 +40,7 @@ def acquire_task_dir(self, task_id=None, dirname_prefix="task_"):
                     suffix_no += 1
                 task_dir = os.path.join(self._task_dirs_root, f"{dirname_prefix}{task_id}_{suffix_no}")
         else:  # 没有传task_id，根据本地task dir名称排序，创建一个新的
-            task_id = 0
+            task_id = 1
             while True:
                 if not os.path.exists(os.path.join(self._task_dirs_root, f"{dirname_prefix}{task_id}")):
                     break

client/node/common/taskdirmgr.py

@@ -75,6 +75,7 @@ def scan_project(self, execute_request_list, proj_conf):
         if execute_request_list and not self._remote_task_names:
             if self._source_dir:
                 task_request = copy.deepcopy(execute_request_list[0])  # 深度copy,以免影响原字典数据
+                task_request["task_name"] = "linecount"
                 RequestModify.modify_local_task_request(task_request, self._task_name_id_maps, self._job_id,
                                                         self._scm_auth_info.ssh_file,
                                                         self._token, self._server_url, self._source_dir, self._scm_info,

client/node/localtask/runlocaltask.py

@@ -193,6 +193,8 @@ def run(self):
                     kill_task_id = task_params['task_id']
                     # kill task时进程如果已不存在,会报异常: ProcessLookupError: [Errno 3] No such process
                     self._terminate_task(kill_task_id)
+                    # 等待一段时间后再接下一个任务
+                    time.sleep(self._get_task_interval)
                     continue
 
                 # 获取到分析任务，向server发送确认信息(kill_task不需要确认)
@@ -229,6 +231,8 @@ def run(self):
                 task = Task(task_id, task_name, request_file, response_file, task_log, env=self._origin_os_env)
                 task.start()
                 self._running_task.append(task)
+                # 等待一段时间后再接下一个任务
+                time.sleep(self._get_task_interval)
             except:
                 # 遇到异常,输出异常信息
                 LogPrinter.exception("task loop encounter error.")

client/node/servertask/looprunner.py

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2022 THL A29 Limited
+#
+# This source code file is made available under MIT License
+# See LICENSE for details
+# ==============================================================================
+
+"""
+通过http方式拉取工具
+"""
+
+import os
+
+from util.logutil import LogPrinter
+from util.api.fileserver import RetryFileServer
+from util.exceptions import FileServerError
+from util.pathlib import PathMgr
+from util.zipmgr import Zip
+
+
+class HttpToolLoader(object):
+    """通过http方式拉取工具，如果目录已存在，不拉取"""
+    @staticmethod
+    def download_tool(tool_url, dest_dir):
+        if os.path.exists(dest_dir):
+            # 工具目录存在时，直接复用，不重新拉取（如需更新，先删除工具目录）
+            LogPrinter.debug(f"tool dir({os.path.basename(dest_dir)}) from zip can be reused.")
+            return
+
+        tool_root_dir = os.path.dirname(dest_dir)
+        if not os.path.exists(tool_root_dir):  # 如果上层目录不存在,先创建
+            os.makedirs(tool_root_dir)
+        zip_file_name = tool_url.split('/')[-1]
+        dest_zip_file_path = os.path.join(tool_root_dir, zip_file_name)
+
+        file_server = RetryFileServer(retry_times=2).get_server(server_url=tool_url)
+        file_server.download_big_file("", dest_zip_file_path)
+
+        if os.path.exists(dest_zip_file_path):
+            LogPrinter.debug(f"download {tool_url} to {dest_zip_file_path}")
+            # 使用7z解压
+            Zip().decompress_by_7z(dest_zip_file_path, tool_root_dir)
+            LogPrinter.debug(f"unzip {dest_zip_file_path} to {dest_dir}")
+            PathMgr().safe_rmpath(dest_zip_file_path)
+        else:
+            raise FileServerError(f"download {tool_url} failed!")
+
+
+if __name__ == '__main__':
+    pass

client/node/toolloader/httploadtool.py

@@ -161,7 +161,8 @@ def __update_tool_names(self, tool_names, task_list):
         tool_names.remove(c_name)
         return tool_names
 
-    def read_tool_config_from_ini_file(self, tool_names=None, custom_tools=None, task_list=None, config_all_tools=False, include_common=True):
+    def read_tool_config_from_ini_file(self, tool_names=None, custom_tools=None, task_list=None, config_all_tools=False,
+                                       include_common=True):
         """
         从ini文件中读取工具配置
         """
@@ -194,6 +195,7 @@ def read_tool_config_from_ini_file(self, tool_names=None, custom_tools=None, tas
                 "env_path": env_path_section,  # env_path_section中现在已经没有key为"PATH"的元素,可以作为编译工具的env_path
                 "env_value": env_value_section
             }
+            self.__format_to_fullpath(compile_config)
             config_dict["compile_config"] = compile_config
             return config_dict
         else:
@@ -236,6 +238,25 @@ def __format_config(self, tool_cfg, env_path_section, env_value_section):
         tool_cfg["env_value"] = self.__str_to_dict(tool_cfg["env_value"], env_value_section)
         tool_cfg["tool_url"] = self.__str_to_list(tool_cfg["tool_url"])
         tool_cfg["path"] = self.__str_to_list(tool_cfg["path"], uniq=False)
+        self.__format_to_fullpath(tool_cfg)
+
+    def __format_to_fullpath(self, tool_cfg):
+        # 将路径类型的环境变量（env_path）中的相对路径转换成绝对路径
+        for env_name, rel_path in tool_cfg["env_path"].items():
+            if "PATH" == env_name:  # PATH应该单独放在path字段中，如果放在env_path中，忽略，避免影响和覆盖原有PATH变量
+                continue
+            full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
+            tool_cfg["env_path"][env_name] = full_path
+
+        # 将PATH环境变量的相对路径转换成绝对路径
+        path_env = []
+        for rel_path in tool_cfg["path"]:
+            if "$" in rel_path or "%" in rel_path:  # 带变量的环境变量，已经是全路径
+                full_path = rel_path
+            else:
+                full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
+            path_env.append(full_path)
+        tool_cfg["path"] = path_env
 
     def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
         """
@@ -266,7 +287,7 @@ def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
         config_dict = {}
         already_config_tools = []  # 记录已经读取到配置的工具
         for task_config in task_list:
-            task_name = task_config["task_name"]
+            task_name = task_config.get("task_name")
             if tool_names and task_name not in tool_names:
                 continue
             task_params = task_config.get("task_params", {})
@@ -282,29 +303,41 @@ def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
                     for tool_lib in tool_libs:
                         lib_support_os = tool_lib.get("os", [])
                         if self._os_type in lib_support_os:
+                            # 使用新的结构存储环境变量，避免修改原有的任务参数
+                            new_lib_envs = {}
                             lib_envs = tool_lib.get("envs", {})
                             if not lib_envs:  # 可能会传空list，这里判空，避免后面格式出错
-                                lib_envs = {}
+                                new_lib_envs = {}
+
                             scm_url = tool_lib.get("scm_url")
-                            lib_dir_name = scm_url.split('/')[-1].strip().replace(".git", "")
-                            # 环境变量中的$ROOT_DIR替换为目录名，后续加载环境变量时会拼接为全路径
+                            # 支持git仓库地址和zip包地址两种格式
+                            lib_dir_name = BaseScmUrlMgr.get_last_dir_name_from_url(scm_url)
+                            lib_dir_path = os.path.join(settings.TOOL_BASE_DIR, lib_dir_name)
+
+                            # 将环境变量中的$ROOT_DIR替换为实际路径，重新保存到new_lib_envs
                             for key, value in lib_envs.items():
                                 if "$ROOT_DIR" in value:
-                                    lib_envs[key] = value.replace("$ROOT_DIR", lib_dir_name)
+                                    new_lib_envs[key] = value.replace("$ROOT_DIR", lib_dir_path)
+                                else:
+                                    new_lib_envs[key] = value
+
                             # PATH和其他环境变量分开处理
                             path_envs = []
                             other_envs = {}
-                            for env_name, env_value in lib_envs.items():
+                            for env_name, env_value in new_lib_envs.items():
                                 if env_name == "PATH":
                                     path_envs = self.__str_to_list(env_value)
                                 else:
-                                    other_envs[env_name] = env_value
+                                    # 根据英文分号拆分后，再环境变量分隔符拼接到一起
+                                    value_list = self.__str_to_list(env_value)
+                                    value_format = os.pathsep.join(value_list)
+                                    other_envs[env_name] = value_format
                             lib_config = {
                                 "tool_url": scm_url,
                                 "scm_type": tool_lib.get("scm_type"),
                                 "auth_info": tool_lib.get("auth_info"),
                                 "path": path_envs,
-                                "env_path": lib_envs,
+                                "env_path": other_envs,
                                 "env_value": {}
                             }
                             lib_name = tool_lib.get("name")

client/node/toolloader/loadconfig.py

@@ -18,16 +18,25 @@
 from node.app import settings
 from node.toolloader.gitload import GitLoader
 from node.toolloader.loadconfig import ConfigLoader, ToolConfig, LoadToolTypes
+from node.toolloader.httploadtool import HttpToolLoader
 from util.envset import EnvSet
 from util.scanlang.callback_queue import CallbackQueue
 from util.pathlib import PathMgr
 from util.logutil import LogPrinter
 from util.scmurlmgr import BaseScmUrlMgr
 from util.subprocc import SubProcController
+from util.textutil import ZIP_EXT
 
 
 class ToolCommonLoader(object):
     @staticmethod
+    def is_zip_url(tool_url):
+        """判断是否是压缩包地址"""
+        if tool_url.lower().endswith(ZIP_EXT):
+            return True
+        else:
+            return False
+    @staticmethod
     def load_tool_type(tool_dirpath, tool_dirname, tool_url=None):
         """
         判断使用哪种方式加载工具
@@ -38,8 +47,10 @@ def load_tool_type(tool_dirpath, tool_dirname, tool_url=None):
         """
         if settings.USE_LOCAL_TOOL == "True":
             if os.path.exists(tool_dirpath):
-                # LogPrinter.info(f"USE_LOCAL_TOOL=True, use local tool dir: {tool_dirpath}")
+                LogPrinter.info(f"USE_LOCAL_TOOL=True, use local tool dir: {tool_dirpath}")
                 return "Local", None
+            elif tool_url and ToolCommonLoader.is_zip_url(tool_url):
+                return "HTTP", None
             else:
                 return "Git", None
         else:
@@ -54,6 +65,8 @@ def load_tool_type(tool_dirpath, tool_dirname, tool_url=None):
                         return "Copy", tool_dirpath_copy_from
                     else:  # 拷贝源目录不存在，从git拉取
                         return "Git", None
+            elif tool_url and ToolCommonLoader.is_zip_url(tool_url):
+                return "HTTP", None
             else:
                 return "Git", None
 
@@ -69,6 +82,8 @@ def load_tool(load_type, tool_dirpath, tool_dirpath_copy_from, git_url, scm_auth
         :param print_enable:
         :return:
         """
+        if settings.DEBUG:
+            print_enable = True
         if load_type == "Local":
             if print_enable:
                 LogPrinter.info(f"Use local tool dir: {tool_dirpath}")
@@ -78,6 +93,11 @@ def load_tool(load_type, tool_dirpath, tool_dirpath_copy_from, git_url, scm_auth
                 LogPrinter.info(f"Copy from {tool_dirpath_copy_from} to {tool_dirpath}")
             PathMgr().retry_copy(tool_dirpath_copy_from, tool_dirpath)
             return tool_dirpath
+        elif load_type == "HTTP":
+            if print_enable:
+                LogPrinter.info(f"Load from {git_url} to {tool_dirpath}")
+            HttpToolLoader.download_tool(git_url, tool_dirpath)
+            return tool_dirpath
         else:
             if print_enable:
                 LogPrinter.info(f"Load from git to {tool_dirpath} ...")