✨工具拉取支持htttp zip包地址

Author: bensonhome

client/node/toolloader/httploadtool.py

@@ -0,0 +1,47 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2022 THL A29 Limited
+#
+# This source code file is made available under MIT License
+# See LICENSE for details
+# ==============================================================================
+
+"""
+通过http方式拉取工具
+"""
+
+import os
+
+from util.logutil import LogPrinter
+from util.api.fileserver import RetryFileServer
+from util.exceptions import FileServerError
+from util.ziplib import ZipMgr
+
+
+class HttpToolLoader(object):
+    """通过http方式拉取工具，如果目录已存在，不拉取"""
+    @staticmethod
+    def download_tool(tool_url, dest_dir):
+        if os.path.exists(dest_dir):
+            # 工具目录存在时，直接复用，不重新拉取（如需更新，先删除工具目录）
+            LogPrinter.debug(f"tool dir({os.path.basename(dest_dir)}) from zip can be reused.")
+            return
+
+        tool_root_dir = os.path.dirname(dest_dir)
+        if not os.path.exists(tool_root_dir):  # 如果上层目录不存在,先创建
+            os.makedirs(tool_root_dir)
+        zip_file_name = tool_url.split('/')[-1]
+        dest_zip_file_path = os.path.join(tool_root_dir, zip_file_name)
+
+        file_server = RetryFileServer(retry_times=2).get_server(server_url=tool_url)
+        file_server.download_big_file("", dest_zip_file_path)
+
+        if os.path.exists(dest_zip_file_path):
+            LogPrinter.debug(f"download {tool_url} to {dest_zip_file_path}")
+            ZipMgr.depress(dest_zip_file_path, tool_root_dir)
+            LogPrinter.debug(f"unzip {dest_zip_file_path} to {dest_dir}")
+        else:
+            raise FileServerError(f"download {tool_url} failed!")
+
+
+if __name__ == '__main__':
+    pass

client/node/toolloader/loadconfig.py

@@ -310,7 +310,8 @@ def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
                                 new_lib_envs = {}
 
                             scm_url = tool_lib.get("scm_url")
-                            lib_dir_name = scm_url.split('/')[-1].strip().replace(".git", "")
+                            # 支持git仓库地址和zip包地址两种格式
+                            lib_dir_name = BaseScmUrlMgr.get_last_dir_name_from_url(scm_url)
                             lib_dir_path = os.path.join(settings.TOOL_BASE_DIR, lib_dir_name)
 
                             # 将环境变量中的$ROOT_DIR替换为实际路径，重新保存到new_lib_envs

client/node/toolloader/loadtool.py

@@ -18,16 +18,25 @@
 from node.app import settings
 from node.toolloader.gitload import GitLoader
 from node.toolloader.loadconfig import ConfigLoader, ToolConfig, LoadToolTypes
+from node.toolloader.httploadtool import HttpToolLoader
 from util.envset import EnvSet
 from util.scanlang.callback_queue import CallbackQueue
 from util.pathlib import PathMgr
 from util.logutil import LogPrinter
 from util.scmurlmgr import BaseScmUrlMgr
 from util.subprocc import SubProcController
+from util.textutil import ZIP_EXT
 
 
 class ToolCommonLoader(object):
     @staticmethod
+    def is_zip_url(tool_url):
+        """判断是否是压缩包地址"""
+        if tool_url.lower().endswith(ZIP_EXT):
+            return True
+        else:
+            return False
+    @staticmethod
     def load_tool_type(tool_dirpath, tool_dirname, tool_url=None):
         """
         判断使用哪种方式加载工具
@@ -38,8 +47,10 @@ def load_tool_type(tool_dirpath, tool_dirname, tool_url=None):
         """
         if settings.USE_LOCAL_TOOL == "True":
             if os.path.exists(tool_dirpath):
-                # LogPrinter.info(f"USE_LOCAL_TOOL=True, use local tool dir: {tool_dirpath}")
+                LogPrinter.info(f"USE_LOCAL_TOOL=True, use local tool dir: {tool_dirpath}")
                 return "Local", None
+            elif tool_url and ToolCommonLoader.is_zip_url(tool_url):
+                return "HTTP", None
             else:
                 return "Git", None
         else:
@@ -54,6 +65,8 @@ def load_tool_type(tool_dirpath, tool_dirname, tool_url=None):
                         return "Copy", tool_dirpath_copy_from
                     else:  # 拷贝源目录不存在，从git拉取
                         return "Git", None
+            elif tool_url and ToolCommonLoader.is_zip_url(tool_url):
+                return "HTTP", None
             else:
                 return "Git", None
 
@@ -69,6 +82,8 @@ def load_tool(load_type, tool_dirpath, tool_dirpath_copy_from, git_url, scm_auth
         :param print_enable:
         :return:
         """
+        if settings.DEBUG:
+            print_enable = True
         if load_type == "Local":
             if print_enable:
                 LogPrinter.info(f"Use local tool dir: {tool_dirpath}")
@@ -78,6 +93,11 @@ def load_tool(load_type, tool_dirpath, tool_dirpath_copy_from, git_url, scm_auth
                 LogPrinter.info(f"Copy from {tool_dirpath_copy_from} to {tool_dirpath}")
             PathMgr().retry_copy(tool_dirpath_copy_from, tool_dirpath)
             return tool_dirpath
+        elif load_type == "HTTP":
+            if print_enable:
+                LogPrinter.info(f"Load from {git_url} to {tool_dirpath}")
+            HttpToolLoader.download_tool(git_url, tool_dirpath)
+            return tool_dirpath
         else:
             if print_enable:
                 LogPrinter.info(f"Load from git to {tool_dirpath} ...")

client/task/basic/downloader.py

@@ -14,7 +14,7 @@
 import tempfile
 
 from node.app import settings
-from task.basic.zipmgr import Zip
+from util.zipmgr import Zip
 from util.exceptions import TransferModuleError
 from util.pathlib import PathMgr
 from util.api.fileserver import RetryFileServer

client/task/transfermgr.py

@@ -17,7 +17,7 @@
 
 from task.basic.downloader import Downloader
 from node.app import settings
-from task.basic.zipmgr import Zip
+from util.zipmgr import Zip
 from util.exceptions import TransferModuleError
 from util.api.fileserver import RetryFileServer
 from util.pathlib import PathMgr

client/tool/customtool.cp37-win_amd64.pyd

@@ -12,11 +12,13 @@
 import logging
 import hashlib
 import os
+import ssl
 
 from urllib.parse import urljoin
 from util import wrapper
 from node.app import settings
 from util.api.httpclient import HttpClient
+from urllib.request import urlopen
 
 logger = logging.getLogger(__name__)
 
@@ -46,15 +48,20 @@ def get_sha256(file_path):
 
 
 class FileServer(object):
-    def __init__(self):
+    def __init__(self, server_url=None, headers=None):
         """
         构造函数
         :return:
         """
-        # 优先从环境变量读取文件服务器url和token，没有再使用默认值
-        self._server_url = os.getenv("FILE_SERVER_URL", settings.FILE_SERVER['URL'])
-        file_server_token = os.getenv("FILE_SERVER_TOKEN", settings.FILE_SERVER['TOKEN'])
-        self._headers = {'Authorization': 'Token %s' % file_server_token}
+        if server_url:
+            self._server_url = server_url
+            self._headers = headers if headers else {}
+        else:
+            # 优先从环境变量读取文件服务器url和token，没有再使用默认值
+            self._server_url = os.getenv("FILE_SERVER_URL", settings.FILE_SERVER['URL'])
+            file_server_token = os.getenv("FILE_SERVER_TOKEN", settings.FILE_SERVER['TOKEN'])
+            self._headers = {'Authorization': 'Token %s' % file_server_token}
+
         self._proxies = None
 
     def modify_save_time(self, rel_dir, days):
@@ -117,6 +124,20 @@ def download_file(self, rel_url, filepath):
             wf.write(data)
         return filepath
 
+    def download_big_file(self, rel_url, filepath):
+        """大文件下载"""
+        context = ssl._create_unverified_context()
+        download_url = urljoin(self._server_url, rel_url)
+        resp = urlopen(download_url, context=context)
+        chunk_size = 16 * 1024
+        with open(filepath, 'wb') as wf:
+            while True:
+                trunk = resp.read(chunk_size)
+                if not trunk:
+                    break
+                wf.write(trunk)
+        return filepath
+
 
 class RetryFileServer(object):
     def __init__(self, retry_times=-1):
@@ -136,10 +157,10 @@ def retry_on_error(self, error, method_name):
         """
         return
 
-    def get_server(self):
+    def get_server(self, server_url=None):
         """
         获取一个server实例
         :return:
         """
-        file_server = FileServer()
+        file_server = FileServer(server_url=server_url)
         return wrapper.Retry(server=file_server, on_error=self.retry_on_error, total=self._retry_times)