Merge branch 'main' of https://github.com/yalechen-cyw/CodeAnalysis

Author: cyw3

client/node/toolloader/loadconfig.py

@@ -161,7 +161,8 @@ def __update_tool_names(self, tool_names, task_list):
         tool_names.remove(c_name)
         return tool_names
 
-    def read_tool_config_from_ini_file(self, tool_names=None, custom_tools=None, task_list=None, config_all_tools=False, include_common=True):
+    def read_tool_config_from_ini_file(self, tool_names=None, custom_tools=None, task_list=None, config_all_tools=False,
+                                       include_common=True):
         """
         从ini文件中读取工具配置
         """
@@ -194,6 +195,7 @@ def read_tool_config_from_ini_file(self, tool_names=None, custom_tools=None, tas
                 "env_path": env_path_section,  # env_path_section中现在已经没有key为"PATH"的元素,可以作为编译工具的env_path
                 "env_value": env_value_section
             }
+            self.__format_to_fullpath(compile_config)
             config_dict["compile_config"] = compile_config
             return config_dict
         else:
@@ -236,6 +238,25 @@ def __format_config(self, tool_cfg, env_path_section, env_value_section):
         tool_cfg["env_value"] = self.__str_to_dict(tool_cfg["env_value"], env_value_section)
         tool_cfg["tool_url"] = self.__str_to_list(tool_cfg["tool_url"])
         tool_cfg["path"] = self.__str_to_list(tool_cfg["path"], uniq=False)
+        self.__format_to_fullpath(tool_cfg)
+
+    def __format_to_fullpath(self, tool_cfg):
+        # 将路径类型的环境变量（env_path）中的相对路径转换成绝对路径
+        for env_name, rel_path in tool_cfg["env_path"].items():
+            if "PATH" == env_name:  # PATH应该单独放在path字段中，如果放在env_path中，忽略，避免影响和覆盖原有PATH变量
+                continue
+            full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
+            tool_cfg["env_path"][env_name] = full_path
+
+        # 将PATH环境变量的相对路径转换成绝对路径
+        path_env = []
+        for rel_path in tool_cfg["path"]:
+            if "$" in rel_path or "%" in rel_path:  # 带变量的环境变量，已经是全路径
+                full_path = rel_path
+            else:
+                full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
+            path_env.append(full_path)
+        tool_cfg["path"] = path_env
 
     def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
         """
@@ -266,7 +287,7 @@ def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
         config_dict = {}
         already_config_tools = []  # 记录已经读取到配置的工具
         for task_config in task_list:
-            task_name = task_config["task_name"]
+            task_name = task_config.get("task_name")
             if tool_names and task_name not in tool_names:
                 continue
             task_params = task_config.get("task_params", {})
@@ -282,29 +303,40 @@ def read_config_from_tool_schemes(self, tool_names=None, task_list=None):
                     for tool_lib in tool_libs:
                         lib_support_os = tool_lib.get("os", [])
                         if self._os_type in lib_support_os:
+                            # 使用新的结构存储环境变量，避免修改原有的任务参数
+                            new_lib_envs = {}
                             lib_envs = tool_lib.get("envs", {})
                             if not lib_envs:  # 可能会传空list，这里判空，避免后面格式出错
-                                lib_envs = {}
+                                new_lib_envs = {}
+
                             scm_url = tool_lib.get("scm_url")
                             lib_dir_name = scm_url.split('/')[-1].strip().replace(".git", "")
-                            # 环境变量中的$ROOT_DIR替换为目录名，后续加载环境变量时会拼接为全路径
+                            lib_dir_path = os.path.join(settings.TOOL_BASE_DIR, lib_dir_name)
+
+                            # 将环境变量中的$ROOT_DIR替换为实际路径，重新保存到new_lib_envs
                             for key, value in lib_envs.items():
                                 if "$ROOT_DIR" in value:
-                                    lib_envs[key] = value.replace("$ROOT_DIR", lib_dir_name)
+                                    new_lib_envs[key] = value.replace("$ROOT_DIR", lib_dir_path)
+                                else:
+                                    new_lib_envs[key] = value
+
                             # PATH和其他环境变量分开处理
                             path_envs = []
                             other_envs = {}
-                            for env_name, env_value in lib_envs.items():
+                            for env_name, env_value in new_lib_envs.items():
                                 if env_name == "PATH":
                                     path_envs = self.__str_to_list(env_value)
                                 else:
-                                    other_envs[env_name] = env_value
+                                    # 根据英文分号拆分后，再环境变量分隔符拼接到一起
+                                    value_list = self.__str_to_list(env_value)
+                                    value_format = os.pathsep.join(value_list)
+                                    other_envs[env_name] = value_format
                             lib_config = {
                                 "tool_url": scm_url,
                                 "scm_type": tool_lib.get("scm_type"),
                                 "auth_info": tool_lib.get("auth_info"),
                                 "path": path_envs,
-                                "env_path": lib_envs,
+                                "env_path": other_envs,
                                 "env_value": {}
                             }
                             lib_name = tool_lib.get("name")

client/settings/base.py

@@ -95,8 +95,6 @@
 # ========================
 # 默认从Git拉取工具；如果使用本地工具，可以在config.ini中配置该值为True，将不自动拉取内置工具和配置文件
 USE_LOCAL_TOOL = False
-# 默认的工具目录，即puppy根目录下的data/tools目录
-DEFAULT_TOOL_BASE_DIR = join(TCA_BASE_DIR, "tools")
 # 扫描工具目录
 TOOL_BASE_DIR = join(TCA_BASE_DIR, "tools")
 # 扫描工具配置文件地址，需要在config.ini中配置

client/util/envset.py

@@ -14,8 +14,6 @@
 import logging
 import platform
 
-from node.app import settings
-from node.toolloader.loadconfig import ConfigLoader
 from util.textutil import StringMgr
 
 logger = logging.getLogger(__name__)
@@ -67,6 +65,7 @@ def __update_os_environ(self, envs, task_mode = True):
                 os.environ.update(new_envs)
         else:
             for env_name, env_value in envs.items():
+                logger.debug(f">>> set env {env_name}={os.path.expandvars(env_value)}")
                 os.environ.update({env_name: os.path.expandvars(env_value)})
 
     def set_tool_env(self, tool_config):
@@ -88,17 +87,13 @@ def set_tool_env(self, tool_config):
         """
         tool_envs = {}
 
-        path_evn = []
+        path_env = []
         for tool_name, tool_params in tool_config.items():
             # 添加路径类型的环境变量
-            for env_name, rel_path in tool_params["env_path"].items():
+            for env_name, full_path in tool_params["env_path"].items():
                 if "PATH" == env_name:  # PATH应该单独放在path字段中，如果放在env_path中，忽略，避免影响和覆盖原有PATH变量
                     continue
                 if env_name not in tool_envs:
-                    full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
-                    # 如果工具目录不存在，重定向到默认的工具目录下
-                    if not os.path.exists(full_path):
-                        full_path = os.path.join(settings.DEFAULT_TOOL_BASE_DIR, rel_path)
                     tool_envs[env_name] = full_path
 
             # 添加值类型的环境变量
@@ -108,21 +103,17 @@ def set_tool_env(self, tool_config):
 
             # 将PATH的环境变量放到一个list中
             for rel_path in tool_params["path"]:
-                if rel_path not in path_evn:
-                    path_evn.append(rel_path)
+                if rel_path not in path_env:
+                    path_env.append(rel_path)
 
         # 添加PATH环境变量
-        path_str = os.environ["PATH"]   # 先读取系统PATH环境变量
-        path_list = path_str.split(os.pathsep)
-        for rel_path in path_evn:
-            if "$" in rel_path or "%" in rel_path:  # 带变量的环境变量，已经是全路径
-                full_path = rel_path
-            else:
-                full_path = os.path.join(settings.TOOL_BASE_DIR, rel_path)
-                # 如果工具目录不存在，重定向到默认的工具目录下
-                if not os.path.exists(full_path):
-                    full_path = os.path.join(settings.DEFAULT_TOOL_BASE_DIR, rel_path)
-            # if absolute_path not in path_list:  # path不要去重,会影响加载顺序
+        path_str = os.getenv("PATH")    # 先读取系统PATH环境变量
+        if path_str:
+            path_list = path_str.split(os.pathsep)
+        else:  # 增加判空，防止读取不到的情况
+            path_list = []
+        for full_path in path_env:
+            # path不要去重,会影响加载顺序
             path_list = [full_path] + path_list  # 后添加的放在前面,优先搜索
         tool_envs["PATH"] = os.pathsep.join(path_list)
 
@@ -156,6 +147,7 @@ def set_task_env(self, task_params):
             if task_envs:
                 # 设置到进程环境变量中
                 self.__update_os_environ(task_envs, True)
+                # logger.debug('设置任务环境变量::\n%s' % '\n'.join(['%s=%s' % (key, value) for key, value in task_envs.items()]))
         EnvSetting.env_setting_init()
 
     def get_origin_env(self, os_env=None):
@@ -175,17 +167,31 @@ def get_origin_env(self, os_env=None):
             env = os_env
         else:
             env = dict(os.environ)
+
         # 判断是否是pyinstaller包状态, 若是源码状态，则不处理
         if len(sys.argv) > 0 and sys.argv[0].endswith(".py"):
             return env
+
         # Linux
         if sys.platform == "linux" or sys.platform == "linux2":
             lp_key = 'LD_LIBRARY_PATH'
-            lp_orig = env.get(lp_key + '_ORIG')
-            if lp_orig is not None:
-                env[lp_key] = lp_orig
-            else:
-                env.pop(lp_key, None)
+            # 收集需要的lp环境变量
+            wanted_env_list = []
+            # 收集现有进程的lp环境变量，剔除掉pyinstaller添加的临时运行目录，则为需要传递给子进程的环境变量
+            cur_lp_value = env.get(lp_key)
+            if cur_lp_value:
+                cur_lp_list = cur_lp_value.split(os.pathsep)
+                for lp_value in cur_lp_list:
+                    # 当前进程的lp环境变量，剔除掉pyinstaller添加的临时运行目录，其他的为需要的
+                    if not lp_value.startswith("/tmp/_MEI"):
+                        wanted_env_list.append(lp_value)
+                if wanted_env_list:  # 现有的lp已经包含_ORIG
+                    env[lp_key] = os.pathsep.join(wanted_env_list)
+                    logger.debug(f"{lp_key}={cur_lp_value}, change to {env[lp_key]}")
+                else:
+                    env.pop(lp_key, None)
+                    logger.debug(f"{lp_key}={cur_lp_value}, delete it.")
+
         # Mac:
         # 由于Mac下二进制包无法检测环境中环境变量原来是否有DYLD_LIBRARY_PATH，加上目前没有发现有项目遇到类似问题
         # 这里暂不做处理，继续观察