param search helper function

Author: katzy687

package/cloudshell/iac/terraform/downloaders/gitlab_downloader.py

@@ -1,9 +1,11 @@
 import re
 from dataclasses import dataclass
+from typing import List
 from urllib.error import HTTPError, URLError
 from retry import retry
 from cloudshell.iac.terraform.downloaders.base_git_downloader import GitScriptDownloaderBase
 from cloudshell.iac.terraform.services.gitlab_api_handler import GitlabApiHandler
+from urllib.parse import unquote
 
 
 @dataclass
@@ -16,7 +18,7 @@ class CommonGitLabUrlData:
 
 
 @dataclass
-class GitLabRawUrlData(CommonGitLabUrlData):
+class GitLabBrowserUrlData(CommonGitLabUrlData):
     gitlab_user: str
     project_name: str
 
@@ -28,7 +30,7 @@ class GitLabApiUrlData(CommonGitLabUrlData):
     api_endpoint: str
 
 
-def extract_data_from_raw_url(url) -> GitLabRawUrlData:
+def extract_data_from_browser_url(url) -> GitLabBrowserUrlData:
     """
     Take api style url and extract data
     Sample Raw Browser url: "http://192.168.85.26/quali_natti/terraformstuff/-/tree/test-branch/rds/project1"
@@ -42,13 +44,24 @@ def extract_data_from_raw_url(url) -> GitLabRawUrlData:
         raise ValueError(f"No GitLab URL Data found in RAW url '{url}'")
 
     groups = match.groupdict()
-    return GitLabRawUrlData(protocol=groups['protocol'],
-                            domain=groups['domain'],
-                            gitlab_user=groups['user'],
-                            project_name=groups['project'],
-                            sha=groups['sha'],
-                            path=groups['path'],
-                            full_url=url)
+    return GitLabBrowserUrlData(protocol=groups['protocol'],
+                                domain=groups['domain'],
+                                gitlab_user=groups['user'],
+                                project_name=groups['project'],
+                                sha=groups['sha'],
+                                path=groups['path'],
+                                full_url=url)
+
+
+def get_query_param_val(param_key: str, params_list: List[List[str]]) -> str:
+    """
+    look for target param in 2D list of key pair values
+    [[k1,v1],[k2,v2]]
+    if not found return empty string
+    """
+    target_param_search = [x for x in params_list if x[0] == param_key]
+    param_val = target_param_search[0][1] if target_param_search else ""
+    return param_val
 
 
 def extract_data_from_api_url(url) -> GitLabApiUrlData:
@@ -68,26 +81,23 @@ def extract_data_from_api_url(url) -> GitLabApiUrlData:
     groups = match.groupdict()
     query_params = groups['params']
 
-    # remove the leading '?'
+    # remove the leading '?' of the query  param string
     query_params = query_params.split("?")[-1]
 
     # split into 2D list [[k1,v1],[k2,v2]]
     params_list = [x.split("=") for x in query_params.split("&")]
 
     # search for target params
-    path_param_search = [x for x in params_list if x[0] == "path"]
-    sha_param_search = [x for x in params_list if x[0] == "sha"]
-    ref_param_search = [x for x in params_list if x[0] == "ref"]
-    sha = sha_param_search[0][1] if sha_param_search else ""
-    path = path_param_search[0][1] if path_param_search else ""
-    ref = ref_param_search[0][1] if ref_param_search else ""
+    path = get_query_param_val("path", params_list)
+    sha = get_query_param_val("sha", params_list)
+    ref = get_query_param_val("ref", params_list)
 
     # take sha param if passed, otherwise use the ref
     sha = sha if sha else ref
 
-    # url encoded path not necessary for archive api
-    path = path.replace("%2F", "/").replace("%2D", "-")
-    sha = sha.replace("%2D", "-")
+    # url encoded path not necessary
+    path = unquote(path)
+    sha = unquote(sha)
     return GitLabApiUrlData(protocol=groups['protocol'],
                             domain=groups['domain'],
                             api_version=groups['api_version'],
@@ -127,7 +137,7 @@ def download_repo(self, url: str, token: str, branch: str = "") -> str:
         if is_api_url:
             url_data = extract_data_from_api_url(url)
         else:
-            url_data = extract_data_from_raw_url(url)
+            url_data = extract_data_from_browser_url(url)
 
         # allow service branch attr to override the url defined sha
         sha = branch if branch else url_data.sha

package/tests/integration_tests/test_gitlab_downloader.py

@@ -20,7 +20,7 @@ def test_api_url_download(self):
         self.working_dir = working_dir
         assert working_dir
 
-    def test_raw_url_download(self):
+    def test_browser_url_download(self):
         working_dir = self.downloader.download_repo(url=self.gitlab_vars.natural_url,
                                                     token=self.gitlab_vars.token,
                                                     branch=self.gitlab_vars.branch)

package/tests/unit_tests/test_gitlab_url_extractor.py

@@ -10,11 +10,11 @@ class TestGitlabUrlExtractor(unittest.TestCase):
     api style url:
     http://<domain>/api/v4/projects/<project_id>/repository/archive.zip?path=<path>=<branch>
     """
-    RAW_URL = "http://192.168.85.26/quali_natti/terraformstuff/-/tree/test-branch/parent-dir/hello-world"
+    BROWSER_URL = "http://192.168.85.26/quali_natti/terraformstuff/-/tree/test-branch/parent-dir/hello-world"
     API_URL = "http://192.168.85.26/api/v4/projects/2/repository/archive.zip?path=parent%2Ddir%2Fhello%2Dworld&sha=test%2Dbranch"
 
-    def test_raw_url(self):
-        url_data = gitlab_downloader.extract_data_from_raw_url(self.RAW_URL)
+    def test_browser_url(self):
+        url_data = gitlab_downloader.extract_data_from_browser_url(self.BROWSER_URL)
         assert url_data
 
     def test_api_url(self):
@@ -23,16 +23,16 @@ def test_api_url(self):
 
     def test_api_url_validate(self):
         assert gitlab_downloader.is_gitlab_api_url(self.API_URL)
-        assert not gitlab_downloader.is_gitlab_api_url(self.RAW_URL)
+        assert not gitlab_downloader.is_gitlab_api_url(self.BROWSER_URL)
 
-    def test_raw_vs_url(self):
-        raw_data = gitlab_downloader.extract_data_from_raw_url(self.RAW_URL)
+    def test_browser_vs_api_url_data(self):
+        browser_data = gitlab_downloader.extract_data_from_browser_url(self.BROWSER_URL)
         api_data = gitlab_downloader.extract_data_from_api_url(self.API_URL)
-        self.assertEqual(raw_data.domain, api_data.domain)
-        self.assertEqual(raw_data.path, api_data.path)
-        self.assertEqual(raw_data.branch, api_data.sha)
+        self.assertEqual(browser_data.domain, api_data.domain)
+        self.assertEqual(browser_data.path, api_data.path)
+        self.assertEqual(browser_data.sha, api_data.sha)
 
     def test_raises(self):
         url_arg = "http://www.google.com"
         self.assertRaises(ValueError, gitlab_downloader.extract_data_from_api_url, url_arg)
-        self.assertRaises(ValueError, gitlab_downloader.extract_data_from_raw_url, url_arg)
+        self.assertRaises(ValueError, gitlab_downloader.extract_data_from_browser_url, url_arg)

package/tests/unit_tests/test_query_param_search.py

@@ -0,0 +1,16 @@
+import unittest
+from cloudshell.iac.terraform.downloaders.gitlab_downloader import get_query_param_val
+
+
+class TestQueryParamSearch(unittest.TestCase):
+    QUERY_PARAMS = [["path", "parentdir/rds"], ["sha", "abcdefg"]]
+
+    def test_param_search(self):
+        path = get_query_param_val("path", self.QUERY_PARAMS)
+        sha = get_query_param_val("sha", self.QUERY_PARAMS)
+        self.assertEqual(path, "parentdir/rds")
+        self.assertEqual(sha, "abcdefg")
+
+    def test_not_found(self):
+        ref = get_query_param_val("ref", self.QUERY_PARAMS)
+        self.assertEqual(ref, "")