Merge pull request #25 from dataiku/bug/sc-99081-escape-dots-w-json-path

Bug/sc 99081 escape dots using json path

Author: alexbourret

CHANGELOG.md

@@ -1,10 +1,12 @@
 # Changelog
 
 
-## [Version 1.0.7](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.0.7) - Bugfix release - 2022-08-30
+## [Version 1.1.0](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.1.0) - Feature and bugfix release - 2022-09-15
 
 - Handling Offset pagination on APIs returning an array
 - Fix throttling calculation
+- Allow dots in `Key to next request URL`
+- Add NTLM authentication
 
 ## [Version 1.0.6](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.0.6) - Feature and bugfix release - 2022-05-19
 

code-env/python/spec/requirements.txt

@@ -0,0 +1,2 @@
+jsonpath-ng==1.5.3
+requests_ntlm==1.1.0

parameter-sets/credential/parameter-set.json

@@ -38,6 +38,10 @@
                 {
                     "value": "bearer_token",
                     "label": "Bearer token"
+                },
+                {
+                    "value": "ntlm",
+                    "label": "NTLM"
                 }
             ]
         },
@@ -46,14 +50,14 @@
             "label": "User name",
             "description": "Can be reused as {{username}}",
             "type": "STRING",
-            "visibilityCondition": "model.login_type == 'basic_login'"
+            "visibilityCondition": "['basic_login', 'ntlm'].includes(model.login_type)"
         },
         {
             "name": "password",
             "label": "Password",
             "description": "Can be reused as {{password}}",
             "type": "PASSWORD",
-            "visibilityCondition": "model.login_type == 'basic_login'"
+            "visibilityCondition": "['basic_login', 'ntlm'].includes(model.login_type)"
         },
         {
             "name": "token",

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "api-connect",
-    "version": "1.0.7",
+    "version": "1.1.0",
     "meta": {
         "label": "API Connect",
         "description": "Retrieve data from any REST API",

python-lib/dku_utils.py

@@ -1,5 +1,6 @@
 import json
 import copy
+from jsonpath_ng.ext import parse
 
 
 def get_dku_key_values(endpoint_query_string):
@@ -46,7 +47,7 @@ def parse_keys_for_json(items):
 def get_value_from_path(dictionary, path, default=None, can_raise=True):
     ret = copy.deepcopy(dictionary)
     for key in path:
-        if key in ret and isinstance(ret, dict):
+        if isinstance(ret, dict) and (key in ret):
             ret = ret.get(key)
         else:
             error_message = "The extraction path {} was not found in the incoming data".format(path)
@@ -84,3 +85,12 @@ def format_template(template, **kwargs):
 def is_string(data):
     data_type = type(data).__name__
     return data_type in ["str", "unicode"]
+
+
+def extract_key_using_json_path(json_dictionary, json_path):
+    matches = parse(json_path).find(json_dictionary)
+    if matches:
+        res = matches[0].value
+        return res
+    else:
+        return None

python-lib/pagination.py

@@ -1,5 +1,5 @@
 from safe_logger import SafeLogger
-from dku_utils import get_value_from_path
+from dku_utils import get_value_from_path, extract_key_using_json_path
 
 
 logger = SafeLogger("api-connect plugin Pagination")
@@ -32,7 +32,7 @@ def configure_paging(self, config=None, skip_key=None, limit_key=None, total_key
         self.pagination_type = config.get("pagination_type", pagination_type)
         if self.pagination_type == "next_page":
             self.next_page_key = config.get("next_page_key", next_page_key)
-            self.next_page_key = None if self.next_page_key == [''] else self.next_page_key
+            self.next_page_key = None if self.next_page_key == '' else self.next_page_key
         elif self.pagination_type in ["offset", "page"]:
             self.skip_key = config.get("skip_key", skip_key)
 
@@ -80,8 +80,8 @@ def update_next_page(self, data, response_links=None):
                 self.is_last_batch_empty = True
         else:
             batch_size = 1
-        if self.next_page_key and (len(self.next_page_key) > 0):
-            self.next_page_url = self.get_from_path(data, self.next_page_key)
+        if self.next_page_key:
+            self.next_page_url = extract_key_using_json_path(data, self.next_page_key)
         if self.skip_key:
             self.skip = data.get(self.skip_key)
         if self.limit_key:
@@ -92,17 +92,6 @@ def update_next_page(self, data, response_links=None):
         if self.total:
             self.remaining_records = self.total - self.records_to_skip
 
-    def get_from_path(self, dictionary, path):
-        if isinstance(path, list):
-            edge = dictionary
-            for key in path:
-                edge = edge.get(key)
-                if edge is None:
-                    return None
-            return edge
-        else:
-            return dictionary.get(path)
-
     def has_next_page(self):
         if self.is_last_batch_empty:
             logger.info("has_next_page:last was batch empty -> False")

python-lib/rest_api_client.py

@@ -60,7 +60,7 @@ def __init__(self, credential, endpoint, custom_key_values={}):
 
         self.requests_kwargs.update({"params": self.params})
         self.pagination = Pagination()
-        next_page_url_key = endpoint.get("next_page_url_key", "").split('.')
+        next_page_url_key = endpoint.get("next_page_url_key", "")
         top_key = endpoint.get("top_key")
         skip_key = endpoint.get("skip_key")
         pagination_type = endpoint.get("pagination_type", "na")
@@ -92,10 +92,16 @@ def __init__(self, credential, endpoint, custom_key_values={}):
     def set_login(self, credential):
         login_type = credential.get("login_type", "no_auth")
         if login_type == "basic_login":
-            self.username = credential.get("username", "")
-            self.password = credential.get("password", "")
-            self.auth = (self.username, self.password)
-            self.requests_kwargs.update({"auth": self.auth})
+            username = credential.get("username", "")
+            password = credential.get("password", "")
+            auth = (username, password)
+            self.requests_kwargs.update({"auth": auth})
+        if login_type == "ntlm":
+            from requests_ntlm import HttpNtlmAuth
+            username = credential.get("username", "")
+            password = credential.get("password", "")
+            auth = HttpNtlmAuth(username, password)
+            self.requests_kwargs.update({"auth": auth})
         if login_type == "bearer_token":
             token = credential.get("token", "")
             bearer_template = credential.get("bearer_template", "Bearer {{token}}")

tests/python/integration/test_scenario.py

@@ -33,3 +33,6 @@ def test_run_api_connect_redirection(user_dss_clients):
 
 def test_run_api_connect_check_sc_84465(user_dss_clients):
     dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="CHECKSC84465")
+
+def test_run_api_connect_ntlm_authentication(user_dss_clients):
+    dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="NTLMAUTHENTICATION")