Fix for [sc-99081] using jon path

Author: alexbourret

code-env/python/spec/requirements.txt

@@ -0,0 +1 @@
+jsonpath_ng

python-lib/dku_utils.py

@@ -1,5 +1,6 @@
 import json
 import copy
+from jsonpath_ng.ext import parse
 
 
 def get_dku_key_values(endpoint_query_string):
@@ -84,3 +85,12 @@ def format_template(template, **kwargs):
 def is_string(data):
     data_type = type(data).__name__
     return data_type in ["str", "unicode"]
+
+
+def extract_key_using_json_path(json_dictionary, json_path):
+    matches = parse(json_path).find(json_dictionary)
+    if matches:
+        res = matches[0].value
+        return res
+    else:
+        return None

python-lib/pagination.py

@@ -1,5 +1,5 @@
 from safe_logger import SafeLogger
-from dku_utils import get_value_from_path
+from dku_utils import get_value_from_path, extract_key_using_json_path
 
 
 logger = SafeLogger("api-connect plugin Pagination")
@@ -32,7 +32,7 @@ def configure_paging(self, config=None, skip_key=None, limit_key=None, total_key
         self.pagination_type = config.get("pagination_type", pagination_type)
         if self.pagination_type == "next_page":
             self.next_page_key = config.get("next_page_key", next_page_key)
-            self.next_page_key = None if self.next_page_key == [''] else self.next_page_key
+            self.next_page_key = None if self.next_page_key == '' else self.next_page_key
         elif self.pagination_type in ["offset", "page"]:
             self.skip_key = config.get("skip_key", skip_key)
 
@@ -80,8 +80,8 @@ def update_next_page(self, data, response_links=None):
                 self.is_last_batch_empty = True
         else:
             batch_size = 1
-        if self.next_page_key and (len(self.next_page_key) > 0):
-            self.next_page_url = self.get_from_path(data, self.next_page_key)
+        if self.next_page_key:
+            self.next_page_url = extract_key_using_json_path(data, self.next_page_key)
         if self.skip_key:
             self.skip = data.get(self.skip_key)
         if self.limit_key:
@@ -92,17 +92,6 @@ def update_next_page(self, data, response_links=None):
         if self.total:
             self.remaining_records = self.total - self.records_to_skip
 
-    def get_from_path(self, dictionary, path):
-        if isinstance(path, list):
-            edge = dictionary
-            for key in path:
-                edge = edge.get(key)
-                if edge is None:
-                    return None
-            return edge
-        else:
-            return dictionary.get(path)
-
     def has_next_page(self):
         if self.is_last_batch_empty:
             logger.info("has_next_page:last was batch empty -> False")

python-lib/rest_api_client.py

@@ -60,7 +60,7 @@ def __init__(self, credential, endpoint, custom_key_values={}):
 
         self.requests_kwargs.update({"params": self.params})
         self.pagination = Pagination()
-        next_page_url_key = endpoint.get("next_page_url_key", "").split('.')
+        next_page_url_key = endpoint.get("next_page_url_key", "")
         top_key = endpoint.get("top_key")
         skip_key = endpoint.get("skip_key")
         pagination_type = endpoint.get("pagination_type", "na")