Merge pull request #27 from dataiku/feature/sc-101330-add-base-url-to-next-record-url

Feature/sc 101330 add base url to next record url

Author: alexbourret

custom-recipes/api-connect/recipe.json

@@ -193,6 +193,22 @@
             "defaultValue": null,
             "visibilityCondition": "model.pagination_type=='next_page'"
         },
+        {
+            "name": "is_next_page_url_relative",
+            "label": " ",
+            "description": "Next page URL is relative",
+            "type": "BOOLEAN",
+            "defaultValue": false,
+            "visibilityCondition": "model.pagination_type=='next_page'"
+        },
+        {
+            "name": "next_page_url_base",
+            "label": "Base URL to next page",
+            "description": "https://mysite.com/bla/",
+            "type": "STRING",
+            "defaultValue": null,
+            "visibilityCondition": "model.pagination_type=='next_page' && (model.is_next_page_url_relative==true)"
+        },
         {
             "name": "top_key",
             "label": "Key limiting elements per page",

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "api-connect",
-    "version": "1.1.0",
+    "version": "1.1.1",
     "meta": {
         "label": "API Connect",
         "description": "Retrieve data from any REST API",

python-connectors/api-connect_dataset/connector.json

@@ -152,6 +152,22 @@
             "defaultValue": null,
             "visibilityCondition": "model.pagination_type=='next_page'"
         },
+        {
+            "name": "is_next_page_url_relative",
+            "label": " ",
+            "description": "Next page URL is relative",
+            "type": "BOOLEAN",
+            "defaultValue": false,
+            "visibilityCondition": "model.pagination_type=='next_page'"
+        },
+        {
+            "name": "next_page_url_base",
+            "label": "Base URL to next page",
+            "description": "https://mysite.com/bla/",
+            "type": "STRING",
+            "defaultValue": null,
+            "visibilityCondition": "model.pagination_type=='next_page' && (model.is_next_page_url_relative==true)"
+        },
         {
             "name": "top_key",
             "label": "Key limiting elements per page",

python-lib/dku_utils.py

@@ -24,7 +24,7 @@ def get_endpoint_parameters(configuration):
         "timeout",
         "requests_per_minute",
         "pagination_type",
-        "next_page_url_key",
+        "next_page_url_key", "is_next_page_url_relative", "next_page_url_base",
         "top_key", "skip_key", "maximum_number_rows"
     ]
     parameters = {endpoint_parameter: configuration.get(endpoint_parameter) for endpoint_parameter in endpoint_parameters if configuration.get(endpoint_parameter) is not None}

python-lib/pagination.py

@@ -9,6 +9,7 @@ class Pagination(object):
 
     def __init__(self, config=None, skip_key=None, limit_key=None, total_key=None, next_page_key=None):
         self.next_page_key = None
+        self.next_page_url_base = None
         self.skip_key = None
         self.limit_key = None
         self.total_key = None
@@ -26,13 +27,16 @@ def __init__(self, config=None, skip_key=None, limit_key=None, total_key=None, n
         self.params_must_be_blanked = False
         self.data_is_list = None
 
-    def configure_paging(self, config=None, skip_key=None, limit_key=None, total_key=None, next_page_key=None, url=None, pagination_type="na"):
+    def configure_paging(self, config=None, skip_key=None, limit_key=None, total_key=None, next_page_key=None, next_page_url_base=None, url=None, pagination_type="na"):
         config = {} if config is None else config
         self.limit_key = config.get("limit_key", limit_key)
         self.pagination_type = config.get("pagination_type", pagination_type)
         if self.pagination_type == "next_page":
             self.next_page_key = config.get("next_page_key", next_page_key)
             self.next_page_key = None if self.next_page_key == '' else self.next_page_key
+            if next_page_url_base:
+                next_page_url_base = next_page_url_base.strip('/')
+            self.next_page_url_base = next_page_url_base
         elif self.pagination_type in ["offset", "page"]:
             self.skip_key = config.get("skip_key", skip_key)
 
@@ -81,7 +85,11 @@ def update_next_page(self, data, response_links=None):
         else:
             batch_size = 1
         if self.next_page_key:
-            self.next_page_url = extract_key_using_json_path(data, self.next_page_key)
+            next_page_path = extract_key_using_json_path(data, self.next_page_key)
+            if self.next_page_url_base and next_page_path:
+                self.next_page_url = "/".join([self.next_page_url_base, next_page_path])
+            else:
+                self.next_page_url = next_page_path
         if self.skip_key:
             self.skip = data.get(self.skip_key)
         if self.limit_key:

python-lib/rest_api_client.py

@@ -61,13 +61,17 @@ def __init__(self, credential, endpoint, custom_key_values={}):
         self.requests_kwargs.update({"params": self.params})
         self.pagination = Pagination()
         next_page_url_key = endpoint.get("next_page_url_key", "")
+        is_next_page_url_relative = endpoint.get("is_next_page_url_relative", False)
+        next_page_url_base = endpoint.get("next_page_url_base", None) if is_next_page_url_relative else None
+        next_page_url_base = format_template(next_page_url_base, **self.presets_variables)
         top_key = endpoint.get("top_key")
         skip_key = endpoint.get("skip_key")
         pagination_type = endpoint.get("pagination_type", "na")
         self.pagination.configure_paging(
             skip_key=skip_key,
             limit_key=top_key,
             next_page_key=next_page_url_key,
+            next_page_url_base=next_page_url_base,
             url=self.endpoint_url,
             pagination_type=pagination_type
         )

tests/python/integration/test_scenario.py

@@ -34,5 +34,10 @@ def test_run_api_connect_redirection(user_dss_clients):
 def test_run_api_connect_check_sc_84465(user_dss_clients):
     dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="CHECKSC84465")
 
+
 def test_run_api_connect_ntlm_authentication(user_dss_clients):
     dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="NTLMAUTHENTICATION")
+
+
+def test_run_api_connect_relative_url_pagination(user_dss_clients):
+    dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="RELATIVEURLPAGINATION")