Merge pull request #16 from dataiku/bug/sc-76640-next-page-pagination-with-extractionn-path

Fix Next page pagination x extraction path

Author: alexbourret

CHANGELOG.md

@@ -1,6 +1,11 @@
 # Changelog
 
 
+## [Version 1.0.4](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.0.4) - Bugfix release - 2021-12-14
+
+- Fix use of "Next page pagination" in combination with extraction path
+- Extraction path added to recipe
+
 ## [Version 1.0.3](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.0.3) - Bugfix and feature release - 2021-11-23
 
 - Fixes error raised on HTTP 204 status codes

custom-recipes/api-connect/recipe.json

@@ -5,6 +5,7 @@
         "icon": "icon-rocket"
     },
     "kind": "PYTHON",
+    "selectableFromDataset": "input_A_role",
     "inputRoles": [
         {
             "name": "input_A_role",

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "api-connect",
-    "version": "1.0.3",
+    "version": "1.0.4",
     "meta": {
         "label": "API Connect",
         "description": "Retrieve data from any REST API",

python-lib/dku_utils.py

@@ -42,11 +42,17 @@ def parse_keys_for_json(items):
     return ret
 
 
-def get_value_from_path(dictionary, path):
+def get_value_from_path(dictionary, path, default=None, can_raise=True):
     ret = copy.deepcopy(dictionary)
     for key in path:
         if key in ret and isinstance(ret, dict):
             ret = ret.get(key)
         else:
-            raise ValueError("The extraction path {} was not found in the incoming data".format(path))
+            error_message = "The extraction path {} was not found in the incoming data".format(path)
+            if can_raise:
+                raise ValueError(error_message)
+            elif default:
+                return default  # [{"error": error_message}]
+            else:
+                return None
     return ret

python-lib/pagination.py

@@ -1,4 +1,5 @@
 from safe_logger import SafeLogger
+from dku_utils import get_value_from_path
 
 
 logger = SafeLogger("api-connect plugin Pagination")
@@ -66,8 +67,11 @@ def update_next_page(self, data, response_links=None):
                 self.is_last_batch_empty = True
             return
         elif self.counting_key:
-            batch_size = len(data.get(self.counting_key, []))
-            if batch_size == 0:
+            extracted_data = get_value_from_path(data, self.counting_key.split("."), can_raise=False)
+            if extracted_data:
+                batch_size = len(extracted_data)
+            else:
+                batch_size = 0
                 self.is_last_batch_empty = True
         else:
             batch_size = 1

python-lib/rest_api_recipe_session.py

@@ -1,7 +1,7 @@
 from dataikuapi.utils import DataikuException
 from rest_api_client import RestAPIClient
 from safe_logger import SafeLogger
-from dku_utils import parse_keys_for_json
+from dku_utils import parse_keys_for_json, get_value_from_path
 from dku_constants import DKUConstants
 import copy
 import json
@@ -23,6 +23,7 @@ def __init__(self, custom_key_values, credential_parameters, endpoint_parameters
         self.display_metadata = display_metadata
         self.maximum_number_rows = maximum_number_rows
         self.is_row_limit = (self.maximum_number_rows > 0)
+        self.can_raise = False
 
     @staticmethod
     def get_column_to_parameter_dict(parameter_columns, parameter_renamings):
@@ -68,9 +69,13 @@ def retrieve_next_page(self, is_raw_output):
         metadata = self.client.get_metadata() if self.display_metadata else {}
         is_api_returning_dict = True
         if self.extraction_key:
-            data_rows = json_response.get(self.extraction_key, [json_response])
+            data_rows = get_value_from_path(json_response, self.extraction_key.split("."), can_raise=False)
             if data_rows is None:
-                raise DataikuException("Extraction key '{}' was not found in the incoming data".format(self.extraction_key))
+                error_message = "Extraction key '{}' was not found in the incoming data".format(self.extraction_key)
+                if self.can_raise:
+                    raise DataikuException(error_message)
+                else:
+                    return [{"error": error_message}]
             page_rows.extend(self.format_page_rows(data_rows, is_raw_output, metadata))
         else:
             # Todo: check api_response key is free and add something overwise