Merge pull request #47 from dataiku/feature/sc-222037-dump-non-decodable-returned-content

[sc-222037] dump non decodable returned content

Author: alexbourret

CHANGELOG.md

@@ -1,8 +1,11 @@
 # Changelog
 
-## [Version 1.2.3](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.3) - Feature release - 2024-11-25
+## [Version 1.2.3](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.3) - Feature and bugfix release - 2024-11-25
 
 - Fix xml decoding for content type application/atom+xml
+- Dump returned content that can't be decoded
+- Fix: Empty cells in the recipe's input dataset now produce an empty string instead of a 'nan' string
+- Fix: UTF-8 encoding of raw mode body
 
 ## [Version 1.2.2](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.2) - Feature release - 2024-02-14
 

python-connectors/api-connect_dataset/connector.py

@@ -4,7 +4,8 @@
 from rest_api_client import RestAPIClient
 from dku_utils import (
     get_dku_key_values, get_endpoint_parameters,
-    parse_keys_for_json, get_value_from_path, get_secure_credentials, decode_csv_data
+    parse_keys_for_json, get_value_from_path, get_secure_credentials,
+    decode_csv_data, decode_bytes
 )
 from dku_constants import DKUConstants
 import json
@@ -57,10 +58,16 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
                 record_count += 1
                 yield self.format_output(data, metadata)
             else:
-                data = decode_csv_data(data)
-                record_count += len(data)
-                for row in data:
-                    yield self.format_output(row, metadata)
+                csv_data = decode_csv_data(data)
+                if csv_data:
+                    record_count += len(csv_data)
+                    for row in csv_data:
+                        yield self.format_output(row, metadata)
+                else:
+                    record_count += 1
+                    yield {
+                        DKUConstants.API_RESPONSE_KEY: "{}".format(decode_bytes(data))
+                    }
             if is_records_limit and record_count >= records_limit:
                 break
 

python-lib/dku_constants.py

@@ -2,6 +2,6 @@ class DKUConstants(object):
     API_RESPONSE_KEY = "api_response"
     FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"]
     FORM_DATA_BODY_FORMAT = "FORM_DATA"
-    PLUGIN_VERSION = "1.2.3-beta.1"
+    PLUGIN_VERSION = "1.2.3-beta.5"
     RAW_BODY_FORMAT = "RAW"
     REPONSE_ERROR_KEY = "dku_error"

python-lib/dku_utils.py

@@ -1,5 +1,6 @@
 import json
 import copy
+import math
 from jsonpath_ng.ext import parse
 from safe_logger import SafeLogger
 
@@ -145,12 +146,24 @@ def decode_csv_data(data):
     import csv
     import io
     json_data = None
-    if isinstance(data, bytes):
-        data = data.decode("utf-8")
+    data = decode_bytes(data)
     try:
         reader = csv.DictReader(io.StringIO(data))
         json_data = list(reader)
     except Exception as error:
         logger.error("Could not extract csv data. Error={}".format(error))
         json_data = data
     return json_data
+
+
+def de_NaN(cell_content):
+    if isinstance(cell_content, float):
+        if math.isnan(cell_content):
+            return ''
+    return cell_content
+
+
+def decode_bytes(content):
+    if isinstance(content, bytes):
+        content = content.decode()
+    return content

python-lib/rest_api_client.py

@@ -128,6 +128,10 @@ def request(self, method, url, can_raise_exeption=True, **kwargs):
         error_message = None
         status_code = None
         response_headers = None
+        if "data" in kwargs:
+            data = kwargs.get("data")
+            if isinstance(data, str):
+                kwargs["data"] = data.encode("utf-8")
         try:
             response = self.request_with_redirect_retry(method, url, **kwargs)
             status_code = response.status_code

python-lib/rest_api_recipe_session.py

@@ -1,7 +1,7 @@
 from dataikuapi.utils import DataikuException
 from rest_api_client import RestAPIClient
 from safe_logger import SafeLogger
-from dku_utils import parse_keys_for_json, get_value_from_path, decode_csv_data
+from dku_utils import parse_keys_for_json, get_value_from_path, decode_csv_data, de_NaN
 from dku_constants import DKUConstants
 import copy
 import json
@@ -49,7 +49,11 @@ def process_dataframe(self, input_parameters_dataframe, is_raw_output):
             self.initial_parameter_columns = {}
             for column_name in self.column_to_parameter_dict:
                 parameter_name = self.column_to_parameter_dict[column_name]
-                self.initial_parameter_columns.update({parameter_name: input_parameters_row.get(column_name)})
+                self.initial_parameter_columns.update(
+                    {
+                        parameter_name: de_NaN(input_parameters_row.get(column_name))
+                    }
+                )
             updated_endpoint_parameters = copy.deepcopy(self.endpoint_parameters)
             updated_endpoint_parameters.update(self.initial_parameter_columns)
             logger.info("Processing row #{}, creating client with credential={}, updated_endpoint={}, custom_key_values={}".format(