keep dku_error and api_response columns in recipe output schema

Author: alexbourret

python-lib/dku_constants.py

@@ -4,3 +4,4 @@ class DKUConstants(object):
     FORM_DATA_BODY_FORMAT = "FORM_DATA"
     PLUGIN_VERSION = "1.2.0"
     RAW_BODY_FORMAT = "RAW"
+    REPONSE_ERROR_KEY = "dku_error"

python-lib/rest_api_client.py

@@ -89,7 +89,7 @@ def __init__(self, credential, endpoint, custom_key_values={}, session=None):
         elif body_format in [DKUConstants.FORM_DATA_BODY_FORMAT]:
             key_value_body = endpoint.get("key_value_body", {})
             self.requests_kwargs.update({"json": get_dku_key_values(key_value_body)})
-        self.metadata = {}
+        self.metadata = {DKUConstants.REPONSE_ERROR_KEY: None}
         self.call_number = 0
         self.session = session or requests.Session()
 
@@ -147,15 +147,15 @@ def request(self, method, url, can_raise_exeption=True, **kwargs):
         self.set_metadata("response_headers", "{}".format(response.headers))
 
         if error_message:
-            return {"error": error_message}
+            return {DKUConstants.REPONSE_ERROR_KEY: error_message}
 
         if response.status_code >= 400:
             error_message = "Error {}: {}".format(response.status_code, response.content)
             self.pagination.is_last_batch_empty = True
             if can_raise_exeption:
                 raise RestAPIClientError(error_message)
             else:
-                return {"error": error_message}
+                return {DKUConstants.REPONSE_ERROR_KEY: error_message}
         if response.status_code in [204]:
             self.pagination.update_next_page({}, response.links)
             return self.empty_json_response()
@@ -168,7 +168,7 @@ def request(self, method, url, can_raise_exeption=True, **kwargs):
             logger.error("response.content={}".format(response.content))
             if can_raise_exeption:
                 raise RestAPIClientError("The API did not return JSON as expected. {}".format(error_message))
-            return {"error": error_message}
+            return {DKUConstants.REPONSE_ERROR_KEY: error_message}
 
         self.pagination.update_next_page(json_response, response.links)
         return json_response

python-lib/rest_api_recipe_session.py

@@ -70,7 +70,7 @@ def retrieve_next_page(self, is_raw_output):
         page_rows = []
         logger.info("retrieve_next_page: Calling next page")
         json_response = self.client.paginated_api_call(can_raise_exeption=False)
-        metadata = self.client.get_metadata() if self.display_metadata else {}
+        metadata = self.client.get_metadata() if self.display_metadata else {DKUConstants.REPONSE_ERROR_KEY: None}
         is_api_returning_dict = True
         if self.extraction_key:
             data_rows = get_value_from_path(json_response, self.extraction_key.split("."), can_raise=False)
@@ -79,7 +79,7 @@ def retrieve_next_page(self, is_raw_output):
                 if self.can_raise:
                     raise DataikuException(error_message)
                 else:
-                    return self.format_page_rows([{"error": error_message}], is_raw_output, metadata)
+                    return self.format_page_rows([{DKUConstants.REPONSE_ERROR_KEY: error_message}], is_raw_output, metadata)
             page_rows.extend(self.format_page_rows(data_rows, is_raw_output, metadata))
         else:
             # Todo: check api_response key is free and add something overwise
@@ -115,6 +115,9 @@ def format_page_rows(self, data_rows, is_raw_output, metadata=None):
             base_row.update(metadata)
             if is_raw_output:
                 if is_error_message(data_row):
+                    base_row.update({
+                        DKUConstants.API_RESPONSE_KEY: None
+                    })
                     base_row.update(parse_keys_for_json(data_row))
                 else:
                     base_row.update({
@@ -129,7 +132,7 @@ def format_page_rows(self, data_rows, is_raw_output, metadata=None):
 def is_error_message(jsons_response):
     if type(jsons_response) not in [dict, list]:
         return False
-    if "error" in jsons_response and len(jsons_response) == 1:
+    if DKUConstants.REPONSE_ERROR_KEY in jsons_response and len(jsons_response) == 1:
         return True
     else:
         return False