Merge pull request #52 from dataiku/feature/add-rss-decoding

Feature/add rss decoding

Author: alexbourret

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [Version 1.2.4](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.4) - Feature and bugfix release - 2025-02-18
+
+- Fix xml decoding for content type application/rss+xml
+- Let use the *Other credentials* stored in the user's profile
+- Prevent recipe from inferring its input data types
+
 ## [Version 1.2.3](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.3) - Feature and bugfix release - 2024-11-25
 
 - Fix xml decoding for content type application/atom+xml

custom-recipes/api-connect/recipe.json

@@ -65,6 +65,13 @@
             "parameterSetId": "secure-basic",
             "visibilityCondition": "model.auth_type == 'secure_basic'"
         },
+        {
+            "name": "should_use_user_secrets",
+            "label": "User 'Other credentials'",
+            "description": "If checked, secrets under Profile > My account > Other credentials, are available to use as {{variables}}",
+            "type": "BOOLEAN",
+            "defaultValue": false
+        },
         {
             "type": "SEPARATOR",
             "label": "API call parameters"

custom-recipes/api-connect/recipe.py

@@ -3,7 +3,7 @@
 from dataiku.customrecipe import get_input_names_for_role, get_recipe_config, get_output_names_for_role
 import pandas as pd
 from safe_logger import SafeLogger
-from dku_utils import get_dku_key_values, get_endpoint_parameters, get_secure_credentials
+from dku_utils import get_dku_key_values, get_endpoint_parameters, get_secure_credentials, get_user_secrets
 from rest_api_recipe_session import RestApiRecipeSession
 from dku_constants import DKUConstants
 
@@ -45,12 +45,14 @@ def get_partitioning_keys(id_list, dku_flow_variables):
     raise ValueError("There is no parameter column selected.")
 parameter_renamings = get_dku_key_values(config.get("parameter_renamings", {}))
 custom_key_values = get_dku_key_values(config.get("custom_key_values", {}))
+user_secrets = get_user_secrets(config)
+custom_key_values.update(user_secrets)
 display_metadata = config.get("display_metadata", False)
 maximum_number_rows = config.get("maximum_number_rows", -1)
 input_parameters_dataset = dataiku.Dataset(input_A_names[0])
 partitioning_keys = get_partitioning_keys(input_parameters_dataset, dku_flow_variables)
 custom_key_values.update(partitioning_keys)
-input_parameters_dataframe = input_parameters_dataset.get_dataframe()
+input_parameters_dataframe = input_parameters_dataset.get_dataframe(infer_with_pandas=False)
 
 recipe_session = RestApiRecipeSession(
     custom_key_values,

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "api-connect",
-    "version": "1.2.3",
+    "version": "1.2.4",
     "meta": {
         "label": "API Connect",
         "description": "Retrieve data from any REST API",

python-connectors/api-connect_dataset/connector.json

@@ -44,6 +44,13 @@
             "parameterSetId": "secure-basic",
             "visibilityCondition": "model.auth_type == 'secure_basic'"
         },
+        {
+            "name": "should_use_user_secrets",
+            "label": " ",
+            "description": "Use profile's 'other credentials'",
+            "type": "BOOLEAN",
+            "defaultValue": false
+        },
         {
             "type": "SEPARATOR",
             "label": "API call parameters"

python-connectors/api-connect_dataset/connector.py

@@ -5,7 +5,7 @@
 from dku_utils import (
     get_dku_key_values, get_endpoint_parameters,
     parse_keys_for_json, get_value_from_path, get_secure_credentials,
-    decode_csv_data, decode_bytes
+    decode_csv_data, decode_bytes, get_user_secrets
 )
 from dku_constants import DKUConstants
 import json
@@ -24,6 +24,8 @@ def __init__(self, config, plugin_config):
         secure_credentials = get_secure_credentials(config)
         credential = config.get("credential", {})
         custom_key_values = get_dku_key_values(config.get("custom_key_values", {}))
+        user_secrets = get_user_secrets(config)
+        custom_key_values.update(user_secrets)
         self.client = RestAPIClient(credential, secure_credentials, endpoint_parameters, custom_key_values)
         extraction_key = endpoint_parameters.get("extraction_key", None)
         self.extraction_key = extraction_key or ''

python-lib/dku_constants.py

@@ -2,6 +2,6 @@ class DKUConstants(object):
     API_RESPONSE_KEY = "api_response"
     FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"]
     FORM_DATA_BODY_FORMAT = "FORM_DATA"
-    PLUGIN_VERSION = "1.2.3"
+    PLUGIN_VERSION = "1.2.4-beta.2"
     RAW_BODY_FORMAT = "RAW"
     REPONSE_ERROR_KEY = "dku_error"

python-lib/dku_utils.py

@@ -126,7 +126,7 @@ def extract_key_using_json_path(json_dictionary, json_path):
 def is_reponse_xml(response):
     content_types = response.headers.get("Content-Type", "").split(";")
     for content_type in content_types:
-        if content_type in ["text/xml", "application/soap+xml", "application/xml", "application/atom+xml"]:
+        if content_type in ["text/xml", "application/soap+xml", "application/xml", "application/atom+xml", "application/rss+xml"]:
             return True
     return False
 
@@ -167,3 +167,19 @@ def decode_bytes(content):
     if isinstance(content, bytes):
         content = content.decode()
     return content
+
+
+def get_user_secrets(configuration):
+    should_use_user_secrets = configuration.get("should_use_user_secrets", False)
+    if should_use_user_secrets:
+        import dataiku
+        logger.info("Using user's secrets:")
+        client = dataiku.api_client()
+        auth_info = client.get_auth_info(with_secrets=True)
+        secrets = auth_info.get("secrets", [])
+        user_secrets = {}
+        for secret in secrets:
+            logger.info("\t-'{}'".format(secret.get("key")))
+            user_secrets[secret.get("key")] = secret.get("value")
+        return user_secrets
+    return {}