Skip to content

Commit 7306d37

Browse files
alexbourretalexbourret
authored
Merge pull request #33 from dataiku/feature/handle-xml-n-csv
Feature/ Adds xml and csv parsing
2 parents 0de270a + 1bb9bcb commit 7306d37

15 files changed

Lines changed: 411 additions & 66 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
# Changelog
22

3+
## [Version 1.2.2](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.2) - Feature release - 2024-02-14
4+
5+
- Handle XML and CSV endpoints
6+
- Add secure SSO preset
7+
- Add secure username / password preset
8+
39
## [Version 1.2.1](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.1) - Bugfix release - 2023-12-13
410

511
- Fix the `Add an error column` error behaviour on the recipe
@@ -15,6 +21,8 @@
1521
- Add Brotli compression
1622
- Faster recurring calls
1723
- dku_error column kept at all time in API-Connect recipe output schema
24+
- Add XML to JSON conversion
25+
- Add CSV decoding
1826
- Updated code-env descriptor for DSS 12
1927

2028
## [Version 1.1.4](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.1.4) - Feature and bugfix release - 2023-02-28

‎code-env/python/spec/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ jsonpath-ng==1.5.3
22
requests_ntlm==1.1.0
33
requests==2.26.0
44
Brotli==1.0.9
5+
xmltodict==0.13.0

‎custom-recipes/api-connect/recipe.json‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,38 @@
3232
"type": "SEPARATOR",
3333
"label": "Authentication"
3434
},
35+
{
36+
"name": "auth_type",
37+
"label": "Authentication type",
38+
"description": "",
39+
"type": "SELECT",
40+
"defaultValue": null,
41+
"selectChoices":[
42+
{"value": "secure_oauth", "label": "SSO"},
43+
{"value": "secure_basic", "label": "Secure username / password"},
44+
{"value": null, "label": "Other"}
45+
]
46+
},
3547
{
3648
"name": "credential",
3749
"label": "Credential preset",
3850
"type": "PRESET",
39-
"parameterSetId": "credential"
51+
"parameterSetId": "credential",
52+
"visibilityCondition": "model.auth_type == null"
53+
},
54+
{
55+
"name": "secure_oauth",
56+
"label": "SSO preset",
57+
"type": "PRESET",
58+
"parameterSetId": "secure-oauth",
59+
"visibilityCondition": "model.auth_type == 'secure_oauth'"
60+
},
61+
{
62+
"name": "secure_basic",
63+
"label": "Credential preset",
64+
"type": "PRESET",
65+
"parameterSetId": "secure-basic",
66+
"visibilityCondition": "model.auth_type == 'secure_basic'"
4067
},
4168
{
4269
"type": "SEPARATOR",
@@ -254,6 +281,7 @@
254281
"name": "ignore_ssl_check",
255282
"label": "Ignore SSL check",
256283
"type": "BOOLEAN",
284+
"visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'",
257285
"defaultValue": false
258286
},
259287
{

‎custom-recipes/api-connect/recipe.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from dataiku.customrecipe import get_input_names_for_role, get_recipe_config, get_output_names_for_role
44
import pandas as pd
55
from safe_logger import SafeLogger
6-
from dku_utils import get_dku_key_values, get_endpoint_parameters
6+
from dku_utils import get_dku_key_values, get_endpoint_parameters, get_secure_credentials
77
from rest_api_recipe_session import RestApiRecipeSession
88
from dku_constants import DKUConstants
99

@@ -37,6 +37,7 @@ def get_partitioning_keys(id_list, dku_flow_variables):
3737
credential_parameters = config.get("credential", {})
3838
behaviour_when_error = config.get("behaviour_when_error", "add-error-column")
3939
endpoint_parameters = get_endpoint_parameters(config)
40+
secure_credentials = get_secure_credentials(config)
4041
extraction_key = endpoint_parameters.get("extraction_key", "")
4142
is_raw_output = endpoint_parameters.get("raw_output", True)
4243
parameter_columns = [column for column in config.get("parameter_columns", []) if column]
@@ -54,6 +55,7 @@ def get_partitioning_keys(id_list, dku_flow_variables):
5455
recipe_session = RestApiRecipeSession(
5556
custom_key_values,
5657
credential_parameters,
58+
secure_credentials,
5759
endpoint_parameters,
5860
extraction_key,
5961
parameter_columns,

‎parameter-sets/secure-basic/parameter-set.json‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"meta" : {
3+
"label": "Secure basic",
4+
"description": "Secured preset to be used solely on one domain. Username / password have to be set by individual users from their own profile's credential tab.",
5+
"icon": "icon-puzzle-piece"
6+
},
7+
"defaultDefinableInline": false,
8+
"defaultDefinableAtProjectLevel": false,
9+
"pluginParams": [
10+
],
11+
"params": [
12+
{
13+
"name": "secure_token",
14+
"type": "CREDENTIAL_REQUEST",
15+
"label": "Azure Single Sign On",
16+
"credentialRequestSettings": {
17+
"type": "BASIC"
18+
},
19+
"mandatory": true
20+
},
21+
{
22+
"name": "secure_domain",
23+
"label": "Domain",
24+
"description": "",
25+
"type": "STRING",
26+
"mandatory": true
27+
},
28+
{
29+
"name": "login_type",
30+
"label": "Login type",
31+
"type": "SELECT",
32+
"defaultValue": "basic_login",
33+
"selectChoices": [
34+
{
35+
"value": "basic_login",
36+
"label": "Basic auth"
37+
},
38+
{
39+
"value": "ntlm",
40+
"label": "NTLM"
41+
}
42+
]
43+
}
44+
]
45+
}

‎parameter-sets/secure-oauth/parameter-set.json‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"meta" : {
3+
"label": "Secure SSO credentials",
4+
"description": "",
5+
"icon": "icon-puzzle-piece"
6+
},
7+
"defaultDefinableInline": false,
8+
"defaultDefinableAtProjectLevel": false,
9+
"pluginParams": [
10+
],
11+
12+
"params": [
13+
{
14+
"name": "secure_token",
15+
"type": "CREDENTIAL_REQUEST",
16+
"label": "Single Sign On",
17+
"credentialRequestSettings": {
18+
"type": "OAUTH2",
19+
"authorizationEndpoint": " ",
20+
"tokenEndpoint": " ",
21+
"scope": " "
22+
},
23+
"mandatory": true
24+
},
25+
{
26+
"name": "authorizationEndpoint",
27+
"label": "Authorization endpoint",
28+
"type": "STRING",
29+
"description": "",
30+
"mandatory": false
31+
},
32+
{
33+
"name": "tokenEndpoint",
34+
"label": "Token endpoint",
35+
"type": "STRING",
36+
"description": "",
37+
"mandatory": false
38+
},
39+
{
40+
"name": "scope",
41+
"label": "Scope",
42+
"type": "STRING",
43+
"description": "",
44+
"mandatory": false
45+
},
46+
{
47+
"name": "secure_domain",
48+
"label": "Domain",
49+
"description": "",
50+
"type": "STRING",
51+
"mandatory": true
52+
}
53+
]
54+
}

‎plugin.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"id": "api-connect",
3-
"version": "1.2.1",
3+
"version": "1.2.2",
44
"meta": {
55
"label": "API Connect",
66
"description": "Retrieve data from any REST API",

‎python-connectors/api-connect_dataset/connector.json‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,38 @@
1111
"type": "SEPARATOR",
1212
"label": "Authentication"
1313
},
14+
{
15+
"name": "auth_type",
16+
"label": "Authentication type",
17+
"description": "",
18+
"type": "SELECT",
19+
"defaultValue": null,
20+
"selectChoices":[
21+
{"value": "secure_oauth", "label": "SSO"},
22+
{"value": "secure_basic", "label": "Secure username / password"},
23+
{"value": null, "label": "Other"}
24+
]
25+
},
1426
{
1527
"name": "credential",
1628
"label": "Credential preset",
1729
"type": "PRESET",
18-
"parameterSetId": "credential"
30+
"parameterSetId": "credential",
31+
"visibilityCondition": "model.auth_type == null"
32+
},
33+
{
34+
"name": "secure_oauth",
35+
"label": "SSO preset",
36+
"type": "PRESET",
37+
"parameterSetId": "secure-oauth",
38+
"visibilityCondition": "model.auth_type == 'secure_oauth'"
39+
},
40+
{
41+
"name": "secure_basic",
42+
"label": "Credential preset",
43+
"type": "PRESET",
44+
"parameterSetId": "secure-basic",
45+
"visibilityCondition": "model.auth_type == 'secure_basic'"
1946
},
2047
{
2148
"type": "SEPARATOR",
@@ -117,8 +144,8 @@
117144
},
118145
{
119146
"name": "raw_output",
120-
"label": "Raw JSON output",
121-
"description": "",
147+
"label": " ",
148+
"description": "Raw JSON output",
122149
"defaultValue": true,
123150
"type": "BOOLEAN"
124151
},
@@ -198,20 +225,23 @@
198225
},
199226
{
200227
"name": "ignore_ssl_check",
201-
"label": "Ignore SSL check",
228+
"label": " ",
229+
"description": "Ignore SSL check",
202230
"type": "BOOLEAN",
231+
"visibilityCondition": "model.auth_type!='secure_oauth' && model.auth_type!='secure_basic'",
203232
"defaultValue": false
204233
},
205234
{
206235
"name": "redirect_auth_header",
207-
"label": "Redirect authorization header",
236+
"label": " ",
237+
"description": "Redirect authorization header",
208238
"type": "BOOLEAN",
209239
"defaultValue": false
210240
},
211241
{
212242
"name": "display_metadata",
213-
"label": "Display metadata",
214-
"description": "Status code, request time...",
243+
"label": " ",
244+
"description": "Display metadata",
215245
"type": "BOOLEAN",
216246
"defaultValue": false
217247
},

‎python-connectors/api-connect_dataset/connector.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@
22
from dataikuapi.utils import DataikuException
33
from safe_logger import SafeLogger
44
from rest_api_client import RestAPIClient
5-
from dku_utils import get_dku_key_values, get_endpoint_parameters, parse_keys_for_json, get_value_from_path
5+
from dku_utils import (
6+
get_dku_key_values, get_endpoint_parameters,
7+
parse_keys_for_json, get_value_from_path, get_secure_credentials, decode_csv_data
8+
)
69
from dku_constants import DKUConstants
710
import json
811

@@ -17,9 +20,10 @@ def __init__(self, config, plugin_config):
1720
logger.info('API-Connect plugin connector v{}'.format(DKUConstants.PLUGIN_VERSION))
1821
logger.info("config={}".format(logger.filter_secrets(config)))
1922
endpoint_parameters = get_endpoint_parameters(config)
23+
secure_credentials = get_secure_credentials(config)
2024
credential = config.get("credential", {})
2125
custom_key_values = get_dku_key_values(config.get("custom_key_values", {}))
22-
self.client = RestAPIClient(credential, endpoint_parameters, custom_key_values)
26+
self.client = RestAPIClient(credential, secure_credentials, endpoint_parameters, custom_key_values)
2327
extraction_key = endpoint_parameters.get("extraction_key", None)
2428
self.extraction_key = extraction_key or ''
2529
self.extraction_path = self.extraction_key.split('.')
@@ -49,9 +53,14 @@ def generate_rows(self, dataset_schema=None, dataset_partitioning=None,
4953
record_count += len(data)
5054
for row in data:
5155
yield self.format_output(row, metadata)
52-
else:
56+
elif isinstance(data, dict):
5357
record_count += 1
5458
yield self.format_output(data, metadata)
59+
else:
60+
data = decode_csv_data(data)
61+
record_count += len(data)
62+
for row in data:
63+
yield self.format_output(row, metadata)
5564
if is_records_limit and record_count >= records_limit:
5665
break
5766

‎python-lib/dku_constants.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
class DKUConstants(object):
22
API_RESPONSE_KEY = "api_response"
3-
FORBIDDEN_KEYS = ["token", "password", "api_key_value"]
3+
FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"]
44
FORM_DATA_BODY_FORMAT = "FORM_DATA"
5-
PLUGIN_VERSION = "1.2.1"
5+
PLUGIN_VERSION = "1.2.2"
66
RAW_BODY_FORMAT = "RAW"
77
REPONSE_ERROR_KEY = "dku_error"

0 commit comments

Comments
 (0)