Update to 0.14.2, use method helpers for message and http, ask for user and token when running plan

Author: pflooky

Dockerfile

@@ -1,3 +1,3 @@
-FROM datacatering/data-caterer:0.14.0
+FROM datacatering/data-caterer:0.14.2
 
 COPY --chown=app:app build/libs/data-caterer-example-0.1.0.jar /opt/app/job.jar

benchmark/run_benchmark.sh

@@ -18,7 +18,7 @@ case "${uname_out}" in
     *)        sed_option="-r";;
 esac
 data_sizes=(10000 100000 1000000)
-job_names=("BenchmarkForeignKeyPlanRun" "BenchmarkJsonPlanRun" "BenchmarkParquetPlanRun") #"BenchmarkAdvancedKafkaPlanRun"
+job_names=("BenchmarkForeignKeyPlanRun" "BenchmarkJsonPlanRun" "BenchmarkParquetPlanRun")
 
 spark_query_execution_engines=("default" "blaze" "comet" "gluten")
 gluten_spark_conf="--conf \"spark.plugins=io.glutenproject.GlutenPlugin\" --conf \"spark.memory.offHeap.enabled=true\" --conf \"spark.memory.offHeap.size=1024mb\" --conf \"spark.shuffle.manager=org.apache.spark.shuffle.sort.ColumnarShuffleManager\""

docker/data/custom/task/http/http-account-task-simple.yaml

@@ -14,26 +14,26 @@ steps:
             fields:
               - name: "id"
                 options:
-                  sql: "bodyContent.account_id"
+                  sql: "body.account_id"
           - name: "queryParam"
             fields:
               - name: "limit"
                 type: "integer"
                 options:
-                  oneOf:
-                    - 10
-                    - 5
+                  min: 1
+                  max: 10
       - name: "httpHeaders"
         fields:
           - name: "Content-Type"
             static: "application/json"
+          - name: "Content-Length"
           - name: "X-Account-Id"
             options:
-              sql: "bodyContent.account_id"
+              sql: "body.account_id"
           - name: "X-Updated"
             type: "timestamp"
             options:
-              sql: "bodyContent.details.updated_by.time"
+              sql: "body.details.updated_by.time"
       - name: "httpBody"
         fields:
           - name: "account_id"

docker/data/custom/task/kafka/kafka-account-task-simple.yaml

@@ -10,7 +10,7 @@ steps:
       - name: "key"
         type: "string"
         options:
-          sql: "content.account_id"
+          sql: "body.account_id"
       - name: "messageBody"
         fields:
           - name: "account_id"
@@ -27,11 +27,10 @@ steps:
           - name: "details"
             fields:
               - name: "name"
-              - name: "txn_date"
+              - name: "first_txn_date"
                 type: "date"
                 options:
-                  min: "2021-01-01"
-                  max: "2021-12-31"
+                  sql: "ELEMENT_AT(SORT_ARRAY(body.transactions.txn_date), 1)"
               - name: "updated_by"
                 fields:
                   - name: "user"
@@ -48,7 +47,7 @@ steps:
         fields:
           - name: "account-id"
             options:
-              sql: "content.account_id"
+              sql: "body.account_id"
           - name: "updated"
             options:
-              sql: "content.details.update_by.time"
+              sql: "body.details.update_by.time"

docker/data/custom/validation/simple-validation.yaml

@@ -7,6 +7,20 @@ dataSources:
         path: "app/src/test/resources/sample/json/txn-gen"
       validations:
         - expr: "amount < 100"
+        - field: amount
+          validation:
+            - type: "null"
+              negate: true
+            - type: "between"
+              min: 1
+              max: 10
+              description: "hello"
+              errorThreshold: 2
+            - type: "quantileValuesBetween"
+              quantileRanges:
+                0.1:
+                  - - 1.0
+                    - 10.0
         - expr: "year == 2021"
           errorThreshold: 0.1
         - expr: "regexp_like(name, 'Peter .*')"

gradle.properties

@@ -8,5 +8,5 @@ version=0.1.0
 
 scalaVersion=2.12
 scalaSpecificVersion=2.12.19
-dataCatererVersion=0.14.1
+dataCatererVersion=0.14.2
 sparkMajorVersion=3.5

helm/data-caterer/values.yaml

@@ -8,7 +8,7 @@ image:
   repository: "datacatering/data-caterer"
   pullPolicy: "IfNotPresent"
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.14.0"
+  tag: "0.14.2"
 
 imagePullSecrets: []
 nameOverride: ""

run.sh

@@ -1,19 +1,26 @@
 #!/usr/bin/env bash
 
+DATA_CATERER_ENV_FILE="$HOME/.data-caterer-env"
+
 data_caterer_version=$(grep dataCatererVersion gradle.properties | cut -d= -f2)
 data_caterer_user=${DATA_CATERER_API_USER:-}
 data_caterer_token=${DATA_CATERER_API_TOKEN:-}
 
 echo "Checking for Data Caterer user and token..."
-if [[ -z ${DATA_CATERER_API_USER} ]]; then
-  read -p "Data Caterer user: " data_caterer_user
-  DATA_CATERER_API_USER=data_caterer_user
-fi
-if [[ -z ${DATA_CATERER_API_TOKEN} ]]; then
-  read -p "Data Caterer token: " -s data_caterer_token
-  DATA_CATERER_API_TOKEN=data_caterer_token
-  echo
+if [[ -f "$DATA_CATERER_ENV_FILE" ]]; then
+  source "$DATA_CATERER_ENV_FILE"
+else
+  if [[ -z ${DATA_CATERER_API_USER} ]]; then
+    read -p "Data Caterer user: " data_caterer_user
+    echo "export DATA_CATERER_API_USER=$data_caterer_user" > "$DATA_CATERER_ENV_FILE"
+  fi
+  if [[ -z ${DATA_CATERER_API_TOKEN} ]]; then
+    read -p "Data Caterer token: " -s data_caterer_token
+    echo "export DATA_CATERER_API_TOKEN=$data_caterer_token" >> "$DATA_CATERER_ENV_FILE"
+    echo
+  fi
 fi
+source "$DATA_CATERER_ENV_FILE"
 
 if [[ -s ".tmp_prev_class_name" ]]; then
   prev_class_name=$(cat .tmp_prev_class_name)
@@ -58,9 +65,9 @@ DOCKER_CMD=(
   -e "DEPLOY_MODE=client"
   -e "DRIVER_MEMORY=2g"
   -e "EXECUTOR_MEMORY=2g"
-  -e "DATA_CATERER_API_USER=$data_caterer_user"
-  -e "DATA_CATERER_API_TOKEN=$data_caterer_token"
-  --network "docker_default"
+  -e "DATA_CATERER_API_USER=$DATA_CATERER_API_USER"
+  -e "DATA_CATERER_API_TOKEN=$DATA_CATERER_API_TOKEN"
+  --network "insta-infra_default"
   datacatering/data-caterer:"$data_caterer_version"
 )
 

src/main/java/io/github/datacatering/plan/AdvancedBatchEventJavaPlanRun.java

@@ -7,7 +7,7 @@
 
 public class AdvancedBatchEventJavaPlanRun extends PlanRun {
     {
-        var kafkaTask = new AdvancedKafkaJavaPlanRun().getKafkaTask();
+        var kafkaTask = new KafkaJavaPlanRun().getKafkaTask();
 
         var csvTask = csv("my_csv", "/opt/app/data/csv/account")
                 .fields(