Update to 0.14.5

Author: pflooky

Dockerfile

@@ -1,3 +1,3 @@
-FROM datacatering/data-caterer:0.14.2
+FROM datacatering/data-caterer:0.14.5
 
 COPY --chown=app:app build/libs/data-caterer-example-0.1.0.jar /opt/app/job.jar

docker/data/custom/plan/http.yaml

@@ -1,6 +1,7 @@
 name: "http_example_plan"
 description: "Hit HTTP endpoint with account data"
 tasks:
-  - name: "json_account_http"
+#  - name: "json_account_http"
+  - name: "simple_http"
     dataSourceName: "httpbin"
     enabled: true

docker/data/custom/task/http/http-account-task-simple.yaml

@@ -2,7 +2,7 @@ name: "simple_http"
 steps:
   - name: "account"
     count:
-      records: 50
+      records: 5
     fields:
       - name: "httpUrl"
         type: struct
@@ -31,6 +31,7 @@ steps:
           - name: "Content-Type"
             static: "application/json"
           - name: "Content-Length"
+            type: "integer"
           - name: "X-Account-Id"
             options:
               sql: "body.account_id"

docker/data/custom/validation/account/iceberg-account-validation.yaml

@@ -0,0 +1,25 @@
+---
+name: "iceberg_account_checks"
+description: "Check account related fields have gone through system correctly"
+dataSources:
+  iceberg:
+    - options:
+        path: "/data/iceberg/customer/account"
+      validations:
+        - field: account_id
+          validation:
+            - type: unique
+        - field: open_time
+          validation:
+            - type: isIncreasing
+        - groupByFields: [ account_id ]
+          aggType: sum
+          aggExpr: sum(balance) > 0
+        - preFilterExpr: status == 'closed'
+          expr: balance == 0
+        - aggType: count
+          aggExpr: count == 1000
+        - upstreamDataSource: "csv_accounts"
+          joinFields: ["account_id"]
+          validations:
+            - expr: "balance == csv_accounts.balance"

gradle.properties

@@ -8,5 +8,5 @@ version=0.1.0
 
 scalaVersion=2.12
 scalaSpecificVersion=2.12.19
-dataCatererVersion=0.14.2
+dataCatererVersion=0.14.5
 sparkMajorVersion=3.5

helm/data-caterer/values.yaml

@@ -8,7 +8,7 @@ image:
   repository: "datacatering/data-caterer"
   pullPolicy: "IfNotPresent"
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.14.2"
+  tag: "0.14.5"
 
 imagePullSecrets: []
 nameOverride: ""

run.sh

@@ -52,6 +52,7 @@ fi
 docker network create --driver bridge docker_default || true
 
 echo "Running Data Caterer via docker, version: $data_caterer_version"
+docker network inspect insta-infra_default >/dev/null 2>&1 || docker network create --driver bridge insta-infra_default --label com.docker.compose.network="default"
 DOCKER_CMD=(
   docker run -p 4040:4040
   -v "$(pwd)/build/libs/data-caterer-example-0.1.0.jar:/opt/app/job.jar"

src/main/java/io/github/datacatering/plan/CsvGenerateIcebergValidateJavaPlan.java

@@ -0,0 +1,39 @@
+package io.github.datacatering.plan;
+
+import io.github.datacatering.datacaterer.api.model.DateType;
+import io.github.datacatering.datacaterer.api.model.DecimalType;
+import io.github.datacatering.datacaterer.api.model.DoubleType;
+import io.github.datacatering.datacaterer.api.model.TimestampType;
+import io.github.datacatering.datacaterer.javaapi.api.PlanRun;
+
+import java.util.List;
+import java.util.Map;
+
+public class CsvGenerateIcebergValidateJavaPlan extends PlanRun {
+    {
+        var accountTask = csv("csv_accounts", "/opt/app/data/csv/customer/account", Map.of("header", "true"))
+                .fields(metadataSource().openDataContractStandard("/opt/app/mount/odcs/full-example.odcs.yaml"));
+
+        var transactionTask = iceberg("iceberg_transactions", "/opt/app/csv/data/customer/transaction", "dev.transactions")
+                .validations(
+                        validation().unique("account_id"),
+                        validation().groupBy("account_id").sum("balance").greaterThan(0),
+                        validation().field("open_time").isIncreasing(),
+                        validation().count().isEqual(1000),
+                        validation().preFilter(validation().field("status").isEqual("closed")).field("balance").isEqual(0),
+                        validation().upstreamData(accountTask)
+                                .joinFields("account_id")
+                                .validations(
+                                        validation().field("open_time").isEqualField("csv_accounts_open_time"),
+                                        validation().groupBy("account_id", "csv_accounts_balance").sum("amount").isEqualField("csv_accounts_balance")
+                                )
+                )
+                .validationWait(waitCondition().file("/opt/app/data/iceberg/customer/transaction"));
+
+        var config = configuration()
+                .generatedReportsFolderPath("/opt/app/data/report")
+                .enableUniqueCheck(true);
+
+        execute(config, accountTask, transactionTask);
+    }
+}

src/main/java/io/github/datacatering/plan/KafkaJavaPlanRun.java

@@ -14,10 +14,7 @@
 public class KafkaJavaPlanRun extends PlanRun {
     {
         var kafkaTask = getKafkaTask();
-
-        var conf = configuration()
-                .generatedReportsFolderPath("/opt/app/data/report");
-
+        var conf = configuration().generatedReportsFolderPath("/opt/app/data/report");
         execute(conf, kafkaTask);
     }
 

src/main/java/io/github/datacatering/plan/PostgresAutoJavaPlan.java

@@ -0,0 +1,12 @@
+package io.github.datacatering.plan;
+
+import io.github.datacatering.datacaterer.javaapi.api.PlanRun;
+
+public class PostgresAutoJavaPlan extends PlanRun {
+    {
+        var accountTask = postgres("customer_accounts", "jdbc:postgresql://host.docker.internal:5432/customer");
+
+        var config = configuration().enableGeneratePlanAndTasks(true);
+        execute(config, accountTask);
+    }
+}