Add in JSON schema for YAML validation of plan, tasks and validations

Author: pflooky

README.md

@@ -34,6 +34,24 @@ Want some YAML instead? Also, no worries. Check the example [plan](docker/data/c
    to [DocumentationPlanRun.scala](src/main/scala/io/github/datacatering/plan/DocumentationPlanRun.scala)
    1. Needs to extend `io.github.datacatering.datacaterer.api.PlanRun`
 
+### YAML
+
+1. Copy existing plan file (such as [foreign-key.yaml](docker/data/custom/plan/foreign-key.yaml)) in directory 
+[docker/data/custom/plan](docker/data/custom/plan)
+2. Copy existing task file (such as [json-account-task.yaml](docker/data/custom/task/file/json/json-account-task.yaml))
+in directory [docker/data/custom/task](docker/data/custom/task)
+   1. If you want to run data validations, copy the file [simple-validation.yaml](docker/data/custom/validation/simple-validation.yaml)
+   and add validation to plan via:
+   ```yaml
+   validations:
+     - "<name of validation (i.e. account_checks)>"
+   ```
+3. [Use JSON schema to help creating metadata for plan, tasks or validations](schema/data-caterer-latest.json).
+You can import this schema into your IDE for validation of your YAML files. Links below show how you can import the schema:
+- [IntelliJ](https://www.jetbrains.com/help/idea/json.html#ws_json_schema_add_custom)
+- [VS Code](https://code.visualstudio.com/docs/languages/json#_json-schemas-and-settings)
+
+
 ## Run
 
 Requires:

docker/data/custom/plan/validation.yaml

@@ -0,0 +1,23 @@
+name: "foreign_key_example_plan"
+description: "Create account data in JSON and Postgres"
+tasks:
+  - name: "json_account_file"
+    dataSourceName: "json"
+    enabled: true
+  - name: "postgres_account"
+    dataSourceName: "postgresCustomer"
+    enabled: true
+
+sinkOptions:
+  foreignKeys:
+    - source:
+        dataSource: "postgresCustomer"
+        step: "accounts"
+        fields: [ "account_number" ]
+      generate:
+        - dataSource: "json"
+          step: "account"
+          fields: [ "account_id" ]
+
+validations:
+  - "json_account_checks"

docker/data/custom/task/file/csv/csv-transaction-task.yaml

@@ -21,7 +21,7 @@ steps:
         options:
           expression: "#{Name.name}"
       - name: "year"
-        type: "int"
+        type: "integer"
         options:
           min: 2021
           max: 2022

docker/data/custom/task/file/json/json-account-task.yaml

@@ -9,7 +9,7 @@ steps:
     fields:
       - name: "account_id"
       - name: "year"
-        type: "int"
+        type: "integer"
         options:
           min: 2021
           max: 2022
@@ -40,6 +40,7 @@ steps:
         options:
           sql: "element_at(sort_array(transactions.txn_date), 1)"
       - name: "details"
+        type: struct
         fields:
           - name: "name"
           - name: "txn_date"
@@ -48,6 +49,7 @@ steps:
               min: "2021-01-01"
               max: "2021-12-31"
           - name: "updated_by"
+            type: struct
             fields:
               - name: "user"
               - name: "time"
@@ -60,4 +62,4 @@ steps:
           - name: "amount"
             type: "double"
           - name: "tags"
-            type: "array<string>"
+            type: "array"

docker/data/custom/task/http/http-account-task-simple.yaml

@@ -5,24 +5,28 @@ steps:
       records: 50
     fields:
       - name: "httpUrl"
+        type: struct
         fields:
           - name: "url"
             static: "http://localhost:80/anything/{id}"
           - name: "method"
             static: "PUT"
           - name: "pathParam"
+            type: array
             fields:
               - name: "id"
                 options:
                   sql: "body.account_id"
           - name: "queryParam"
+            type: array
             fields:
               - name: "limit"
                 type: "integer"
                 options:
                   min: 1
                   max: 10
       - name: "httpHeaders"
+        type: array
         fields:
           - name: "Content-Type"
             static: "application/json"
@@ -35,6 +39,7 @@ steps:
             options:
               sql: "body.details.updated_by.time"
       - name: "httpBody"
+        type: struct
         fields:
           - name: "account_id"
             options:
@@ -50,6 +55,7 @@ steps:
               min: 10.0
               max: 100.0
           - name: "details"
+            type: struct
             fields:
               - name: "name"
               - name: "txn_date"
@@ -58,6 +64,7 @@ steps:
                   min: "2021-01-01"
                   max: "2021-12-31"
               - name: "updated_by"
+                type: struct
                 fields:
                   - name: "user"
                   - name: "time"

docker/data/custom/task/http/http-account-task.yaml

@@ -23,6 +23,7 @@ steps:
         options:
           sql: "to_json(content)"
       - name: "content"
+        type: struct
         fields:
           - name: "account_id"
           - name: "year"
@@ -36,6 +37,7 @@ steps:
               min: 10.0
               max: 100.0
           - name: "details"
+            type: struct
             fields:
               - name: "name"
               - name: "txn_date"
@@ -44,6 +46,7 @@ steps:
                   min: "2021-01-01"
                   max: "2021-12-31"
               - name: "updated_by"
+                type: struct
                 fields:
                   - name: "user"
                   - name: "time"

docker/data/custom/task/jdbc/postgres/postgres-account-task.yaml

@@ -3,7 +3,7 @@ steps:
   - name: "accounts"
     type: "postgres"
     count:
-      records: "1000"
+      records: 1000
     options:
       dbtable: "account.accounts"
     fields:
@@ -32,7 +32,7 @@ steps:
       - name: "customer_id_bigint"
         type: "long"
       - name: "customer_id_decimal"
-        type: "decimal(26,3)"
+        type: "decimal"
       - name: "customer_id_real"
         type: "float"
       - name: "customer_id_double"

docker/data/custom/task/jms/solace/jms-account-task.yaml

@@ -1,7 +1,7 @@
 name: "json_account_jms"
 steps:
   - name: "jms_account"
-    type: "json"
+    type: "solace"
     count:
       records: 50
     options:
@@ -12,10 +12,11 @@ steps:
         options:
           sql: "to_json(content)"
       - name: "content"
+        type: struct
         fields:
           - name: "account_id"
           - name: "year"
-            type: "int"
+            type: "integer"
             options:
               min: 2021
               max: 2022
@@ -25,6 +26,7 @@ steps:
               min: 10.0
               max: 100.0
           - name: "details"
+            type: "struct"
             fields:
               - name: "name"
               - name: "txn_date"
@@ -33,6 +35,7 @@ steps:
                   min: "2021-01-01"
                   max: "2021-12-31"
               - name: "updated_by"
+                type: "struct"
                 fields:
                   - name: "user"
                   - name: "time"

docker/data/custom/task/kafka/kafka-account-task-simple.yaml

@@ -1,9 +1,9 @@
 name: "simple_kafka"
 steps:
   - name: "kafka_account"
-    type: "json"
+    type: "kafka"
     count:
-      records: "10"
+      records: 10
     options:
       topic: "account-topic"
     fields:
@@ -12,26 +12,29 @@ steps:
         options:
           sql: "body.account_id"
       - name: "messageBody"
+        type: struct
         fields:
           - name: "account_id"
           - name: "year"
             type: "int"
             options:
-              min: "2021"
-              max: "2022"
+              min: 2021
+              max: 2022
           - name: "amount"
             type: "double"
             options:
-              min: "10.0"
-              max: "100.0"
+              min: 10.0
+              max: 100.0
           - name: "details"
+            type: struct
             fields:
               - name: "name"
               - name: "first_txn_date"
                 type: "date"
                 options:
                   sql: "ELEMENT_AT(SORT_ARRAY(body.transactions.txn_date), 1)"
               - name: "updated_by"
+                type: struct
                 fields:
                   - name: "user"
                   - name: "time"
@@ -44,6 +47,7 @@ steps:
               - name: "amount"
                 type: "double"
       - name: "messageHeaders"
+        type: struct
         fields:
           - name: "account-id"
             options:

docker/data/custom/task/kafka/kafka-account-task.yaml

@@ -1,7 +1,7 @@
 name: "json_account_kafka"
 steps:
   - name: "kafka_account"
-    type: "json"
+    type: "kafka"
     count:
       records: 10
     options:
@@ -26,6 +26,7 @@ steps:
 #          options:
 #            sql: "1"
       - name: "content"
+        type: struct
         fields:
           - name: "account_id"
           - name: "year"
@@ -39,6 +40,7 @@ steps:
               min: 10.0
               max: 100.0
           - name: "details"
+            type: struct
             fields:
               - name: "name"
               - name: "txn_date"
@@ -47,6 +49,7 @@ steps:
                   min: "2021-01-01"
                   max: "2021-12-31"
               - name: "updated_by"
+                type: struct
                 fields:
                   - name: "user"
                   - name: "time"