Update schema to include field validations

Author: pflooky

docker/data/custom/validation/all/all-validation.yaml

@@ -9,6 +9,7 @@ dataSources:
         pauseInSeconds: 1
       validations:
         - expr: "amount < 100"
+          description: "Amount should be less than 100"
         - expr: "year == 2021"
           errorThreshold: 0.1
         - expr: "regexp_like(name, 'Peter .*')"
@@ -18,64 +19,68 @@ dataSources:
         - groupByFields: ["account_id"]
           aggType: "count"
           aggExpr: "count == 1"
-        - fieldNameType: "columnCountEqual"
-          count: "3"
-        - fieldNameType: "columnCountBetween"
-          minCount: "1"
-          maxCount: "2"
-        - fieldNameType: "columnNameMatchOrder"
+          errorThreshold: 10
+        - groupByFields: ["account_id"]
+          aggExpr: "count == 1"
+          aggType: count
+        - fieldNameType: "fieldCountEqual"
+          count: 3
+          errorThreshold: 0.1
+        - fieldNameType: "fieldCountBetween"
+          minCount: 1
+          maxCount: 2
+        - fieldNameType: "fieldNameMatchOrder"
           names: ["account_id", "amount", "name"]
-        - fieldNameType: "columnNameMatchSet"
+        - fieldNameType: "fieldNameMatchSet"
           names: ["account_id", "my_name"]
         - upstreamDataSource: "my_first_json"
           upstreamReadOptions: {}
           joinFields: ["account_id"]
           joinType: "outer"
-          validation:
-            expr: "my_first_json_customer_details.name == name"
+          validations:
+            - expr: "my_first_json_customer_details.name == name"
         - upstreamDataSource: "my_first_json"
           upstreamReadOptions: {}
           joinFields: ["account_id"]
           joinType: "outer"
-          validation:
-            expr: "amount != my_first_json_balance"
+          validations:
+            - expr: "amount != my_first_json_balance"
         - upstreamDataSource: "my_first_json"
-          upstreamReadOptions: {}
           joinFields: ["expr:account_id == my_first_json_account_id"]
           joinType: "outer"
-          validation:
-            groupByCols: ["account_id", "my_first_json_balance"]
-            aggExpr: "sum(amount) BETWEEN my_first_json_balance * 0.8 AND my_first_json_balance * 1.2"
+          validations:
+            - groupByFields: ["account_id", "my_first_json_balance"]
+              aggExpr: "sum(amount) BETWEEN my_first_json_balance * 0.8 AND my_first_json_balance * 1.2"
+              aggType: sum
         - upstreamDataSource: "my_first_json"
-          upstreamReadOptions: {}
           joinFields: ["account_id"]
           joinType: "outer"
-          validation:
-            groupByCols: ["account_id", "my_first_json_balance"]
-            aggExpr: "sum(amount) BETWEEN my_first_json_balance * 0.8 AND my_first_json_balance * 1.2"
+          validations:
+            - groupByFields: ["account_id", "my_first_json_balance"]
+              aggExpr: "sum(amount) BETWEEN my_first_json_balance * 0.8 AND my_first_json_balance * 1.2"
+              aggType: sum
         - upstreamDataSource: "my_first_json"
           upstreamReadOptions: {}
           joinFields: ["account_id"]
           joinType: "anti"
-          validation:
-            aggType: "count"
-            aggExpr: "count == 0"
+          validations:
+            - aggType: "count"
+              aggExpr: "count == 0"
         - upstreamDataSource: "my_first_json"
           upstreamReadOptions: {}
           joinFields: ["account_id"]
           joinType: "outer"
-          validation:
-            aggType: "count"
-            aggExpr: "count == 30"
+          validations:
+            - aggType: "count"
+              aggExpr: "count == 30"
         - upstreamDataSource: "my_first_json"
           upstreamReadOptions: {}
           joinFields: ["account_id"]
           joinType: "outer"
-          validation:
-            upstreamDataSource: "my_third_json"
-            upstreamReadOptions: {}
-            joinFields: ["account_id"]
-            joinType: "outer"
-            validation:
-              aggType: "count"
-              aggExpr: "count == 30"
+          validations:
+            - upstreamDataSource: "my_third_json"
+              joinFields: ["account_id"]
+              joinType: "outer"
+              validations:
+                - aggType: "count"
+                  aggExpr: "count == 30"

docker/data/custom/validation/json/json-validation.yaml

@@ -38,7 +38,7 @@ dataSources:
             - type: "hasType"
               value: "string"
               negate: true
-            - type: "hasType"
+            - type: "hasTypes"
               values: [ "string", "double" ]
               negate: true
             - type: "distinctInSet"
@@ -70,13 +70,13 @@ dataSources:
         - field: links
           validation:
             - type: "size"
-              value: 5
+              size: 5
               negate: true
             - type: "lessThanSize"
-              value: 5
+              size: 5
               strictly: false
             - type: "greaterThanSize"
-              value: 5
+              size: 5
               strictly: false
         - field: amount
           validation:
@@ -128,6 +128,6 @@ dataSources:
             - type: "quantileValuesBetween"
               quantileRanges:
                 "0.1":
-                  - 1.0
-                  - 10.0
+                  - - 1.0
+                    - 10.0
               negate: true

docker/data/custom/validation/simple-validation.yaml

@@ -7,6 +7,10 @@ dataSources:
         path: "app/src/test/resources/sample/json/txn-gen"
       validations:
         - expr: "amount < 100"
+        - field: name
+          validation:
+            - type: "null"
+              negate: false
         - field: amount
           validation:
             - type: "null"