Upgrade to 0.14.7, update links in README, use timeTakenMs for HTTP response validation

Author: pflooky

Dockerfile

@@ -1,3 +1,3 @@
-FROM datacatering/data-caterer:0.14.6
+FROM datacatering/data-caterer:0.14.7
 
 COPY --chown=app:app build/libs/data-caterer-example-0.1.0.jar /opt/app/job.jar

README.md

@@ -14,10 +14,10 @@ This repo contains example Java and Scala API usage for Data Caterer.
 
 Check out any of [the Scala examples](src/main/scala/io/github/datacatering/plan) or 
 [Java examples](src/main/java/io/github/datacatering/plan). If you are looking for more information, you can follow 
-[**detailed documentation found here**](https://data.catering/setup/).
+[**detailed documentation found here**](https://data.catering/latest/docs/).
 
 Not comfortable with Java or Scala? No worries.
-[You can use the UI via these steps](https://data.catering/get-started/quick-start/).
+[You can use the UI via these steps](https://data.catering/latest/get-started/quick-start/).
 
 Want some YAML instead? Also, no worries. Check the example [plan](docker/data/custom/plan) and 
 [task](docker/data/custom/task) YAML files here.
@@ -84,7 +84,7 @@ cd docker
 PLAN_CLASS=io.github.datacatering.plan.DocumentationPlanRun DATA_SOURCE=postgres docker-compose up -d datacaterer
 ```
 
-[Details from docs](https://data.catering/get-started/docker/).  
+[Details from docs](https://data.catering/latest/get-started/quick-start/).  
 Docker compose sample found under `docker` folder.
 
 ```shell

docker/mount/http/petstore.json

@@ -22,60 +22,6 @@
   ],
   "paths": {
     "/pets": {
-      "get": {
-        "description": "Returns all pets from the system that the user has access to\nNam sed condimentum est. Maecenas tempor sagittis sapien, nec rhoncus sem sagittis sit amet. Aenean at gravida augue, ac iaculis sem. Curabitur odio lorem, ornare eget elementum nec, cursus id lectus. Duis mi turpis, pulvinar ac eros ac, tincidunt varius justo. In hac habitasse platea dictumst. Integer at adipiscing ante, a sagittis ligula. Aenean pharetra tempor ante molestie imperdiet. Vivamus id aliquam diam. Cras quis velit non tortor eleifend sagittis. Praesent at enim pharetra urna volutpat venenatis eget eget mauris. In eleifend fermentum facilisis. Praesent enim enim, gravida ac sodales sed, placerat id erat. Suspendisse lacus dolor, consectetur non augue vel, vehicula interdum libero. Morbi euismod sagittis libero sed lacinia.\n\nSed tempus felis lobortis leo pulvinar rutrum. Nam mattis velit nisl, eu condimentum ligula luctus nec. Phasellus semper velit eget aliquet faucibus. In a mattis elit. Phasellus vel urna viverra, condimentum lorem id, rhoncus nibh. Ut pellentesque posuere elementum. Sed a varius odio. Morbi rhoncus ligula libero, vel eleifend nunc tristique vitae. Fusce et sem dui. Aenean nec scelerisque tortor. Fusce malesuada accumsan magna vel tempus. Quisque mollis felis eu dolor tristique, sit amet auctor felis gravida. Sed libero lorem, molestie sed nisl in, accumsan tempor nisi. Fusce sollicitudin massa ut lacinia mattis. Sed vel eleifend lorem. Pellentesque vitae felis pretium, pulvinar elit eu, euismod sapien.\n",
-        "operationId": "findPets",
-        "parameters": [
-          {
-            "name": "tags",
-            "in": "query",
-            "description": "tags to filter by",
-            "required": false,
-            "style": "form",
-            "schema": {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            }
-          },
-          {
-            "name": "limit",
-            "in": "query",
-            "description": "maximum number of results to return",
-            "required": false,
-            "schema": {
-              "type": "integer",
-              "format": "int32"
-            }
-          }
-        ],
-        "responses": {
-          "200": {
-            "description": "pet response",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "type": "array",
-                  "items": {
-                    "$ref": "#/components/schemas/Pet"
-                  }
-                }
-              }
-            }
-          },
-          "default": {
-            "description": "unexpected error",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/Error"
-                }
-              }
-            }
-          }
-        }
-      },
       "post": {
         "description": "Creates a new pet in the store. Duplicates are allowed",
         "operationId": "addPet",

gradle.properties

@@ -8,5 +8,5 @@ version=0.1.0
 
 scalaVersion=2.12
 scalaSpecificVersion=2.12.19
-dataCatererVersion=0.14.6
+dataCatererVersion=0.14.7
 sparkMajorVersion=3.5

helm/data-caterer/values.yaml

@@ -8,7 +8,7 @@ image:
   repository: "datacatering/data-caterer"
   pullPolicy: "IfNotPresent"
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.14.6"
+  tag: "0.14.7"
 
 imagePullSecrets: []
 nameOverride: ""

src/main/java/io/github/datacatering/plan/CsvGenerateIcebergValidateJavaPlan.java

@@ -1,12 +1,7 @@
 package io.github.datacatering.plan;
 
-import io.github.datacatering.datacaterer.api.model.DateType;
-import io.github.datacatering.datacaterer.api.model.DecimalType;
-import io.github.datacatering.datacaterer.api.model.DoubleType;
-import io.github.datacatering.datacaterer.api.model.TimestampType;
 import io.github.datacatering.datacaterer.javaapi.api.PlanRun;
 
-import java.util.List;
 import java.util.Map;
 
 public class CsvGenerateIcebergValidateJavaPlan extends PlanRun {

src/main/java/io/github/datacatering/plan/HttpOpenAPIJavaPlanRun.java

@@ -9,7 +9,7 @@ public class HttpOpenAPIJavaPlanRun extends PlanRun {
     {
         var httpTask = http("my_http", Map.of(Constants.ROWS_PER_SECOND(), "1"))
                 .fields(metadataSource().openApi("/opt/app/mount/http/petstore.json"))
-                .fields(field().name("body").fields(field().name("id").regex("ID[0-9]{8}")))
+                .fields(field().httpBody(field().name("id").regex("ID[0-9]{8}")))
                 .count(count().records(2));
 
         var myPlan = plan().addForeignKeyRelationship(

src/main/scala/io/github/datacatering/plan/CsvGenerateIcebergValidatePlan.scala

@@ -1,7 +1,6 @@
 package io.github.datacatering.plan
 
 import io.github.datacatering.datacaterer.api.PlanRun
-import io.github.datacatering.datacaterer.api.model.{DateType, DecimalType, DoubleType, TimestampType}
 
 class CsvGenerateIcebergValidatePlan extends PlanRun {
 

src/main/scala/io/github/datacatering/plan/HttpOpenAPIPlanRun.scala

@@ -7,26 +7,29 @@ class HttpOpenAPIPlanRun extends PlanRun {
 
   val httpTask = http("my_http", options = Map(ROWS_PER_SECOND -> "1"))
     .fields(metadataSource.openApi("/opt/app/mount/http/petstore.json"))
-    .fields(field.name("body").fields(field.name("id").regex("ID[0-9]{8}")))
+    .fields(field.httpBody(field.name("id").regex("ID[0-9]{8}")))
     .count(count.records(2))
 
-  val httpGetTask = http("get_http", options = Map(VALIDATION_IDENTIFIER -> "GET/pets/{id}"))
+  val httpGetTask = http("my_http", options = Map(VALIDATION_IDENTIFIER -> "GET/pets/{id}"))
     .validations(
       validation.field("request.method").isEqual("GET"),
       validation.field("request.method").isEqualField("response.statusText"),
       validation.field("response.statusCode").isEqual(200),
+      validation.field("response.timeTakenMs").lessThan(10),
       validation.field("response.headers.Content-Length").greaterThan(0),
       validation.field("response.headers.Content-Type").isEqual("application/json"),
     )
 
   val myPlan = plan.addForeignKeyRelationship(
     foreignField("my_http", "POST/pets", "body.id"),
-    foreignField("my_http", "DELETE/pets/{id}", "pathParamid"),
-    foreignField("my_http", "GET/pets/{id}", "pathParamid")
+    foreignField("my_http", "GET/pets/{id}", "pathParamid"),
+    foreignField("my_http", "DELETE/pets/{id}", "pathParamid")
   )
 
   val conf = configuration.enableGeneratePlanAndTasks(true)
     .generatedReportsFolderPath("/opt/app/data/report")
+    .recordTrackingFolderPath("/opt/app/data/record-tracking")
+    .recordTrackingForValidationFolderPath("/opt/app/data/record-tracking-valid")
 
   execute(myPlan, conf, httpTask, httpGetTask)
 }

src/main/scala/io/github/datacatering/plan/HttpPlanRun.scala

@@ -11,12 +11,12 @@ class HttpPlanRun extends PlanRun {
     .fields(field.httpUrl(
       "http://host.docker.internal:80/anything/pets", //url
       HttpMethodEnum.POST //method
-    ): _*)
+    ))
     .fields(
       field.httpBody(
         field.name("id").regex("[0-9]{8}"),
         field.name("name").expression("#{Name.name}"),
-      ): _*
+      )
     )
     .count(count.records(10))
     .validations(
@@ -44,7 +44,7 @@ class HttpPlanRun extends PlanRun {
         List(
           field.name("limit").`type`(IntegerType).min(1).max(10) //query parameters
         )
-      ): _*
+      )
     )
     .validations(
       validation.field("request.method").isEqual("GET"),