Add gradle task to check YAML files adhere to JSON schema

Author: pflooky

build.gradle.kts

@@ -1,21 +1,12 @@
-/*
- * This file was generated by the Gradle 'init' task.
- *
- * This is a general purpose Gradle build.
- * To learn more about Gradle by exploring our Samples at https://docs.gradle.org/8.2.1/samples
- * This project uses @Incubating APIs which are subject to change.
- */
 val scalaVersion: String by project
 val scalaSpecificVersion: String by project
 val dataCatererVersion: String by project
 
-
 plugins {
     scala
 }
 
 repositories {
-    // Use Maven Central for resolving dependencies.
     mavenCentral()
 }
 
@@ -25,3 +16,11 @@ dependencies {
     compileOnly("io.github.data-catering:data-caterer-api:$dataCatererVersion")
 }
 
+tasks.register<ValidateYamlAgainstSchema>("validateYaml") {
+    yamlDirectory.set(layout.projectDirectory.dir("docker/data/custom"))
+    schemaFile.set(layout.projectDirectory.file("schema/data-caterer-latest.json"))
+}
+
+tasks.build {
+    dependsOn("validateYaml")
+}

buildSrc/build.gradle.kts

@@ -0,0 +1,18 @@
+plugins {
+    kotlin("jvm") version "1.9.0" // Ensure Kotlin is applied
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.18.2")
+    implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2")
+    implementation("com.networknt:json-schema-validator:1.0.87")
+    implementation(kotlin("stdlib-jdk8"))
+}
+
+kotlin {
+    jvmToolchain(17)
+}

buildSrc/src/main/kotlin/ValidateYamlAgainstSchema.kt

@@ -0,0 +1,61 @@
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
+import com.networknt.schema.JsonSchema
+import com.networknt.schema.JsonSchemaFactory
+import com.networknt.schema.SpecVersion
+import com.networknt.schema.ValidationMessage
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.tasks.InputDirectory
+import org.gradle.api.tasks.InputFile
+import org.gradle.api.tasks.TaskAction
+import org.gradle.api.GradleException
+
+abstract class ValidateYamlAgainstSchema : DefaultTask() {
+
+    @get:InputDirectory
+    abstract val yamlDirectory: DirectoryProperty
+
+    @get:InputFile
+    abstract val schemaFile: RegularFileProperty
+
+    @TaskAction
+    fun validateYaml() {
+        val objectMapper = ObjectMapper(YAMLFactory()) // Registers YAML module
+        objectMapper.readTree(schemaFile.get().asFile.readText())
+        val schemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V201909) //Or V7, V201909, etc.
+        val schemaNode: JsonNode = objectMapper.readTree(schemaFile.asFile.get())
+        val schema: JsonSchema = schemaFactory.getSchema(schemaNode)
+
+        val validationResults = yamlDirectory.asFile.get().walk()
+            .filter { it.isFile && (it.extension == "yaml" || it.extension == "yml") }
+            .map { file ->
+                try {
+                    val yamlNode: JsonNode = objectMapper.readTree(file)
+                    val validationResult: Set<ValidationMessage> = schema.validate(yamlNode)
+                    file to validationResult
+                } catch (e: Exception) {
+                    println("Fail read or validation failed, path: ${file.absolutePath}, message: ${e.message}")
+                    file to setOf() // Create a validation message for errors
+                }
+            }
+            .toList() // Evaluate and collect all results immediately
+
+        val failedValidations = validationResults.filter { it.second.isNotEmpty() }
+
+        if (failedValidations.isNotEmpty()) {
+            val errorMessage = StringBuilder("YAML validation failed for the following files:\n")
+            failedValidations.forEach { (file, errors) ->
+                errorMessage.append("  - file://${file}:\n")
+                errors.forEach { message ->
+                    errorMessage.append("    - ${message.message} (path: ${message.schemaPath})\n")
+                }
+            }
+            throw GradleException(errorMessage.toString())
+        } else {
+            println("All YAML files validated successfully.")
+        }
+    }
+}

docker/data/custom/task/http/http-account-task.yaml

@@ -4,25 +4,32 @@ steps:
     count:
       records: 50
     fields:
-      - name: "url"
-        options:
-          sql: "CONCAT('http://http:80/anything/', content.account_id)"
-      - name: "method"
-        static: "PUT"
-      - name: "content_type"
-        static: "application/json"
-      - name: "headers"
-        type: "array<struct<key: string, value: string>>"
-        options:
-          sql: >-
-            array(
-              named_struct('key', 'account-id', 'value', content.account_id),
-              named_struct('key', 'updated', 'value', content.details.updated_by.time)
-            )
-      - name: "value"
-        options:
-          sql: "to_json(content)"
-      - name: "content"
+      - name: "httpUrl"
+        type: struct
+        fields:
+          - name: "url"
+            static: "http://localhost:80/anything/{id}"
+          - name: "method"
+            static: "PUT"
+          - name: "pathParam"
+            type: struct
+            fields:
+              - name: "id"
+                options:
+                  sql: "body.account_id"
+      - name: "httpHeaders"
+        type: struct
+        fields:
+          - name: "Content-Type"
+            static: "application/json"
+          - name: "account_id"
+            options:
+              sql: "body.account_id"
+          - name: "updated"
+            type: "timestamp"
+            options:
+              sql: "body.details.updated_by.time"
+      - name: "httpBody"
         type: struct
         fields:
           - name: "account_id"

docker/data/custom/task/jdbc/postgres/postgres-account-task.yaml

@@ -12,9 +12,7 @@ steps:
           regex: "ACC1[0-9]{5,10}"
       - name: "account_status"
         options:
-          oneOf:
-            - "open"
-            - "closed"
+          oneOf: ["open", "closed"]
       - name: "created_by"
         options:
           expression: "#{Name.username}"

docker/data/custom/task/kafka/kafka-account-task.yaml

@@ -13,14 +13,6 @@ steps:
       - name: "value"
         options:
           sql: "to_json(content)"
-      - name: "headers"
-        type: "array<struct<key: string, value: binary>>"
-        options:
-          sql: >-
-            array(
-              named_struct('key', 'account-id', 'value', to_binary(content.account_id, 'utf-8')),
-              named_struct('key', 'updated', 'value', to_binary(content.details.updated_by.time, 'utf-8'))
-            )
 #        - name: "partition"
 #          type: "int"
 #          options:

schema/data-caterer-latest.json

@@ -728,6 +728,11 @@
           "description": "Generate only unique values.",
           "default": false
         },
+        "isPrimaryKey": {
+          "type": "boolean",
+          "description": "Define the field as part of the primary key.",
+          "default": false
+        },
         "seed": {
           "type": "number",
           "description": "Seed for generating consistent random values.",
@@ -1678,7 +1683,8 @@
           }
         }
       },
-      "additionalProperties": false
+      "required": ["field", "validation"],
+      "unevaluatedProperties": false
     },
     "BasicValidation": {
       "type": "object",