Added example and default config validation

Author: kevinwang5658

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@codifycli/plugin-test",
-  "version": "1.1.0-beta2",
+  "version": "1.1.0-beta4",
   "description": "Testing framework for Codify plugins with complete lifecycle testing, IPC communication, and cross-platform support",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -27,7 +27,7 @@
     "prepublishOnly": "tsc"
   },
   "dependencies": {
-    "@codifycli/schemas": "1.0.0",
+    "@codifycli/schemas": "1.1.0-beta3",
     "ajv": "^8.18.0",
     "ajv-formats": "^3.0.1",
     "lodash.matches": "^4.6.0",
@@ -39,7 +39,7 @@
     "strip-ansi": "^7.1.2"
   },
   "devDependencies": {
-    "@codifycli/plugin-core": "1.0.0",
+    "@codifycli/plugin-core": "1.1.0-beta10",
     "@oclif/prettier-config": "^0.2.1",
     "@types/debug": "^4.1.12",
     "@types/node": "^22",

package.json

@@ -2,6 +2,8 @@ import {
   ApplyRequestData,
   CommandRequestData,
   CommandRequestDataSchema,
+  GetResourceInfoRequestData,
+  GetResourceInfoResponseData,
   ImportRequestData,
   ImportResponseData,
   InitializeResponseData,
@@ -101,6 +103,14 @@ export class PluginProcess {
     });
   }
 
+  async getResourceInfo(data: GetResourceInfoRequestData): Promise<GetResourceInfoResponseData> {
+    return TestUtils.sendMessageAndAwaitResponse(this.childProcess, {
+      cmd: 'getResourceInfo',
+      data,
+      requestId: nanoid(6),
+    });
+  }
+
   kill() {
     this.childProcess.kill();
   }

src/plugin-process.ts

@@ -46,8 +46,11 @@ export class PluginTester {
     }
 
     const plugin = new PluginProcess(pluginPath);
+
     try {
       await this.initializeAndValidate(plugin, configs);
+      
+      await this.validateExampleAndDefaultConfigs(plugin, configs);
 
       console.info(chalk.cyan('Testing plan...'))
       const plans = await this.planConfigs(plugin, configs);
@@ -215,7 +218,79 @@ export class PluginTester {
 
     const invalidConfigs = validate.resourceValidations.filter((v) => !v.isValid)
     if (invalidConfigs.length > 0) {
-      throw new Error(`The following configs did not validate:\n ${JSON.stringify(invalidConfigs, null, 2)}`)
+      const invalidWithConfigs = invalidConfigs.map((v) => ({
+        config: configs.find((c) => c.type === v.resourceType && (!v.resourceName || c.name === v.resourceName)),
+        validation: v,
+      }));
+      throw new Error(`The following configs did not validate:\n ${JSON.stringify(invalidWithConfigs, null, 2)}`)
+    }
+  }
+
+  private static async validateExampleAndDefaultConfigs(
+    plugin: PluginProcess,
+    configs: ResourceConfig[],
+  ): Promise<void> {
+    console.info(chalk.cyan('Validating examples and default config...'))
+    const uniqueTypes = [...new Set(configs.map((c) => c.type))];
+
+    const errors: string[] = [];
+
+    for (const type of uniqueTypes) {
+      const { defaultConfig, exampleConfigs } = await plugin.getResourceInfo({ type });
+
+      // Validate exampleConfigs — these should be fully valid configs
+      const exampleEntries: Array<[string, Array<Record<string, unknown>>]> = [];
+      if (exampleConfigs?.example1?.configs) exampleEntries.push(['exampleConfigs.example1', exampleConfigs.example1.configs as Array<Record<string, unknown>>]);
+      if (exampleConfigs?.example2?.configs) exampleEntries.push(['exampleConfigs.example2', exampleConfigs.example2.configs as Array<Record<string, unknown>>]);
+
+      for (const [label, exampleConfigList] of exampleEntries) {
+        for (const [idx, exampleConfig] of exampleConfigList.entries()) {
+          const configLabel = `resource '${type}' ${label}[${idx}]`;
+          await this.validateSingleExampleConfig(plugin, configLabel, exampleConfig, errors);
+        }
+      }
+
+      // Validate defaultConfig — partial config, so only warn on failure
+      if (defaultConfig) {
+        try {
+          const validate = await plugin.validate({
+            configs: [{ core: { type }, parameters: defaultConfig }]
+          });
+          const invalid = validate.resourceValidations.filter((v) => !v.isValid);
+          if (invalid.length > 0) {
+            console.warn(chalk.yellow(`Resource '${type}' defaultConfig failed validation (this may be expected for partial configs):\n${JSON.stringify(invalid, null, 2)}`));
+          }
+        } catch (error) {
+          console.warn(chalk.yellow(`Resource '${type}' defaultConfig could not be validated: ${error}`));
+        }
+      }
+    }
+
+    if (errors.length > 0) {
+      const message = errors.length === 1
+        ? errors[0]
+        : `Multiple example config validation failures:\n\n${errors.map((e, i) => `[${i + 1}] ${e}`).join('\n\n')}`;
+      throw new Error(message);
+    }
+  }
+
+  private static async validateSingleExampleConfig(
+    plugin: PluginProcess,
+    label: string,
+    exampleConfig: Record<string, unknown>,
+    errors: string[],
+  ): Promise<void> {
+    try {
+      const { coreParameters, parameters } = splitUserConfig(exampleConfig as ResourceConfig);
+      const validate = await plugin.validate({
+        configs: [{ core: coreParameters, parameters }]
+      });
+      const invalid = validate.resourceValidations.filter((v) => !v.isValid);
+      if (invalid.length > 0) {
+        errors.push(`${label} failed validation:\nConfig: ${JSON.stringify(exampleConfig, null, 2)}\nErrors: ${JSON.stringify(invalid, null, 2)}`);
+      }
+    } catch (error) {
+      errors.push(`${label} could not be validated:\nConfig: ${JSON.stringify(exampleConfig, null, 2)}\nError: ${error}`);
     }
   }
 

src/plugin-tester.ts

@@ -1,6 +1,6 @@
 import {
   CreatePlan,
-  DestroyPlan,
+  DestroyPlan, ExampleConfig,
   ModifyPlan,
   ParameterChange,
   Plugin,
@@ -22,13 +22,37 @@ export interface TestConfig2 extends StringIndexedObject {
   propB: string[];
 }
 
+const defaultConfig: Partial<TestConfig> = {
+  propA: ''
+}
+
+const example1: ExampleConfig = {
+  configs: [
+    { type: 'test', propA: 'abc', propB: 6, propC: 'def' }
+  ],
+  title: 'Example 1'
+}
+
+const example2: ExampleConfig = {
+  configs: [
+    { type: 'test', propA: 'abc', propB: 6, propC: 'def', propD: 'asda' },
+    { type: 'test2', propA: 'ghi', propB: ['a', 'b', 'c'] }
+  ],
+  title: 'Example 2',
+  description: 'ABCDEF'
+}
 
 export class TestResource extends Resource<TestConfig> {
   getSettings(): ResourceSettings<TestConfig> {
     return {
       id: 'test',
       operatingSystems: [OS.Linux, OS.Darwin],
       allowMultiple: true,
+      defaultConfig,
+      exampleConfigs: {
+        example1,
+        example2
+      }
     };
   }
 
@@ -58,6 +82,10 @@ export class TestResource2 extends Resource<TestConfig2> {
       operatingSystems: [OS.Linux, OS.Darwin],
       parameterSettings: {
         propB: { type: 'array' }
+      },
+      exampleConfigs: {
+        example1,
+        example2
       }
     };
   }
@@ -82,6 +110,7 @@ export class TestResource2 extends Resource<TestConfig2> {
 
 export class TestUninstallResource extends Resource<TestConfig> {
   first = true;
+
   getSettings(): ResourceSettings<TestConfig> {
     return {
       id: 'test-uninstall',
@@ -134,9 +163,11 @@ export class TestModifyResource extends Resource<TestConfig> {
     return super.modify(pc, plan);
   }
 
-  async create(plan: CreatePlan<TestConfig>): Promise<void> {}
+  async create(plan: CreatePlan<TestConfig>): Promise<void> {
+  }
 
-  async destroy(plan: DestroyPlan<TestConfig>): Promise<void> {}
+  async destroy(plan: DestroyPlan<TestConfig>): Promise<void> {
+  }
 }
 
 export class TestDestroyResource extends Resource<TestConfig> {
@@ -188,9 +219,11 @@ export class WindowsOnlyResource extends Resource<TestConfig> {
     return super.modify(pc, plan);
   }
 
-  async create(plan: CreatePlan<TestConfig>): Promise<void> {}
+  async create(plan: CreatePlan<TestConfig>): Promise<void> {
+  }
 
-  async destroy(plan: DestroyPlan<TestConfig>): Promise<void> {}
+  async destroy(plan: DestroyPlan<TestConfig>): Promise<void> {
+  }
 }
 
 export class TestDestroyResource2 extends TestDestroyResource {