Kevin/automated testing (#28)

Added automated tests on github actions:

Shard macOS and linux tests to improve speed
Fix testing problems
Fix platform problems (had to integrate the existing tests with the differences between tart and the github actions runner)
Fixed structural problems
Added claude to run on test failure to attempt to auto fixing

Author: kevinwang5658

.circleci/config.yml

@@ -0,0 +1,55 @@
+version: 2.1
+
+jobs:
+  test:
+    parameters:
+      executor:
+        type: executor
+    executor: << parameters.executor >>
+    steps:
+      - checkout
+      - node/install:
+          node-version: '24'
+      - run:
+          name: Enable linger for current user (Linux only)
+          command: |
+            if [[ "$OSTYPE" == "linux"* ]]; then
+              loginctl enable-linger $(whoami)
+              sudo chown -R circleci:circleci /usr/local /opt /var/tmp
+            fi
+      - run:
+          name: Install dependencies
+          command: npm ci
+      - run:
+          name: Clean up pre-installed tools
+          command: npx tsx scripts/cleanup-circleci.ts
+          no_output_timeout: 10m
+      - run:
+          name: Run tests
+          command: npm run test -- ./test --no-file-parallelism --disable-console-intercept
+          no_output_timeout: 30m
+
+orbs:
+  node: circleci/node@6
+
+executors:
+  linux-x86:
+    machine:
+      image: ubuntu-2404:current
+    resource_class: medium
+  linux-arm:
+    machine:
+      image: ubuntu-2404:current
+    resource_class: arm.medium
+  macos:
+    macos:
+      xcode: '26.4.0'
+    resource_class: m4pro.medium
+
+#workflows:
+#  test-all:
+#    jobs:
+#      - test:
+#          matrix:
+#            parameters:
+#              executor: [linux-arm]

.github/workflows/claude-fixer.yml

@@ -0,0 +1,30 @@
+name: Claude Test Fixer
+on:
+  workflow_run:
+    workflows: ["Test all cron (Linux)", "Test all cron (MacOS)"]
+    types: [completed]
+
+jobs:
+  fix-on-failure:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
+    permissions:
+      contents: write
+      pull-requests: write
+      actions: read # Allows Claude to read the logs of the failed run
+    steps:
+      - uses: actions/checkout@v4
+      - name: Claude Fix Failed Tests
+        uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          prompt: |
+            The "CI Tests" workflow just failed. 
+            1. Analyze the logs from the last failed run.
+            2. Identify the root cause of the test failure.
+            3. Implement a fix and create a new pull request.
+            
+            
+
+          additional_permissions: |
+            actions: read

.github/workflows/run-all-tests-cron-linux.yaml

@@ -0,0 +1,45 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
+
+name: Test all cron (Linux)
+
+on:
+  pull_request:
+    branches:
+      - release
+  schedule:
+    - cron: '0 0 * * *'  # Every day at midnight UTC
+  workflow_dispatch:
+
+jobs:
+  build-and-test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+#        os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest]
+        shard: ["1/20", "2/20", "3/20", "4/20", "5/20", "6/20", "7/20", "8/20", "9/20", "10/20", "11/20", "12/20", "13/20", "14/20", "15/20", "16/20", "17/20", "18/20", "19/20", "20/20"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 24
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24.x'
+          cache: 'npm'
+      - name: Enable linger for admin user (Linux only)
+        if: runner.os == 'Linux'
+        run: loginctl enable-linger $(whoami)
+
+      - run: npm ci
+      - run: npx tsx scripts/cleanup-github-actions.ts
+
+#      - name: Setup tmate session
+#        uses: mxschmitt/action-tmate@v3
+
+      - name: Run tests (Linux)
+        if: runner.os == 'Linux'
+        run: npm run test -- ./test --no-file-parallelism --disable-console-intercept  --shard ${{ matrix.shard }}
+

.github/workflows/run-all-tests-cron-macos.yaml

@@ -0,0 +1,54 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-nodejs
+
+name: Test all cron (MacOS)
+
+on:
+  pull_request:
+    branches:
+      - release
+  schedule:
+    - cron: '0 0 * * *'  # Every day at midnight UTC
+  workflow_dispatch:
+
+jobs:
+  build-and-test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest]
+#        os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest]
+        shard: ["1/5", "2/5", "3/5", "4/5", "5/5"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 24
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24.x'
+          cache: 'npm'
+
+      - run: npm ci
+      - run: npx tsx scripts/cleanup-github-actions.ts
+
+#      - name: Setup tmate session
+#        uses: mxschmitt/action-tmate@v3
+
+      - name: Run tests (macOS - zsh login shell)
+        if: runner.os == 'macOS'
+        shell: zsh {0}
+        run: |
+          sudo chsh -s $(which zsh) $USER
+          echo $0
+
+          echo $ZSH_NAME $ZSH_VERSION
+          export SHELL=/bin/zsh
+          touch ~/.zshrc
+          unset JAVA_HOME
+          export PATH=/Users/runner/.local/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/opt/curl/bin:/usr/local/bin:/usr/local/sbin:/Users/runner/bin:/usr/bin:/bin:/usr/sbin:/sbin
+          export CI=true
+
+          npm run test -- ./test --no-file-parallelism --disable-console-intercept --exclude ./test/homebrew --shard ${{ matrix.shard }}
+

.github/workflows/run-all-unit-tests.yaml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="test:integration:dev -- $FilePathRelativeToProjectRoot$" type="js.build_tools.npm" nameIsGenerated="true">
+  <configuration default="false" name="test:integration:dev -- $FilePathRelativeToProjectRoot$" type="js.build_tools.npm">
     <package-json value="$PROJECT_DIR$/package.json" />
     <command value="run" />
     <scripts>

.run/test_integration_dev -- $FilePathRelativeToProjectRoot$.run.xml

@@ -42,7 +42,7 @@
   "license": "ISC",
   "type": "module",
   "dependencies": {
-    "@codifycli/plugin-core": "1.1.0-beta6",
+    "@codifycli/plugin-core": "1.1.0-beta10",
     "@codifycli/schemas": "1.0.0",
     "ajv": "^8.18.0",
     "ajv-formats": "^2.1.1",
@@ -58,7 +58,7 @@
   "devDependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.2.97",
     "@apidevtools/json-schema-ref-parser": "^11.7.2",
-    "@codifycli/plugin-test": "^1.0.0",
+    "@codifycli/plugin-test": "1.1.0-beta3",
     "@fastify/merge-json-schemas": "^0.2.0",
     "@oclif/prettier-config": "^0.2.1",
     "@oclif/test": "^3",

package-lock.json

@@ -2,7 +2,7 @@ import { JSONSchema } from '@apidevtools/json-schema-ref-parser';
 import { createRequire } from 'node:module';
 import { Ajv } from 'ajv';
 import { VerbosityLevel } from '@codifycli/plugin-core';
-import { SequentialPty } from '@codifycli/plugin-core/dist/pty/seqeuntial-pty';
+import { SequentialPty } from '@codifycli/plugin-core';
 import { IpcMessage, IpcMessageSchema, MessageStatus, ResourceSchema } from '@codifycli/schemas';
 import mergeJsonSchemas from 'merge-json-schemas';
 import { ChildProcess, fork } from 'node:child_process';

package.json

@@ -0,0 +1,33 @@
+import { PluginTester, testSpawn } from '@codifycli/plugin-test';
+import path from 'node:path';
+import { Utils } from '@codifycli/plugin-core';
+
+const pluginPath = path.resolve('./src/index.ts');
+
+
+if (Utils.isLinux()) {
+  // Uninstall resources that have Codify resource definitions
+  await PluginTester.uninstall(pluginPath, [
+    { type: 'docker' },
+    { type: 'aws-cli'}
+  ]);
+
+  await testSpawn('apt-get autoremove -y ruby rpm python awscli needrestart', { requiresRoot: true }); // remove needrestart to keep logs clean.
+
+  await testSpawn('rustup self uninstall -y');
+
+  await testSpawn('rm -rf /usr/bin/go', { requiresRoot: true })
+  await testSpawn('rm -rf /usr/bin/python', { requiresRoot: true })
+  await testSpawn('rm -rf /usr/bin/ruby', { requiresRoot: true })
+
+// await testSpawn('apt install --reinstall command-not-found', { requiresRoot: true });
+
+  // MacOS
+} else {
+  await PluginTester.uninstall(pluginPath, [
+    { type: 'aws-cli' },
+  ]);
+
+  await testSpawn('brew uninstall ant gradle kotlin maven selenium-server google-chrome pipx $(brew list | grep -E \'^python(@|$)\') $(brew list | grep -E \'^ruby(@|$)\') aws-sam-cli azure-cli rustup git-lfs $(brew list | grep -E \'^openjdk(@|$)\')', { interactive: true });
+
+}