feat: Refactored plugin resolution and added an ApiClient for network calls

Author: kevinwang5658

package.json

@@ -22,6 +22,7 @@
     "parse-json": "^8.1.0",
     "react": "^18.3.1",
     "semver": "^7.5.4",
+    "latest-semver": "^4.0.0",
     "supports-color": "^9.4.0",
     "nanoid": "^5.0.9"
   },

src/api/index.ts

@@ -0,0 +1,39 @@
+import * as fsSync from 'node:fs';
+import * as fs from 'node:fs/promises';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { finished } from 'node:stream/promises';
+
+import { PluginSearchQuery, PluginSearchResult } from './types.js';
+
+const API_BASE_URL = 'https://api.codifycli.com'
+
+export const ApiClient = {
+  async searchPlugins(query: PluginSearchQuery[]): Promise<PluginSearchResult> {
+    const body = JSON.stringify({ query });
+    const res = await fetch(
+      `${API_BASE_URL}/v1/plugins/versions/search`,
+      { method: 'POST', body, headers: { 'Content-Type': 'application/json' } }
+    );
+
+    const json = await res.json();
+    if (!res.ok) {
+      throw new Error(JSON.stringify(json, null, 2));
+    }
+
+    return json.results as unknown as PluginSearchResult;
+  },
+
+  async downloadPlugin(filePath: string, url: string): Promise<void> {
+    const { body } = await fetch(url)
+    
+    const dirname = path.dirname(filePath);
+    if (!await fs.stat(dirname).then((s) => s.isDirectory()).catch(() => false)) {
+      await fs.mkdir(dirname, { recursive: true });
+    }
+    
+    const ws = fsSync.createWriteStream(filePath)
+    // Different type definitions here for readable stream (NodeJS vs DOM). Small hack to fix that
+    await finished(Readable.fromWeb(body as never).pipe(ws));
+  },
+};

src/api/types.ts

@@ -0,0 +1,15 @@
+export interface PluginSearchQuery {
+  name: string;
+  version?: string;
+}
+
+export interface PluginSearchResult {
+  [x: string]: PluginInfo;
+}
+
+export interface PluginInfo {
+  version: string;
+  isLatest: boolean;
+  isBeta: boolean;
+  downloadLink: string;
+}

src/plugins/plugin-manager.ts

@@ -130,13 +130,7 @@ export class PluginManager {
       ...project?.projectConfig?.plugins,
     };
 
-    const configPlugins = await Promise.all(Object.entries(pluginDefinitions).map(([name, version]) =>
-      PluginResolver.resolve(name, version)
-    ));
-
-    const existingPlugins = await PluginResolver.resolveExisting(Object.keys(pluginDefinitions));
-
-    return [...configPlugins, ...existingPlugins.filter((p) => !configPlugins.some((p2) => p2.name === p.name))];
+    return PluginResolver.resolveAll(pluginDefinitions);
   }
 
   private async initializePlugins(plugins: Plugin[], secureMode: boolean): Promise<Map<string, string[]>> {

src/plugins/resolver.ts

@@ -1,95 +1,109 @@
+import latestSemver from 'latest-semver';
 import * as fsSync from 'node:fs';
 import * as fs from 'node:fs/promises';
 import * as os from 'node:os';
 import path from 'node:path';
-import { Readable } from 'node:stream';
-import { finished } from 'node:stream/promises';
 
+import { ApiClient } from '../api/index.js';
+import { PluginInfo } from '../api/types.js';
 import { ctx } from '../events/context.js';
 import { Plugin } from './plugin.js';
 
-const DEFAULT_PLUGIN_URL = 'https://codify-plugin-library.s3.amazonaws.com/codify-core/index.js';
 const PLUGIN_CACHE_DIR = path.resolve(os.homedir(), '.codify/plugins')
 
 export class PluginResolver {
 
-  static async resolve(name: string, version: string): Promise<Plugin> {
+  static async resolveAll(definitions: Record<string, string>): Promise<Plugin[]> {
     await PluginResolver.checkAndCreateCacheDirIfNotExists()
 
-    let directoryStat;
-    try {
-      directoryStat = await fs.stat(version);
-    } catch {
-    }
+    const localPluginDefs = Object.entries(definitions)
+      .filter(([k, v]) => v.endsWith('.js') || v.endsWith('.ts'))
+    const localPlugins = await Promise.all(localPluginDefs.map(([name, path]) =>
+      PluginResolver.resolveLocalPlugin(name, path)
+    ))
 
-    // For easier development. A direct js file can be specified for the plugin.
-    if (directoryStat && directoryStat.isFile()) {
-      return PluginResolver.resolvePluginFs(name, version)
-    }
+    const networkPluginDefs = Object.entries(definitions)
+      .filter(([k]) => !localPluginDefs.some(([lk]) => k === lk))
 
-    if (name === 'default') {
-      return PluginResolver.resolvePluginDefault(name, version)
+    if (networkPluginDefs.length === 0) {
+      return localPlugins;
     }
 
-    throw new Error('Non-default plugins are not currently supported');
-  }
-
-  static async resolveExisting(exclude: string[]): Promise<Plugin[]> {
-    let files;
-    try {
-      files = await fs.readdir(PLUGIN_CACHE_DIR);
-    } catch {
-    }
+    // Fetch the latest plugin info from the server
+    const latestPluginInfo = await ApiClient
+      .searchPlugins(networkPluginDefs.map(([name, version]) => ({ name, version })))
+      .catch((e: Error) => {
+        console.warn('Unable to fetch latest plugin info');
+        ctx.debug(`Unable to fetch latest plugin info:\n${e.message}`);
+      }) ?? undefined;
 
-    if (!files) {
-      return [];
-    }
+    const networkPlugins = await Promise.all(networkPluginDefs.map(([name, version]) =>
+      PluginResolver.resolvePluginNetwork(name, version, latestPluginInfo?.[name])
+    ))
 
-    return files
-      .filter((f) => f.endsWith('.js'))
-      .filter((f) => !exclude.includes(getPluginName(f)))
-      .map((f) => {
-        const name = getPluginName(f);
-        const p = path.join(PLUGIN_CACHE_DIR, f);
-
-        return new Plugin(name, '', p);
-      })
-
-    function getPluginName(fileName: string) {
-      return fileName.split('.')[0];
-    }
+    return [...networkPlugins, ...localPlugins];
   }
 
-  private static async resolvePluginFs(name: string, filePath: string): Promise<Plugin> {
+  static async resolveLocalPlugin(name: string, filePath: string, version?: string): Promise<Plugin> {
     const fileExtension = filePath.slice(filePath.lastIndexOf('.'))
     if (fileExtension !== '.js' && fileExtension !== '.ts') {
       throw new Error(`Only .js and .ts plugins are support currently. Can't resolve ${filePath}`);
     }
 
+    let stats: fsSync.Stats;
+    try {
+      stats = await fs.stat(filePath);
+    } catch (e) {
+      throw new Error(`Unable to find plugin file path ${filePath}`)
+    }
+
+    if (!stats.isFile()) {
+      throw new Error(`Provided plugin path ${filePath} does not reference a file`);
+    }
+
     return new Plugin(
       name,
-      '0.0.0',
+      version ?? '0.0.0',
       filePath,
     )
   }
 
-  private static async resolvePluginDefault(name: string, version: string): Promise<Plugin> {
-    const { body } = await fetch(DEFAULT_PLUGIN_URL)
-    if (!body) {
-      throw new Error('Un-able to fetch the default plugin (body not found). Exiting');
+  private static async resolvePluginNetwork(name: string, version: string, latestInfoFromNetwork?: PluginInfo): Promise<Plugin> {
+    const resolvedVersion = (version === 'latest') ? await PluginResolver.resolveLatestLocalVersion(name) : version;
+    if (!resolvedVersion && !latestInfoFromNetwork) {
+      throw new Error(`Plugin ${name} not found and not able to download from registry. Please try again at a later time`);
     }
 
-    const filePath = path.join(PLUGIN_CACHE_DIR, 'default.js');
-    const ws = fsSync.createWriteStream(filePath)
+    if (!resolvedVersion) {
+      return downloadFreshPlugin();
+    }
 
-    // Different type definitions here for readable stream (NodeJS vs DOM). Small hack to fix that
-    await finished(Readable.fromWeb(body as never).pipe(ws));
+    const localPluginExists = await PluginResolver.localPluginExists(name, resolvedVersion);
+    if (!localPluginExists && !latestInfoFromNetwork) {
+      throw new Error(`Plugin ${name} not found and not able to download from registry. Please try again at a later time`);
+    }
 
-    return new Plugin(
-      name,
-      version,
-      filePath,
-    )
+    // Plugin already exists, then no need to download. OR couldn't fetch plugin info from online then just resolve local version. OR we already have the latest version
+    if ((version !== 'latest' && localPluginExists)
+      || (localPluginExists && !latestInfoFromNetwork)
+      || (resolvedVersion === latestInfoFromNetwork!.version)
+    ) {
+      return PluginResolver.resolveLocalPlugin(name, `${PLUGIN_CACHE_DIR}/${name}/${version}/index.js`);
+    }
+
+    return downloadFreshPlugin();
+
+    // Set up folders and download plugin from the network.
+    async function downloadFreshPlugin(): Promise<Plugin> {
+      const filePath = `${PLUGIN_CACHE_DIR}/${name}/${version}/index.js`;
+      await ApiClient.downloadPlugin(filePath, latestInfoFromNetwork!.downloadLink);
+
+      return new Plugin(
+        name,
+        version,
+        filePath,
+      )
+    }
   }
 
   private static async checkAndCreateCacheDirIfNotExists() {
@@ -111,4 +125,24 @@ export class PluginResolver {
     ctx.log('Creating a new cache dir for codify');
     await fs.mkdir(PLUGIN_CACHE_DIR, { recursive: true });
   }
+
+  private static async resolveLatestLocalVersion(name: string): Promise<string | undefined> {
+    try {
+      const pluginPath = path.join(PLUGIN_CACHE_DIR, name);
+      const versions = await fs.readdir(pluginPath);
+      return latestSemver(versions);
+    } catch (e) {
+      return undefined;
+    }
+  }
+
+  private static async localPluginExists(name: string, version: string): Promise<boolean> {
+    const pluginPath = path.join(PLUGIN_CACHE_DIR, name, version, 'index.js');
+    try {
+      const fileStats = await fs.stat(pluginPath)
+      return fileStats.isFile();
+    } catch (e) {
+      return false;
+    }
+  }
 }