flow decorator upserts two versions

Author: Andrei Bratu

src/evals/run.ts

@@ -48,6 +48,7 @@ import { HumanloopRuntimeError } from "../error";
 import { Humanloop, HumanloopClient } from "../index";
 import {
     Dataset,
+    EvalFileType,
     Evaluator,
     EvaluatorCheck,
     File as FileEvalConfig,
@@ -412,7 +413,7 @@ function callableIsHumanloopDecorator<
     I extends Record<string, unknown> & { messages?: any[] },
     O,
 >(file: FileEvalConfig<I, O>): boolean {
-    return file.callable !== undefined && "decorator" in file.callable;
+    return file.callable !== undefined && "file" in file.callable;
 }
 
 function fileOrFileInsideHLUtility<
@@ -476,7 +477,7 @@ function getFileType<I extends Record<string, unknown> & { messages?: any[] }, O
 ): FileEvalConfig<I, O> {
     // Determine the `type` of the `file` to Evaluate - if not `type` provided, default to `flow`
     try {
-        let type_ = file.type as FileType;
+        let type_ = file.type as EvalFileType;
         console.info(
             `${CYAN}Evaluating your ${type_} function corresponding to \`${file.path || file.id}\` on Humanloop${RESET}\n\n`,
         );
@@ -541,7 +542,7 @@ async function getHLFile<I extends Record<string, unknown> & { message?: any[] }
     fileConfig: FileEvalConfig<I, O>,
 ): Promise<[EvaluatedFile, FileEvalConfig<I, O>["callable"]]> {
     let file_ = fileOrFileInsideHLUtility(fileConfig);
-    file_ = getFileType(fileConfig);
+    file_ = getFileType(file_);
 
     return await resolveFile(client, file_);
 }
@@ -626,14 +627,27 @@ async function resolveFile<I extends Record<string, unknown> & { message?: any[]
             `You are trying to create a new version of the File by passing the ${callable} argument. You must pass either the \`file.path\` or \`file.fileId\` argument and provide proper \`file.version\` for upserting the File.`,
         );
     }
-    let hlFile = await safeGetDefaultFileVersion(client, fileConfig);
 
     if ((versionId || environment) && (callable || version)) {
         throw new HumanloopRuntimeError(
             "You are trying to create a local Evaluation while requesting a specific File version by version ID or environment.",
         );
     }
 
+    let hlFile: EvaluatedFile;
+
+    try {
+        hlFile = (await safeGetDefaultFileVersion(client, fileConfig)) as EvaluatedFile;
+    } catch (error: any) {
+        if (!version || !path || fileId) {
+            throw new HumanloopRuntimeError(
+                "File does not exist on Humanloop. Please provide a `file.path` and a version to create a new version.",
+            );
+        }
+        console.log("UPSERTING FILE", JSON.stringify(fileConfig, null, 2));
+        return [await upsertFile(client, fileConfig), callable];
+    }
+
     if (version) {
         // User responsibility to provide adequate file.version for upserting the file
         console.info(
@@ -716,7 +730,7 @@ async function upsertFile<I extends Record<string, unknown> & { messages?: any[]
     switch (type) {
         case "flow":
             // Be more lenient with Flow versions as they are arbitrary json
-            const flowVersion = { attributes: version };
+            const flowVersion = version ? { ...version } : { attributes: {} };
             const fileDictWithFlowVersion = { ...fileConfig, ...flowVersion };
             hlFile = await client.flows.upsert(fileDictWithFlowVersion as FlowRequest);
             break;
@@ -726,9 +740,6 @@ async function upsertFile<I extends Record<string, unknown> & { messages?: any[]
         case "tool":
             hlFile = await client.tools.upsert(fileDict as ToolRequest);
             break;
-        case "evaluator":
-            hlFile = await client.evaluators.upsert(fileDict as EvaluatorRequest);
-            break;
         default:
             throw new HumanloopRuntimeError(`Unsupported File type: ${type}`);
     }

src/evals/types.ts

@@ -50,10 +50,12 @@ interface Identifiers {
     environment?: string;
 }
 
+export type EvalFileType = "prompt" | "tool" | "flow" | "agent";
+
 export interface File<I extends Record<string, unknown> & { messages?: any[] }, O>
     extends Identifiers {
     /** The type of File this callable relates to on Humanloop. */
-    type?: FileType;
+    type?: EvalFileType;
     /** The contents uniquely define the version of the File on Humanloop. */
     version?: Version;
     /**