add support for agents + online files eval

Author: Andrei Bratu

src/evals/run.ts

@@ -11,6 +11,7 @@ import cliProgress from "cli-progress";
 import _, { capitalize } from "lodash";
 
 import {
+    AgentResponse,
     BooleanEvaluatorStatsResponse,
     CreateEvaluationRequestEvaluatorsItem,
     DatapointResponse,
@@ -216,6 +217,13 @@ function getFileCallable<I extends Record<string, unknown> & { messages?: any[]
     return function_;
 }
 
+type EvaluatedFile =
+    | PromptResponse
+    | FlowResponse
+    | ToolResponse
+    | EvaluatorResponse
+    | AgentResponse;
+
 export async function runEval<
     I extends Record<string, unknown> & { messages?: any[] },
     O,
@@ -249,14 +257,35 @@ export async function runEval<
         }
     }
 
-    let hlFile: PromptResponse | FlowResponse | ToolResponse | EvaluatorResponse;
+    // The assert promises the compiler hlFile will be initialized before use
+    // This is handled in the section below, but the compiler is stubborn
+    let hlFile!: EvaluatedFile;
+    let fileExists = false;
+    try {
+        const path_ = file_.path;
+        if (path_) {
+            // @ts-ignore retrieveByPath returns a polymorphic type that is equivalent with FileResponses above
+            hlFile = await client.files.retrieveByPath({ path: path_ });
+            if (hlFile.type !== type_) {
+                throw new HumanloopRuntimeError(
+                    `Error in your \`file\` argument: The file type ${type_} does ` +
+                        `not match the type of the File at ${path_}: ${hlFile.type}.`,
+                );
+            }
+            fileExists = true;
+        }
+    } catch (e: any) {
+        // File does not exist, try upserting it
+    }
+    if (!fileExists) {
     try {
         hlFile = await upsertFile({ file: file_, type: type_, client: client });
     } catch (e: any) {
         console.error(
             `${RED}Error in your \`file\` argument:\n\n${e.constructor.name}: ${e.message}${RESET}`,
         );
         return [];
+        }
     }
 
     let hlDataset: DatasetResponse;
@@ -645,7 +674,7 @@ async function getNewRun({
     client: HumanloopClient;
     evaluationName: string | undefined;
     evaluators: Evaluator[];
-    hlFile: PromptResponse | FlowResponse | ToolResponse | EvaluatorResponse;
+    hlFile: EvaluatedFile;
     hlDataset: DatasetResponse;
     func: ((inputs: Record<string, unknown>) => Promise<unknown>) | undefined;
 }): Promise<{ evaluation: EvaluationResponse; run: EvaluationRunResponse }> {

src/evals/types.ts

@@ -50,7 +50,7 @@ interface Identifiers {
 export interface File<I extends Record<string, unknown> & { messages?: any[] }, O>
     extends Identifiers {
     /** The type of File this callable relates to on Humanloop. */
-    type?: "flow" | "prompt";
+    type?: "flow" | "prompt" | "agent";
     /** The contents uniquely define the version of the File on Humanloop. */
     version?: Version;
     /**
@@ -117,7 +117,7 @@ export interface LocalEvaluator<ReturnType, ArgsType> extends Evaluator {
  * @param dataset - The dataset to map your function over to produce the outputs required by the Evaluation.
  * @param name - The name of the Evaluation to run. If it does not exist, a new Evaluation will be created under your File.
  * @param evaluators - Define how judgments are provided for this Evaluation.
- * @param workers - The number of threads to process datapoints using your function concurrently.
+ * @param workers - The number of datapoints to process concurrently.
  * @returns Per Evaluator checks.
  */
 export interface EvaluatorCheck {