test for prompt decorator

Author: Andrei Bratu

tests/integration/decorators.test.ts

@@ -0,0 +1,68 @@
+import {
+    TestPrompt,
+    TestSetup,
+    cleanupTestEnvironment,
+    createTestPrompt,
+    deleteTestPrompt,
+    setupTestEnvironment,
+    testPromptDecorator,
+} from "./test-utils";
+
+// Long timeout per test
+jest.setTimeout(30 * 1000);
+
+describe("decorators", () => {
+    let testSetup: TestSetup;
+    let testPrompt: TestPrompt | null = null;
+
+    beforeAll(async () => {
+        testSetup = await setupTestEnvironment("decorators");
+    });
+
+    afterAll(async () => {
+        await cleanupTestEnvironment(testSetup);
+    });
+
+    afterEach(async () => {
+        // Clean up individual test resources
+        if (testPrompt) {
+            await deleteTestPrompt(testSetup, testPrompt.id);
+            testPrompt = null;
+        }
+    });
+
+    it("should create a prompt log when using the decorator", async () => {
+        try {
+            // Create test prompt
+            testPrompt = await createTestPrompt(testSetup);
+
+            // Check initial version count
+            const promptVersionsResponse =
+                await testSetup.humanloopClient.prompts.listVersions(testPrompt.id);
+            expect(promptVersionsResponse.records.length).toBe(1);
+
+            // Test the prompt decorator
+            await testPromptDecorator(testSetup, testPrompt);
+
+            // Verify a new version was created
+            const updatedPromptVersionsResponse =
+                await testSetup.humanloopClient.prompts.listVersions(testPrompt.id);
+            expect(updatedPromptVersionsResponse.records.length).toBe(2);
+
+            // Verify logs were created
+            const logsResponse = await testSetup.humanloopClient.logs.list({
+                fileId: testPrompt.id,
+                page: 1,
+                size: 50,
+            });
+            expect(logsResponse.data.length).toBe(1);
+        } catch (error) {
+            // Make sure to clean up if the test fails
+            if (testPrompt) {
+                await deleteTestPrompt(testSetup, testPrompt.id);
+                testPrompt = null;
+            }
+            throw error;
+        }
+    });
+});

tests/integration/test-utils.ts

@@ -0,0 +1,174 @@
+import dotenv from "dotenv";
+import { OpenAI } from "openai";
+import { v4 as uuidv4 } from "uuid";
+
+import { PromptRequest, PromptResponse } from "../../src/api";
+import { HumanloopClient } from "../../src/humanloop.client";
+
+export interface TestIdentifiers {
+    id: string;
+    path: string;
+}
+
+export interface TestPrompt {
+    id: string;
+    path: string;
+    response: PromptResponse;
+}
+
+export interface TestSetup {
+    sdkTestDir: TestIdentifiers;
+    testPromptConfig: PromptRequest;
+    openaiApiKey: string;
+    humanloopClient: HumanloopClient;
+}
+
+export function readEnvironment(): void {
+    if (![process.env.HUMANLOOP_API_KEY, process.env.OPENAI_API_KEY].every(Boolean)) {
+        // Testing locally not in CI, running dotenv.config() would override the secrets set for GitHub Action
+        dotenv.config({});
+    }
+    if (!process.env.HUMANLOOP_API_KEY) {
+        throw new Error("HUMANLOOP_API_KEY is not set");
+    }
+    if (!process.env.OPENAI_API_KEY) {
+        throw new Error("OPENAI_API_KEY is not set for integration tests");
+    }
+}
+
+export async function setupTestEnvironment(testName: string): Promise<TestSetup> {
+    readEnvironment();
+
+    const openaiApiKey = process.env.OPENAI_API_KEY!;
+    const humanloopClient = new HumanloopClient({
+        apiKey: process.env.HUMANLOOP_API_KEY,
+        instrumentProviders: {
+            OpenAI: OpenAI,
+        },
+    });
+
+    // Create a test directory
+    const directoryPath = `SDK_TEST_${testName}_${uuidv4()}`;
+    const response = await humanloopClient.directories.create({
+        path: directoryPath,
+    });
+
+    const sdkTestDir = {
+        id: response.id,
+        path: response.path,
+    };
+
+    // Create test prompt config
+    const testPromptConfig: PromptRequest = {
+        provider: "openai",
+        model: "gpt-4o-mini",
+        temperature: 0.5,
+        template: [
+            {
+                role: "system",
+                content: "You are a helpful assistant. Answer concisely.",
+            },
+            {
+                role: "user",
+                content: "{{question}}",
+            },
+        ],
+    };
+
+    return {
+        sdkTestDir,
+        testPromptConfig,
+        openaiApiKey,
+        humanloopClient,
+    };
+}
+
+/**
+ * Creates a test prompt in the specified test environment
+ */
+export async function createTestPrompt(
+    setup: TestSetup,
+    name: string = "test_prompt",
+    customConfig?: Partial<PromptRequest>,
+): Promise<TestPrompt> {
+    const promptPath = `${setup.sdkTestDir.path}/${name}`;
+    const config = customConfig
+        ? { ...setup.testPromptConfig, ...customConfig }
+        : setup.testPromptConfig;
+
+    const promptResponse = await setup.humanloopClient.prompts.upsert({
+        path: promptPath,
+        ...config,
+    });
+
+    return {
+        id: promptResponse.id,
+        path: promptPath,
+        response: promptResponse,
+    };
+}
+
+/**
+ * Creates a base function for LLM calls that can be decorated
+ */
+export function createBaseLLMFunction(setup: TestSetup, model: string = "gpt-4o-mini") {
+    return async (question: string): Promise<string> => {
+        const openaiClient = new OpenAI({ apiKey: setup.openaiApiKey });
+
+        const response = await openaiClient.chat.completions.create({
+            model: model,
+            messages: [{ role: "user", content: question }],
+        });
+
+        return response.choices[0].message.content || "";
+    };
+}
+
+/**
+ * Applies the prompt decorator to a function and tests it
+ */
+export async function testPromptDecorator(
+    setup: TestSetup,
+    prompt: TestPrompt,
+    input: string = "What is the capital of the France?",
+    expectedSubstring: string = "paris",
+): Promise<void> {
+    // Create the base function
+    const myPromptBase = createBaseLLMFunction(setup);
+
+    // Apply the higher-order function instead of decorator
+    const myPrompt = setup.humanloopClient.prompt({
+        path: prompt.path,
+        callable: myPromptBase,
+    });
+
+    // Call the decorated function
+    const result = await myPrompt(input);
+    if (result) {
+        expect(result.toLowerCase()).toContain(expectedSubstring.toLowerCase());
+    } else {
+        throw new Error("Expected result to be defined");
+    }
+
+    // Wait for 5 seconds for the log to be created
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+}
+
+/**
+ * Deletes a test prompt
+ */
+export async function deleteTestPrompt(
+    setup: TestSetup,
+    promptId: string,
+): Promise<void> {
+    if (promptId) {
+        await setup.humanloopClient.prompts.delete(promptId);
+    }
+}
+
+export async function cleanupTestEnvironment(setup: TestSetup): Promise<void> {
+    // Clean up the test directory
+    if (setup.sdkTestDir.id) {
+        await setup.humanloopClient.directories.delete(setup.sdkTestDir.id);
+    }
+}