Skip to content

Commit 1a20d7e

Browse files
AlePouroullisAlePouroullis
committed
test: Write integration tests for FileSyncer:
1 parent 58d3edd commit 1a20d7e

2 files changed

Lines changed: 246 additions & 5 deletions

File tree

‎tests/custom/integration/FileSyncer.test.ts‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
import * as fs from "fs";
2+
import * as path from "path";
3+
import { v4 as uuidv4 } from "uuid";
4+
5+
import { FileType } from "../../../src/api";
6+
import { HumanloopRuntimeError } from "../../../src/error";
7+
import { HumanloopClient } from "../../../src/humanloop.client";
8+
import { createTempDir } from "../fixtures";
9+
import {
10+
SyncableFile,
11+
TestSetup,
12+
cleanupTestEnvironment,
13+
createSyncableFilesFixture,
14+
setupTestEnvironment,
15+
} from "./fixtures";
16+
17+
describe("FileSyncer Integration Tests", () => {
18+
let testSetup: TestSetup;
19+
let syncableFiles: SyncableFile[] = [];
20+
let tempDirInfo: { tempDir: string; cleanup: () => void };
21+
22+
beforeAll(async () => {
23+
// Set up test environment
24+
testSetup = await setupTestEnvironment("file_sync");
25+
tempDirInfo = createTempDir("file-sync-integration");
26+
27+
// Create test files in Humanloop for syncing
28+
syncableFiles = await createSyncableFilesFixture(testSetup);
29+
});
30+
31+
afterAll(async () => {
32+
// Clean up resources only if they were created
33+
if (tempDirInfo) {
34+
tempDirInfo.cleanup();
35+
}
36+
if (testSetup) {
37+
await cleanupTestEnvironment(
38+
testSetup,
39+
syncableFiles.map((file) => ({
40+
type: file.type as FileType,
41+
id: file.id as string,
42+
})),
43+
);
44+
}
45+
});
46+
47+
test("pull_basic: should pull all files from remote to local filesystem", async () => {
48+
// GIVEN a set of files in the remote system (from syncableFiles)
49+
const client = new HumanloopClient({
50+
apiKey: process.env.HUMANLOOP_API_KEY,
51+
localFilesDirectory: tempDirInfo.tempDir,
52+
useLocalFiles: true,
53+
});
54+
55+
// WHEN running the pull operation
56+
await client.pull();
57+
58+
// THEN our local filesystem should mirror the remote filesystem in the HL Workspace
59+
for (const file of syncableFiles) {
60+
const extension = `.${file.type}`;
61+
const localPath = path.join(
62+
tempDirInfo.tempDir,
63+
`${file.path}${extension}`,
64+
);
65+
66+
// THEN the file and its directory should exist
67+
expect(fs.existsSync(localPath)).toBe(true);
68+
expect(fs.existsSync(path.dirname(localPath))).toBe(true);
69+
70+
// THEN the file should not be empty
71+
const content = fs.readFileSync(localPath, "utf8");
72+
expect(content).toBeTruthy();
73+
}
74+
});
75+
76+
test("pull_with_invalid_path: should handle error when path doesn't exist", async () => {
77+
// GIVEN a client
78+
const client = new HumanloopClient({
79+
apiKey: process.env.HUMANLOOP_API_KEY,
80+
localFilesDirectory: tempDirInfo.tempDir,
81+
useLocalFiles: true,
82+
});
83+
84+
const nonExistentPath = `${testSetup.sdkTestDir.path}/non_existent_directory`;
85+
86+
// WHEN/THEN pulling with an invalid path should throw an error
87+
await expect(client.pull(nonExistentPath)).rejects.toThrow(
88+
HumanloopRuntimeError,
89+
);
90+
// The error message might be different in TypeScript, so we don't assert on the exact message
91+
});
92+
93+
test("pull_with_invalid_environment: should handle error when environment doesn't exist", async () => {
94+
// GIVEN a client
95+
const client = new HumanloopClient({
96+
apiKey: process.env.HUMANLOOP_API_KEY,
97+
localFilesDirectory: tempDirInfo.tempDir,
98+
useLocalFiles: true,
99+
});
100+
101+
// WHEN/THEN pulling with an invalid environment should throw an error
102+
await expect(client.pull(undefined, "invalid_environment")).rejects.toThrow(
103+
HumanloopRuntimeError,
104+
);
105+
});
106+
107+
test("pull_with_path_filter: should only pull files from specified path", async () => {
108+
// GIVEN a client and a clean temp directory
109+
const pathFilterTempDir = createTempDir("file-sync-path-filter");
110+
111+
const client = new HumanloopClient({
112+
apiKey: process.env.HUMANLOOP_API_KEY,
113+
localFilesDirectory: pathFilterTempDir.tempDir,
114+
useLocalFiles: true,
115+
});
116+
117+
// WHEN pulling only files from the testSetup.sdkTestDir.path
118+
await client.pull(testSetup.sdkTestDir.path);
119+
120+
// THEN count the total number of files pulled
121+
let pulledFileCount = 0;
122+
123+
// Collect expected file paths (relative to sdkTestDir.path)
124+
const expectedFiles = new Set(
125+
syncableFiles.map((file) =>
126+
path.join(
127+
pathFilterTempDir.tempDir,
128+
file.path + (file.type === "prompt" ? ".prompt" : ".agent"),
129+
),
130+
),
131+
);
132+
133+
const foundFiles = new Set<string>();
134+
135+
function countFilesRecursive(dirPath: string): void {
136+
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
137+
for (const entry of entries) {
138+
const fullPath = path.join(dirPath, entry.name);
139+
if (entry.isDirectory()) {
140+
countFilesRecursive(fullPath);
141+
} else if (entry.isFile()) {
142+
if (expectedFiles.has(fullPath)) {
143+
const content = fs.readFileSync(fullPath, "utf8");
144+
expect(content).toBeTruthy();
145+
foundFiles.add(fullPath);
146+
}
147+
}
148+
}
149+
}
150+
151+
if (fs.existsSync(pathFilterTempDir.tempDir)) {
152+
countFilesRecursive(pathFilterTempDir.tempDir);
153+
}
154+
155+
expect(foundFiles.size).toBe(expectedFiles.size);
156+
157+
// Clean up
158+
pathFilterTempDir.cleanup();
159+
});
160+
});

‎tests/custom/integration/fixtures.ts‎

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { v4 as uuidv4 } from "uuid";
55
import { FileType, PromptRequest, PromptResponse } from "../../../src/api";
66
import { HumanloopClient } from "../../../src/humanloop.client";
77

8-
export interface TestIdentifiers {
8+
export interface ResourceIdentifiers {
99
id: string;
1010
path: string;
1111
}
@@ -16,15 +16,23 @@ export interface TestPrompt {
1616
response: PromptResponse;
1717
}
1818

19+
export interface SyncableFile {
20+
path: string;
21+
type: "prompt" | "agent";
22+
model: string;
23+
id?: string;
24+
versionId?: string;
25+
}
26+
1927
export interface TestSetup {
20-
sdkTestDir: TestIdentifiers;
28+
sdkTestDir: ResourceIdentifiers;
2129
testPromptConfig: PromptRequest;
2230
openaiApiKey: string;
2331
humanloopClient: HumanloopClient;
24-
evalDataset: TestIdentifiers;
25-
evalPrompt: TestIdentifiers;
32+
evalDataset: ResourceIdentifiers;
33+
evalPrompt: ResourceIdentifiers;
2634
stagingEnvironmentId: string;
27-
outputNotNullEvaluator: TestIdentifiers;
35+
outputNotNullEvaluator: ResourceIdentifiers;
2836
}
2937

3038
export interface CleanupResources {
@@ -244,3 +252,76 @@ export async function cleanupTestEnvironment(
244252
console.error("Error during cleanup:", error);
245253
}
246254
}
255+
256+
/**
257+
* Creates a predefined structure of files in Humanloop for testing sync,
258+
* mirroring the Python syncable_files_fixture
259+
*/
260+
export async function createSyncableFilesFixture(
261+
testSetup: TestSetup,
262+
): Promise<SyncableFile[]> {
263+
const fileDefinitions: SyncableFile[] = [
264+
{
265+
path: "prompts/gpt-4",
266+
type: "prompt",
267+
model: "gpt-4o-mini", // Using gpt-4o-mini as safer default for tests
268+
},
269+
{
270+
path: "prompts/gpt-4o",
271+
type: "prompt",
272+
model: "gpt-4o-mini",
273+
},
274+
{
275+
path: "prompts/nested/complex/gpt-4o",
276+
type: "prompt",
277+
model: "gpt-4o-mini",
278+
},
279+
{
280+
path: "agents/gpt-4",
281+
type: "agent",
282+
model: "gpt-4o-mini",
283+
},
284+
{
285+
path: "agents/gpt-4o",
286+
type: "agent",
287+
model: "gpt-4o-mini",
288+
},
289+
];
290+
291+
const createdFiles: SyncableFile[] = [];
292+
293+
for (const file of fileDefinitions) {
294+
const fullPath = `${testSetup.sdkTestDir.path}/${file.path}`;
295+
let response;
296+
297+
try {
298+
if (file.type === "prompt") {
299+
response = await testSetup.humanloopClient.prompts.upsert({
300+
path: fullPath,
301+
...testSetup.testPromptConfig,
302+
model: file.model,
303+
});
304+
} else if (file.type === "agent") {
305+
// Assuming agent creation works similar to your Python implementation
306+
response = await testSetup.humanloopClient.agents.upsert({
307+
path: fullPath,
308+
model: file.model,
309+
});
310+
}
311+
312+
if (response) {
313+
createdFiles.push({
314+
path: fullPath,
315+
type: file.type,
316+
model: file.model,
317+
id: response.id,
318+
versionId: response.versionId,
319+
});
320+
}
321+
} catch (error) {
322+
console.warn(`Failed to create ${file.type} at ${fullPath}: ${error}`);
323+
}
324+
}
325+
326+
return createdFiles;
327+
}

0 commit comments

Comments
 (0)