Skip to content

Commit 941693b

Browse files
refactor: standardize max_turns to 3 across all evaluation datasets and remove redundant cloud-sql-instance-not-found entry
1 parent 42ce176 commit 941693b

1 file changed

Lines changed: 4 additions & 18 deletions

File tree

evals/dataset.json

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"GOOGLE_CLOUD_PROJECT": "ext-test-cloud-sql-postgres"
1313
},
1414
"kind": "tools",
15-
"max_turns": 4
15+
"max_turns": 3
1616
},
1717
{
1818
"id": "cloud-sql-schema-tables-explore",
@@ -26,7 +26,7 @@
2626
"GOOGLE_CLOUD_PROJECT": "ext-test-cloud-sql-postgres"
2727
},
2828
"kind": "tools",
29-
"max_turns": 6
29+
"max_turns": 3
3030
},
3131
{
3232
"id": "cloud-sql-performance-check",
@@ -40,7 +40,7 @@
4040
"GOOGLE_CLOUD_PROJECT": "ext-test-cloud-sql-postgres"
4141
},
4242
"kind": "tools",
43-
"max_turns": 6
43+
"max_turns": 3
4444
},
4545
{
4646
"id": "cloud-sql-metrics-cpu-investigation",
@@ -54,21 +54,7 @@
5454
"GOOGLE_CLOUD_PROJECT": "ext-test-cloud-sql-postgres"
5555
},
5656
"kind": "tools",
57-
"max_turns": 6
58-
},
59-
{
60-
"id": "cloud-sql-instance-not-found",
61-
"starting_prompt": "Get details for the instance 'missing-db-123'.",
62-
"conversation_plan": "The user asks for details of an instance named 'missing-db-123' that doesn't exist. The agent should try to get it, fail, and inform the user. The user will then ask to list instances to find the correct name.",
63-
"expected_trajectory": [
64-
"get_instance",
65-
"list_instances"
66-
],
67-
"env": {
68-
"GOOGLE_CLOUD_PROJECT": "ext-test-cloud-sql-postgres"
69-
},
70-
"kind": "tools",
71-
"max_turns": 4
57+
"max_turns": 3
7258
}
7359
]
7460
}

0 commit comments

Comments
 (0)