Skip to content

Commit 8725ce0

Browse files
AAgnihotryAAgnihotry
authored
feat: add Simulate Input span to input mocker with observability attributes (#1519)
1 parent b78b498 commit 8725ce0

16 files changed

Lines changed: 2928 additions & 3 deletions

File tree

‎packages/uipath/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "uipath"
3-
version = "2.10.33"
3+
version = "2.10.34"
44
description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
55
readme = { file = "README.md", content-type = "text/markdown" }
66
requires-python = ">=3.11"

‎packages/uipath/samples/greeter/README.md‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Greeter Sample - Simulated Input Testing
2+
3+
This sample demonstrates LLM-based input generation (simulation) during evaluations and proper span instrumentation.
4+
5+
## Overview
6+
7+
The greeter function generates personalized greetings based on:
8+
- **name**: The person's name
9+
- **greeting_style**: The style of greeting (formal, casual, enthusiastic)
10+
11+
## Running the Function
12+
13+
```bash
14+
# From the greeter directory
15+
uv run uipath run main '{"name": "Alice", "greeting_style": "formal"}'
16+
```
17+
18+
**Expected output:**
19+
```json
20+
{
21+
"message": "Good day, Alice. It is a pleasure to meet you.",
22+
"recipient": "Alice",
23+
"style": "formal"
24+
}
25+
```
26+
27+
## Running Evaluations
28+
29+
### Option 1: Basic Evaluation (No Authentication Required)
30+
31+
Test the function with predefined inputs - no LLM calls needed:
32+
33+
```bash
34+
uv run uipath eval main ./evaluations/eval-sets/basic.json --output-file output-basic.json
35+
```
36+
37+
This will run 3 test cases with fixed inputs and verify the outputs match expected results.
38+
39+
### Option 2: Simulated Input Evaluation (Requires Authentication)
40+
41+
The `simulated-input.json` eval set demonstrates LLM-based input generation:
42+
43+
```bash
44+
# First, authenticate with UiPath
45+
uipath auth
46+
47+
# Or set environment variables
48+
export UIPATH_URL=https://your-tenant.uipath.com
49+
export UIPATH_ACCESS_TOKEN=your-token
50+
51+
# Then run the simulation eval
52+
uv run uipath eval main ./evaluations/eval-sets/simulated-input.json --output-file output.json
53+
```
54+
55+
### How Simulated Input Works
56+
57+
Each evaluation in `simulated-input.json` has:
58+
- `"inputs": {}` - Empty inputs
59+
- `"inputMockingStrategy"` - Configuration for LLM-based input generation with a prompt
60+
61+
The evaluation framework will:
62+
1. Extract the function's input schema from the dataclass (`GreeterInput`)
63+
2. Call an LLM with the generation prompt and schema
64+
3. Parse the LLM's response to generate valid inputs
65+
4. Execute the function with the generated inputs
66+
5. Evaluate the output against expected results
67+
68+
### Observability - "Simulate Input" Span
69+
70+
When running evals with input simulation, you'll see a **"Simulate Input"** span in the traces with these attributes:
71+
72+
**Custom attributes:**
73+
- `span_type`: `"simulatedInput"`
74+
- `type`: `"simulatedInput"`
75+
- `uipath.custom_instrumentation`: `true`
76+
77+
**Standard @traced attributes:**
78+
- `input.mime_type`: `"application/json"`
79+
- `input.value`: JSON with `mocking_strategy`, `input_schema`, `expected_behavior`, `expected_output`
80+
- `output.mime_type`: `"application/json"`
81+
- `output.value`: The generated input (LLM response)
82+
83+
This span matches the pattern used in the uipath-agents-python repository.

‎packages/uipath/samples/greeter/bindings.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"version": "2.0",
3+
"resources": []
4+
}

‎packages/uipath/samples/greeter/evaluations/eval-sets/basic.json‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
{
2+
"version": "1.0",
3+
"id": "BasicGreeterEvalSet",
4+
"name": "Basic Greeter Evaluation Set (No Simulation)",
5+
"evaluatorRefs": [
6+
"JsonSimilarityEvaluator"
7+
],
8+
"evaluations": [
9+
{
10+
"id": "test-formal-greeting",
11+
"name": "Test Formal Greeting",
12+
"inputs": {
13+
"name": "Alice",
14+
"greeting_style": "formal"
15+
},
16+
"evaluationCriterias": {
17+
"JsonSimilarityEvaluator": {
18+
"expectedOutput": {
19+
"message": "Good day, Alice. It is a pleasure to meet you.",
20+
"recipient": "Alice",
21+
"style": "formal"
22+
}
23+
}
24+
}
25+
},
26+
{
27+
"id": "test-casual-greeting",
28+
"name": "Test Casual Greeting",
29+
"inputs": {
30+
"name": "Bob",
31+
"greeting_style": "casual"
32+
},
33+
"evaluationCriterias": {
34+
"JsonSimilarityEvaluator": {
35+
"expectedOutput": {
36+
"message": "Hey Bob, what's up?",
37+
"recipient": "Bob",
38+
"style": "casual"
39+
}
40+
}
41+
}
42+
},
43+
{
44+
"id": "test-enthusiastic-greeting",
45+
"name": "Test Enthusiastic Greeting",
46+
"inputs": {
47+
"name": "Charlie",
48+
"greeting_style": "enthusiastic"
49+
},
50+
"evaluationCriterias": {
51+
"JsonSimilarityEvaluator": {
52+
"expectedOutput": {
53+
"message": "Wow! Hi Charlie!! So great to see you!!!",
54+
"recipient": "Charlie",
55+
"style": "enthusiastic"
56+
}
57+
}
58+
}
59+
}
60+
]
61+
}

‎packages/uipath/samples/greeter/evaluations/eval-sets/simulated-input.json‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
{
2+
"version": "1.0",
3+
"id": "SimulatedInputEvalSet",
4+
"name": "Greeter Simulated Input Evaluation Set",
5+
"evaluatorRefs": [
6+
"JsonSimilarityEvaluator"
7+
],
8+
"evaluations": [
9+
{
10+
"id": "test-formal-greeting-simulated",
11+
"name": "Test Formal Greeting with Simulated Input",
12+
"inputs": {},
13+
"evaluationCriterias": {
14+
"JsonSimilarityEvaluator": {
15+
"expectedOutput": {
16+
"message": "Good day, Alice. It is a pleasure to meet you.",
17+
"recipient": "Alice",
18+
"style": "formal"
19+
}
20+
}
21+
},
22+
"inputMockingStrategy": {
23+
"prompt": "Generate a formal greeting request for a person named Alice. The name field must be exactly 'Alice' and the greeting_style field must be exactly 'formal'."
24+
}
25+
},
26+
{
27+
"id": "test-casual-greeting-simulated",
28+
"name": "Test Casual Greeting with Simulated Input",
29+
"inputs": {},
30+
"evaluationCriterias": {
31+
"JsonSimilarityEvaluator": {
32+
"expectedOutput": {
33+
"message": "Hey Bob, what's up?",
34+
"recipient": "Bob",
35+
"style": "casual"
36+
}
37+
}
38+
},
39+
"inputMockingStrategy": {
40+
"prompt": "Generate a casual greeting request for a person named Bob. The name field must be exactly 'Bob' and the greeting_style field must be exactly 'casual'."
41+
}
42+
},
43+
{
44+
"id": "test-enthusiastic-greeting-simulated",
45+
"name": "Test Enthusiastic Greeting with Simulated Input",
46+
"inputs": {},
47+
"evaluationCriterias": {
48+
"JsonSimilarityEvaluator": {
49+
"expectedOutput": {
50+
"message": "Wow! Hi Charlie!! So great to see you!!!",
51+
"recipient": "Charlie",
52+
"style": "enthusiastic"
53+
}
54+
}
55+
},
56+
"inputMockingStrategy": {
57+
"prompt": "Generate an enthusiastic greeting request. The name field must be exactly 'Charlie' and the greeting_style field must be exactly 'enthusiastic'."
58+
}
59+
}
60+
]
61+
}

‎packages/uipath/samples/greeter/evaluations/evaluators/json-similarity.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"version": "1.0",
3+
"id": "JsonSimilarityEvaluator",
4+
"description": "Checks if the response JSON is similar to the expected JSON structure",
5+
"evaluatorTypeId": "uipath-json-similarity",
6+
"evaluatorConfig": {
7+
"name": "JsonSimilarityEvaluator",
8+
"targetOutputKey": "*",
9+
"defaultEvaluationCriteria": {
10+
"expectedOutput": {
11+
"message": "Good day, Alice. It is a pleasure to meet you.",
12+
"recipient": "Alice",
13+
"style": "formal"
14+
}
15+
}
16+
}
17+
}

‎packages/uipath/samples/greeter/main.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""Simple greeter function for testing input simulation."""
2+
3+
from pydantic.dataclasses import dataclass
4+
5+
6+
@dataclass
7+
class GreeterInput:
8+
"""Input for the greeter function."""
9+
10+
name: str
11+
greeting_style: str = "formal"
12+
13+
14+
@dataclass
15+
class GreeterOutput:
16+
"""Output from the greeter function."""
17+
18+
message: str
19+
recipient: str
20+
style: str
21+
22+
23+
def main(input: GreeterInput) -> GreeterOutput:
24+
"""Generate a greeting based on the name and style.
25+
26+
Args:
27+
input: The greeter input containing name and greeting_style
28+
29+
Returns:
30+
A GreeterOutput containing the greeting message
31+
"""
32+
greetings = {
33+
"formal": f"Good day, {input.name}. It is a pleasure to meet you.",
34+
"casual": f"Hey {input.name}, what's up?",
35+
"enthusiastic": f"Wow! Hi {input.name}!! So great to see you!!!",
36+
}
37+
38+
greeting = greetings.get(input.greeting_style, greetings["formal"])
39+
40+
return GreeterOutput(
41+
message=greeting, recipient=input.name, style=input.greeting_style
42+
)

‎packages/uipath/samples/greeter/output-basic.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"evaluationSetName": "Basic Greeter Evaluation Set (No Simulation)", "evaluationSetResults": [{"evaluationName": "Test Formal Greeting", "evaluationRunResults": [{"evaluatorName": "JsonSimilarityEvaluator", "evaluatorId": "JsonSimilarityEvaluator", "result": {"score": 1.0, "details": {"expected": "{'message': 'Good day, Alice. It is a pleasure to meet you.', 'recipient': 'Alice', 'style': 'formal'}", "actual": "{'message': 'Good day, Alice. It is a pleasure to meet you.', 'recipient': 'Alice', 'style': 'formal'}", "matched_leaves": 3.0, "total_leaves": 3.0}, "evaluationTime": 0.0002827644348144531}, "isLineResult": false}], "agentExecutionOutput": null}, {"evaluationName": "Test Casual Greeting", "evaluationRunResults": [{"evaluatorName": "JsonSimilarityEvaluator", "evaluatorId": "JsonSimilarityEvaluator", "result": {"score": 1.0, "details": {"expected": "{'message': \"Hey Bob, what's up?\", 'recipient': 'Bob', 'style': 'casual'}", "actual": "{'message': \"Hey Bob, what's up?\", 'recipient': 'Bob', 'style': 'casual'}", "matched_leaves": 3.0, "total_leaves": 3.0}, "evaluationTime": 5.459785461425781e-05}, "isLineResult": false}], "agentExecutionOutput": null}, {"evaluationName": "Test Enthusiastic Greeting", "evaluationRunResults": [{"evaluatorName": "JsonSimilarityEvaluator", "evaluatorId": "JsonSimilarityEvaluator", "result": {"score": 1.0, "details": {"expected": "{'message': 'Wow! Hi Charlie!! So great to see you!!!', 'recipient': 'Charlie', 'style': 'enthusiastic'}", "actual": "{'message': 'Wow! Hi Charlie!! So great to see you!!!', 'recipient': 'Charlie', 'style': 'enthusiastic'}", "matched_leaves": 3.0, "total_leaves": 3.0}, "evaluationTime": 0.00021982192993164062}, "isLineResult": false}], "agentExecutionOutput": null}]}

‎packages/uipath/samples/greeter/output.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"evaluationSetName": "Greeter Simulated Input Evaluation Set", "evaluationSetResults": [{"evaluationName": "Test Formal Greeting with Simulated Input", "evaluationRunResults": [{"evaluatorName": "JsonSimilarityEvaluator", "evaluatorId": "JsonSimilarityEvaluator", "result": {"score": 1.0, "details": {"expected": "{'message': 'Good day, Alice. It is a pleasure to meet you.', 'recipient': 'Alice', 'style': 'formal'}", "actual": "{'message': 'Good day, Alice. It is a pleasure to meet you.', 'recipient': 'Alice', 'style': 'formal'}", "matched_leaves": 3.0, "total_leaves": 3.0}, "evaluationTime": 0.0007398128509521484}, "isLineResult": false}], "agentExecutionOutput": null}, {"evaluationName": "Test Casual Greeting with Simulated Input", "evaluationRunResults": [{"evaluatorName": "JsonSimilarityEvaluator", "evaluatorId": "JsonSimilarityEvaluator", "result": {"score": 1.0, "details": {"expected": "{'message': \"Hey Bob, what's up?\", 'recipient': 'Bob', 'style': 'casual'}", "actual": "{'message': \"Hey Bob, what's up?\", 'recipient': 'Bob', 'style': 'casual'}", "matched_leaves": 3.0, "total_leaves": 3.0}, "evaluationTime": 0.0002269744873046875}, "isLineResult": false}], "agentExecutionOutput": null}, {"evaluationName": "Test Enthusiastic Greeting with Simulated Input", "evaluationRunResults": [{"evaluatorName": "JsonSimilarityEvaluator", "evaluatorId": "JsonSimilarityEvaluator", "result": {"score": 1.0, "details": {"expected": "{'message': 'Wow! Hi Charlie!! So great to see you!!!', 'recipient': 'Charlie', 'style': 'enthusiastic'}", "actual": "{'message': 'Wow! Hi Charlie!! So great to see you!!!', 'recipient': 'Charlie', 'style': 'enthusiastic'}", "matched_leaves": 3.0, "total_leaves": 3.0}, "evaluationTime": 0.0009617805480957031}, "isLineResult": false}], "agentExecutionOutput": null}]}

‎packages/uipath/samples/greeter/pyproject.toml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[project]
2+
name = "greeter"
3+
version = "0.0.1"
4+
description = "greeter"
5+
authors = [{ name = "John Doe", email = "john.doe@myemail.com" }]
6+
dependencies = [
7+
"uipath",
8+
]
9+
requires-python = ">=3.11"
10+
11+
[dependency-groups]
12+
dev = [
13+
"uipath-dev",
14+
]

0 commit comments

Comments
 (0)