Skip to content

Commit 2f7838e

Browse files
alexander-alderman-webbalexander-alderman-webb
authored
feat(anthropic): Set system instruction attribute (#5353)
Set the system instruction attribute on `ai_chat` spans in the `AnthropicIntegration`. Extract the instruction text when the `system` parameter is a string or an iterable of dictionaries with text fields. If `system` is another type, leave the attribute unset.
1 parent 33ba1d2 commit 2f7838e

4 files changed

Lines changed: 106 additions & 64 deletions

File tree

‎sentry_sdk/_types.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,3 +359,7 @@ class SDKInfo(TypedDict):
359359
)
360360

361361
HttpStatusCodeRange = Union[int, Container[int]]
362+
363+
class TextPart(TypedDict):
364+
type: Literal["text"]
365+
content: str

‎sentry_sdk/consts.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,12 @@ class SPANDATA:
542542
Example: 2048
543543
"""
544544

545+
GEN_AI_SYSTEM_INSTRUCTIONS = "gen_ai.system_instructions"
546+
"""
547+
The system instructions passed to the model.
548+
Example: [{"type": "text", "text": "You are a helpful assistant."},{"type": "text", "text": "Be concise and clear."}]
549+
"""
550+
545551
GEN_AI_REQUEST_MESSAGES = "gen_ai.request.messages"
546552
"""
547553
The messages passed to the model. The "content" can be a string or an array of objects.

‎sentry_sdk/integrations/anthropic.py‎

Lines changed: 34 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,14 @@
3939
from anthropic.resources import AsyncMessages, Messages
4040

4141
if TYPE_CHECKING:
42-
from anthropic.types import MessageStreamEvent
42+
from anthropic.types import MessageStreamEvent, TextBlockParam
4343
except ImportError:
4444
raise DidNotEnable("Anthropic not installed")
4545

4646
if TYPE_CHECKING:
4747
from typing import Any, AsyncIterator, Iterator, List, Optional, Union
4848
from sentry_sdk.tracing import Span
49+
from sentry_sdk._types import TextPart
4950

5051

5152
class AnthropicIntegration(Integration):
@@ -177,44 +178,53 @@ def _transform_anthropic_content_block(
177178
return result if result is not None else content_block
178179

179180

181+
def _transform_system_instructions(
182+
system_instructions: "Union[str, Iterable[TextBlockParam]]",
183+
) -> "list[TextPart]":
184+
if isinstance(system_instructions, str):
185+
return [
186+
{
187+
"type": "text",
188+
"content": system_instructions,
189+
}
190+
]
191+
192+
return [
193+
{
194+
"type": "text",
195+
"content": instruction["text"],
196+
}
197+
for instruction in system_instructions
198+
if isinstance(instruction, dict) and "text" in instruction
199+
]
200+
201+
180202
def _set_input_data(
181203
span: "Span", kwargs: "dict[str, Any]", integration: "AnthropicIntegration"
182204
) -> None:
183205
"""
184206
Set input data for the span based on the provided keyword arguments for the anthropic message creation.
185207
"""
186208
set_data_normalized(span, SPANDATA.GEN_AI_OPERATION_NAME, "chat")
187-
system_prompt = kwargs.get("system")
209+
system_instructions: "Union[str, Iterable[TextBlockParam]]" = kwargs.get("system") # type: ignore
188210
messages = kwargs.get("messages")
189211
if (
190212
messages is not None
191213
and len(messages) > 0
192214
and should_send_default_pii()
193215
and integration.include_prompts
194216
):
195-
normalized_messages = []
196-
if system_prompt:
197-
system_prompt_content: "Optional[Union[str, List[dict[str, Any]]]]" = None
198-
if isinstance(system_prompt, str):
199-
system_prompt_content = system_prompt
200-
elif isinstance(system_prompt, Iterable):
201-
system_prompt_content = []
202-
for item in system_prompt:
203-
if (
204-
isinstance(item, dict)
205-
and item.get("type") == "text"
206-
and item.get("text")
207-
):
208-
system_prompt_content.append(item.copy())
209-
210-
if system_prompt_content:
211-
normalized_messages.append(
212-
{
213-
"role": GEN_AI_ALLOWED_MESSAGE_ROLES.SYSTEM,
214-
"content": system_prompt_content,
215-
}
216-
)
217+
if isinstance(system_instructions, str) or isinstance(
218+
system_instructions, Iterable
219+
):
220+
set_data_normalized(
221+
span,
222+
SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS,
223+
_transform_system_instructions(system_instructions),
224+
unpack=False,
225+
)
217226

227+
normalized_messages = []
218228
for message in messages:
219229
if (
220230
message.get("role") == GEN_AI_ALLOWED_MESSAGE_ROLES.USER

‎tests/integrations/anthropic/test_anthropic.py‎

Lines changed: 62 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1074,17 +1074,22 @@ def test_nonstreaming_create_message_with_system_prompt(
10741074
assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model"
10751075

10761076
if send_default_pii and include_prompts:
1077+
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"]
1078+
system_instructions = json.loads(
1079+
span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]
1080+
)
1081+
assert system_instructions == [
1082+
{"type": "text", "content": "You are a helpful assistant."}
1083+
]
1084+
10771085
assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"]
10781086
stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES])
1079-
assert len(stored_messages) == 2
1080-
# System message should be first
1081-
assert stored_messages[0]["role"] == "system"
1082-
assert stored_messages[0]["content"] == "You are a helpful assistant."
1083-
# User message should be second
1084-
assert stored_messages[1]["role"] == "user"
1085-
assert stored_messages[1]["content"] == "Hello, Claude"
1087+
assert len(stored_messages) == 1
1088+
assert stored_messages[0]["role"] == "user"
1089+
assert stored_messages[0]["content"] == "Hello, Claude"
10861090
assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude."
10871091
else:
1092+
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"]
10881093
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"]
10891094
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"]
10901095

@@ -1153,17 +1158,22 @@ async def test_nonstreaming_create_message_with_system_prompt_async(
11531158
assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model"
11541159

11551160
if send_default_pii and include_prompts:
1161+
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"]
1162+
system_instructions = json.loads(
1163+
span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]
1164+
)
1165+
assert system_instructions == [
1166+
{"type": "text", "content": "You are a helpful assistant."}
1167+
]
1168+
11561169
assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"]
11571170
stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES])
1158-
assert len(stored_messages) == 2
1159-
# System message should be first
1160-
assert stored_messages[0]["role"] == "system"
1161-
assert stored_messages[0]["content"] == "You are a helpful assistant."
1162-
# User message should be second
1163-
assert stored_messages[1]["role"] == "user"
1164-
assert stored_messages[1]["content"] == "Hello, Claude"
1171+
assert len(stored_messages) == 1
1172+
assert stored_messages[0]["role"] == "user"
1173+
assert stored_messages[0]["content"] == "Hello, Claude"
11651174
assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi, I'm Claude."
11661175
else:
1176+
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"]
11671177
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"]
11681178
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"]
11691179

@@ -1264,18 +1274,23 @@ def test_streaming_create_message_with_system_prompt(
12641274
assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model"
12651275

12661276
if send_default_pii and include_prompts:
1277+
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"]
1278+
system_instructions = json.loads(
1279+
span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]
1280+
)
1281+
assert system_instructions == [
1282+
{"type": "text", "content": "You are a helpful assistant."}
1283+
]
1284+
12671285
assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"]
12681286
stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES])
1269-
assert len(stored_messages) == 2
1270-
# System message should be first
1271-
assert stored_messages[0]["role"] == "system"
1272-
assert stored_messages[0]["content"] == "You are a helpful assistant."
1273-
# User message should be second
1274-
assert stored_messages[1]["role"] == "user"
1275-
assert stored_messages[1]["content"] == "Hello, Claude"
1287+
assert len(stored_messages) == 1
1288+
assert stored_messages[0]["role"] == "user"
1289+
assert stored_messages[0]["content"] == "Hello, Claude"
12761290
assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!"
12771291

12781292
else:
1293+
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"]
12791294
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"]
12801295
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"]
12811296

@@ -1379,18 +1394,23 @@ async def test_streaming_create_message_with_system_prompt_async(
13791394
assert span["data"][SPANDATA.GEN_AI_REQUEST_MODEL] == "model"
13801395

13811396
if send_default_pii and include_prompts:
1397+
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"]
1398+
system_instructions = json.loads(
1399+
span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS]
1400+
)
1401+
assert system_instructions == [
1402+
{"type": "text", "content": "You are a helpful assistant."}
1403+
]
1404+
13821405
assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"]
13831406
stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES])
1384-
assert len(stored_messages) == 2
1385-
# System message should be first
1386-
assert stored_messages[0]["role"] == "system"
1387-
assert stored_messages[0]["content"] == "You are a helpful assistant."
1388-
# User message should be second
1389-
assert stored_messages[1]["role"] == "user"
1390-
assert stored_messages[1]["content"] == "Hello, Claude"
1407+
assert len(stored_messages) == 1
1408+
assert stored_messages[0]["role"] == "user"
1409+
assert stored_messages[0]["content"] == "Hello, Claude"
13911410
assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hi! I'm Claude!"
13921411

13931412
else:
1413+
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"]
13941414
assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"]
13951415
assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"]
13961416

@@ -1437,21 +1457,23 @@ def test_system_prompt_with_complex_structure(sentry_init, capture_events):
14371457
(span,) = event["spans"]
14381458

14391459
assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat"
1460+
1461+
assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS in span["data"]
1462+
system_instructions = json.loads(span["data"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS])
1463+
1464+
# System content should be a list of text blocks
1465+
assert isinstance(system_instructions, list)
1466+
assert system_instructions == [
1467+
{"type": "text", "content": "You are a helpful assistant."},
1468+
{"type": "text", "content": "Be concise and clear."},
1469+
]
1470+
14401471
assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"]
14411472
stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES])
14421473

1443-
# Should have system message first, then user message
1444-
assert len(stored_messages) == 2
1445-
assert stored_messages[0]["role"] == "system"
1446-
# System content should be a list of text blocks
1447-
assert isinstance(stored_messages[0]["content"], list)
1448-
assert len(stored_messages[0]["content"]) == 2
1449-
assert stored_messages[0]["content"][0]["type"] == "text"
1450-
assert stored_messages[0]["content"][0]["text"] == "You are a helpful assistant."
1451-
assert stored_messages[0]["content"][1]["type"] == "text"
1452-
assert stored_messages[0]["content"][1]["text"] == "Be concise and clear."
1453-
assert stored_messages[1]["role"] == "user"
1454-
assert stored_messages[1]["content"] == "Hello"
1474+
assert len(stored_messages) == 1
1475+
assert stored_messages[0]["role"] == "user"
1476+
assert stored_messages[0]["content"] == "Hello"
14551477

14561478

14571479
# Tests for transform_content_part (shared) and _transform_anthropic_content_block helper functions

0 commit comments

Comments
 (0)