Skip to content

Commit 3f1fe16

Browse files
izadoesdevizadoesdev
committed
fix(observability): normalize evlog wide events for Axiom error fields
Drain: move string error to error_message, downgrade 4xx to warn. captureError: log client EvlogErrors as warn. Process logs use error_message to avoid clobbering structured error.message. Document in databuddy skill.
1 parent 201075e commit 3f1fe16

7 files changed

Lines changed: 100 additions & 19 deletions

File tree

‎.agents/skills/databuddy/SKILL.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ Read [codebase-map.md](./references/codebase-map.md) when you need deeper routin
7878
- Start in `apps/basket/src`
7979
- Request validation, billing checks, geo/IP parsing, producer logic, and structured errors are important here
8080
- Storage and schema concerns usually continue into `packages/db`
81+
- **evlog → Axiom:** never use top-level `error` as a **string** on `log.error({ ... })` (e.g. process handlers); it overwrites structured `error.message` on the wide event. Use `error_message` instead. Basket/API drains run `normalizeWideEventForAxiom` before ingest; 4xx `EvlogError` rows are emitted as `level: "warn"` with `client_http_error: true` so Axiom “errors” are not inflated by expected client failures.
8182

8283
### Database work
8384

‎apps/api/src/index.ts‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,15 @@ try {
4949
service: "api",
5050
component: "tcc_otel",
5151
message: "TCC tracing disabled (init failed)",
52-
error: error instanceof Error ? error.message : String(error),
52+
error_message: error instanceof Error ? error.message : String(error),
5353
});
5454
}
5555

5656
process.on("unhandledRejection", (reason, _promise) => {
5757
captureError(reason);
5858
log.error({
5959
process: "unhandledRejection",
60-
error: reason instanceof Error ? reason.message : String(reason),
60+
error_message: reason instanceof Error ? reason.message : String(reason),
6161
error_stack: reason instanceof Error ? reason.stack : undefined,
6262
error_source: "process",
6363
});
@@ -67,7 +67,7 @@ process.on("uncaughtException", (error) => {
6767
captureError(error);
6868
log.error({
6969
process: "uncaughtException",
70-
error: error instanceof Error ? error.message : String(error),
70+
error_message: error instanceof Error ? error.message : String(error),
7171
error_stack: error instanceof Error ? error.stack : undefined,
7272
error_source: "process",
7373
});
@@ -387,13 +387,13 @@ process.on("SIGINT", async () => {
387387
flushBatchedApiDrain().catch((error) =>
388388
log.error({
389389
lifecycle: "drainFlush",
390-
error: error instanceof Error ? error.message : String(error),
390+
error_message: error instanceof Error ? error.message : String(error),
391391
})
392392
),
393393
shutdownTccTracing().catch((error) =>
394394
log.error({
395395
lifecycle: "tccOtelShutdown",
396-
error: error instanceof Error ? error.message : String(error),
396+
error_message: error instanceof Error ? error.message : String(error),
397397
})
398398
),
399399
]);
@@ -406,13 +406,13 @@ process.on("SIGTERM", async () => {
406406
flushBatchedApiDrain().catch((error) =>
407407
log.error({
408408
lifecycle: "drainFlush",
409-
error: error instanceof Error ? error.message : String(error),
409+
error_message: error instanceof Error ? error.message : String(error),
410410
})
411411
),
412412
shutdownTccTracing().catch((error) =>
413413
log.error({
414414
lifecycle: "tccOtelShutdown",
415-
error: error instanceof Error ? error.message : String(error),
415+
error_message: error instanceof Error ? error.message : String(error),
416416
})
417417
),
418418
]);

‎apps/api/src/lib/evlog-api.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,31 @@ const devFsDrain = useLocalEvlogFiles
3636

3737
const DURATION_REGEX = /^([\d.]+)(ms|s)$/;
3838

39+
/**
40+
* Before Axiom: fix `error` string vs object collision; downgrade 4xx to warn.
41+
*/
42+
function normalizeWideEventForAxiom(event: Record<string, unknown>): void {
43+
if (typeof event.error === "string") {
44+
event.error_message = event.error;
45+
event.error = undefined;
46+
}
47+
48+
if (event.level !== "error") {
49+
return;
50+
}
51+
52+
const err = event.error;
53+
if (!err || typeof err !== "object" || Array.isArray(err)) {
54+
return;
55+
}
56+
57+
const status = (err as { status?: number }).status;
58+
if (typeof status === "number" && status >= 400 && status < 500) {
59+
event.level = "warn";
60+
event.client_http_error = true;
61+
}
62+
}
63+
3964
function parseDurationMs(duration: unknown): number | undefined {
4065
if (typeof duration !== "string") {
4166
return undefined;
@@ -54,6 +79,8 @@ function parseDurationMs(duration: unknown): number | undefined {
5479
* and still sends to Axiom via the batched pipeline. Production: Axiom only.
5580
*/
5681
export async function apiLoggerDrain(ctx: DrainContext): Promise<void> {
82+
normalizeWideEventForAxiom(ctx.event as Record<string, unknown>);
83+
5784
const durationMs = parseDurationMs(ctx.event.duration);
5885
if (durationMs !== undefined) {
5986
ctx.event.duration_ms = durationMs;

‎apps/api/src/lib/tracing.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { log } from "evlog";
1+
import { EvlogError, log } from "evlog";
22
import { useLogger as getRequestLogger } from "evlog/elysia";
33

44
/**
@@ -25,6 +25,19 @@ export function captureError(
2525
const err = error instanceof Error ? error : new Error(String(error));
2626
try {
2727
const requestLog = getRequestLogger();
28+
if (err instanceof EvlogError && err.status >= 400 && err.status < 500) {
29+
requestLog.set({
30+
client_http_error: true,
31+
http_status: err.status,
32+
error_message: err.message,
33+
});
34+
if (fields) {
35+
requestLog.warn(err.message, fields as Record<string, unknown>);
36+
} else {
37+
requestLog.warn(err.message);
38+
}
39+
return;
40+
}
2841
if (fields) {
2942
requestLog.error(err, fields as Record<string, unknown>);
3043
} else {
@@ -33,7 +46,7 @@ export function captureError(
3346
} catch {
3447
log.error({
3548
service: "api",
36-
error: err.message,
49+
error_message: err.message,
3750
...(fields ?? {}),
3851
});
3952
}

‎apps/basket/src/index.ts‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ process.on("unhandledRejection", (reason, _promise) => {
3131
captureError(reason);
3232
log.error({
3333
process: "unhandledRejection",
34-
error: reason instanceof Error ? reason.message : String(reason),
34+
error_message: reason instanceof Error ? reason.message : String(reason),
3535
error_stack: reason instanceof Error ? reason.stack : undefined,
3636
error_source: "process",
3737
});
@@ -41,7 +41,7 @@ process.on("uncaughtException", (error) => {
4141
captureError(error);
4242
log.error({
4343
process: "uncaughtException",
44-
error: error instanceof Error ? error.message : String(error),
44+
error_message: error instanceof Error ? error.message : String(error),
4545
error_stack: error instanceof Error ? error.stack : undefined,
4646
error_source: "process",
4747
});
@@ -52,19 +52,19 @@ process.on("SIGTERM", async () => {
5252
await flushBatchedAxiomDrain().catch((error) =>
5353
log.error({
5454
lifecycle: "drainFlush",
55-
error: error instanceof Error ? error.message : String(error),
55+
error_message: error instanceof Error ? error.message : String(error),
5656
})
5757
);
5858
await runPromise(disconnect).catch((error) =>
5959
log.error({
6060
lifecycle: "shutdown",
61-
error: error instanceof Error ? error.message : String(error),
61+
error_message: error instanceof Error ? error.message : String(error),
6262
})
6363
);
6464
await disposeRuntime().catch((error) =>
6565
log.error({
6666
lifecycle: "runtimeDispose",
67-
error: error instanceof Error ? error.message : String(error),
67+
error_message: error instanceof Error ? error.message : String(error),
6868
})
6969
);
7070
closeGeoIPReader();
@@ -76,19 +76,19 @@ process.on("SIGINT", async () => {
7676
await flushBatchedAxiomDrain().catch((error) =>
7777
log.error({
7878
lifecycle: "drainFlush",
79-
error: error instanceof Error ? error.message : String(error),
79+
error_message: error instanceof Error ? error.message : String(error),
8080
})
8181
);
8282
await runPromise(disconnect).catch((error) =>
8383
log.error({
8484
lifecycle: "shutdown",
85-
error: error instanceof Error ? error.message : String(error),
85+
error_message: error instanceof Error ? error.message : String(error),
8686
})
8787
);
8888
await disposeRuntime().catch((error) =>
8989
log.error({
9090
lifecycle: "runtimeDispose",
91-
error: error instanceof Error ? error.message : String(error),
91+
error_message: error instanceof Error ? error.message : String(error),
9292
})
9393
);
9494
closeGeoIPReader();

‎apps/basket/src/lib/evlog-basket.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,31 @@ const devFsDrain = useLocalEvlogFiles
3939

4040
const DURATION_MS_REGEX = /^([\d.]+)(ms|s)$/;
4141

42+
/**
43+
* Before Axiom: fix `error` string vs object collision; downgrade 4xx to warn.
44+
*/
45+
function normalizeWideEventForAxiom(event: Record<string, unknown>): void {
46+
if (typeof event.error === "string") {
47+
event.error_message = event.error;
48+
event.error = undefined;
49+
}
50+
51+
if (event.level !== "error") {
52+
return;
53+
}
54+
55+
const err = event.error;
56+
if (!err || typeof err !== "object" || Array.isArray(err)) {
57+
return;
58+
}
59+
60+
const status = (err as { status?: number }).status;
61+
if (typeof status === "number" && status >= 400 && status < 500) {
62+
event.level = "warn";
63+
event.client_http_error = true;
64+
}
65+
}
66+
4267
function parseDurationMs(duration: unknown): number | undefined {
4368
if (typeof duration !== "string") {
4469
return undefined;
@@ -61,6 +86,8 @@ export async function basketLoggerDrain(ctx: DrainContext): Promise<void> {
6186
return;
6287
}
6388

89+
normalizeWideEventForAxiom(ctx.event as Record<string, unknown>);
90+
6491
const durationMs = parseDurationMs(ctx.event.duration);
6592
if (durationMs !== undefined) {
6693
ctx.event.duration_ms = durationMs;

‎apps/basket/src/lib/tracing.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { log } from "evlog";
1+
import { EvlogError, log } from "evlog";
22
import { useLogger } from "evlog/elysia";
33

44
/**
@@ -20,6 +20,19 @@ export function captureError(
2020
const err = error instanceof Error ? error : new Error(String(error));
2121
try {
2222
const requestLog = useLogger();
23+
if (err instanceof EvlogError && err.status >= 400 && err.status < 500) {
24+
requestLog.set({
25+
client_http_error: true,
26+
http_status: err.status,
27+
error_message: err.message,
28+
});
29+
if (attributes) {
30+
requestLog.warn(err.message, attributes as Record<string, unknown>);
31+
} else {
32+
requestLog.warn(err.message);
33+
}
34+
return;
35+
}
2336
if (attributes) {
2437
requestLog.error(err, attributes as Record<string, unknown>);
2538
} else {
@@ -28,7 +41,7 @@ export function captureError(
2841
} catch {
2942
log.error({
3043
service: "basket",
31-
error: err.message,
44+
error_message: err.message,
3245
...(attributes ?? {}),
3346
});
3447
}

0 commit comments

Comments
 (0)