Skip to content

Commit c3f0ca3

Browse files
diegomarquezpdiegomarquezp
committed
test(o11y): confirm behavior of golden signals in compute
1 parent 684511a commit c3f0ca3

2 files changed

Lines changed: 386 additions & 0 deletions

File tree

‎java-compute/google-cloud-compute/pom.xml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,45 @@
9292
<artifactId>google-cloud-core</artifactId>
9393
<scope>test</scope>
9494
</dependency>
95+
<dependency>
96+
<groupId>io.opentelemetry</groupId>
97+
<artifactId>opentelemetry-sdk</artifactId>
98+
<scope>test</scope>
99+
</dependency>
100+
<dependency>
101+
<groupId>io.opentelemetry</groupId>
102+
<artifactId>opentelemetry-exporter-otlp</artifactId>
103+
<scope>test</scope>
104+
</dependency>
105+
<dependency>
106+
<groupId>io.opentelemetry</groupId>
107+
<artifactId>opentelemetry-sdk-testing</artifactId>
108+
<scope>test</scope>
109+
</dependency>
110+
<!-- Logback dependencies are used to intercept logs for validation in tests, consistent with ITActionableErrorsLogging -->
111+
<dependency>
112+
<groupId>ch.qos.logback</groupId>
113+
<artifactId>logback-classic</artifactId>
114+
<version>1.5.25</version>
115+
<scope>test</scope>
116+
</dependency>
117+
<dependency>
118+
<groupId>ch.qos.logback</groupId>
119+
<artifactId>logback-core</artifactId>
120+
<version>1.5.25</version>
121+
<scope>test</scope>
122+
</dependency>
123+
<dependency>
124+
<groupId>com.google.cloud</groupId>
125+
<artifactId>google-cloud-trace</artifactId>
126+
<version>2.89.0-SNAPSHOT</version><!-- {x-version-update:google-cloud-trace:current} -->
127+
<scope>test</scope>
128+
</dependency>
129+
<dependency>
130+
<groupId>com.google.truth</groupId>
131+
<artifactId>truth</artifactId>
132+
<scope>test</scope>
133+
</dependency>
95134

96135
<!-- Need testing utility classes for generated REST clients tests -->
97136
<dependency>

‎java-compute/google-cloud-compute/src/test/java/com/google/cloud/compute/v1/integration/ITComputeGoldenSignals.java‎

Lines changed: 347 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,347 @@
1+
package com.google.cloud.compute.v1.integration;
2+
3+
import static com.google.common.truth.Truth.assertThat;
4+
import static org.junit.Assert.fail;
5+
6+
import com.google.api.gax.tracing.CompositeTracerFactory;
7+
import com.google.api.gax.tracing.OpenTelemetryTracingFactory;
8+
import com.google.api.gax.tracing.ObservabilityAttributes;
9+
import com.google.api.gax.tracing.OpenTelemetryMetricsFactory;
10+
import com.google.api.gax.tracing.LoggingTracerFactory;
11+
import com.google.api.gax.tracing.ApiTracerFactory;
12+
import com.google.cloud.compute.v1.InstancesClient;
13+
import com.google.cloud.compute.v1.InstancesSettings;
14+
import com.google.cloud.trace.v1.TraceServiceClient;
15+
import com.google.cloud.trace.v1.TraceServiceSettings;
16+
import com.google.devtools.cloudtrace.v1.GetTraceRequest;
17+
import com.google.api.gax.retrying.RetrySettings;
18+
import com.google.api.gax.rpc.StatusCode;
19+
import java.time.Duration;
20+
import com.google.devtools.cloudtrace.v1.Trace;
21+
import com.google.devtools.cloudtrace.v1.TraceSpan;
22+
import io.opentelemetry.api.OpenTelemetry;
23+
import io.opentelemetry.api.trace.Span;
24+
import io.opentelemetry.api.trace.SpanContext;
25+
import io.opentelemetry.api.trace.TraceFlags;
26+
import io.opentelemetry.api.trace.TraceState;
27+
import io.opentelemetry.api.trace.Tracer;
28+
import io.opentelemetry.context.Context;
29+
import io.opentelemetry.context.Scope;
30+
import io.opentelemetry.sdk.OpenTelemetrySdk;
31+
import io.opentelemetry.sdk.trace.SdkTracerProvider;
32+
import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
33+
import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
34+
import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader;
35+
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
36+
import io.opentelemetry.sdk.metrics.data.MetricData;
37+
import java.util.Collection;
38+
import com.google.auth.oauth2.GoogleCredentials;
39+
import org.slf4j.LoggerFactory;
40+
import ch.qos.logback.classic.Logger;
41+
import ch.qos.logback.classic.spi.ILoggingEvent;
42+
import ch.qos.logback.core.AppenderBase;
43+
import java.util.ArrayList;
44+
import java.util.HashMap;
45+
import org.slf4j.event.KeyValuePair;
46+
import io.opentelemetry.api.common.Attributes;
47+
import io.opentelemetry.api.common.AttributeKey;
48+
import io.opentelemetry.sdk.resources.Resource;
49+
import java.util.Arrays;
50+
import java.util.List;
51+
import java.util.UUID;
52+
import java.util.Map;
53+
import org.junit.After;
54+
import org.junit.Before;
55+
import org.junit.Test;
56+
57+
/**
58+
* Integration tests for Compute observability "golden signals".
59+
* Validates that traces, metrics, and actionable error logs are correctly recorded and exported.
60+
*/
61+
public class ITComputeGoldenSignals extends BaseTest {
62+
private static final Logger logger = (Logger) LoggerFactory.getLogger(ITComputeGoldenSignals.class);
63+
private static final String TELEMETRY_ENDPOINT = "https://telemetry.googleapis.com";
64+
65+
private OpenTelemetrySdk openTelemetrySdk;
66+
private TraceServiceClient traceClient;
67+
private String traceId;
68+
private String rootSpanName;
69+
private Tracer tracer;
70+
private CompositeTracerFactory compositeFactory;
71+
private InMemoryMetricReader metricReader;
72+
private TestAppender testAppender;
73+
74+
@Before
75+
public void setUp() throws Exception {
76+
traceId = generateRandomHexString(32);
77+
rootSpanName = "ComputeRootSpan-" + generateRandomHexString(8);
78+
79+
GoogleCredentials credentials = GoogleCredentials.getApplicationDefault();
80+
credentials.refreshIfExpired();
81+
String token = credentials.getAccessToken().getTokenValue();
82+
83+
OtlpGrpcSpanExporter spanExporter = OtlpGrpcSpanExporter.builder()
84+
.setEndpoint(TELEMETRY_ENDPOINT)
85+
.addHeader("Authorization", "Bearer " + token)
86+
.addHeader("x-goog-user-project", DEFAULT_PROJECT)
87+
.build();
88+
89+
BatchSpanProcessor spanProcessor = BatchSpanProcessor.builder(spanExporter).build();
90+
91+
Resource resource = Resource.getDefault()
92+
.merge(Resource.create(Attributes.of(AttributeKey.stringKey("gcp.project_id"), DEFAULT_PROJECT)));
93+
94+
metricReader = InMemoryMetricReader.create();
95+
openTelemetrySdk = OpenTelemetrySdk.builder()
96+
.setTracerProvider(
97+
SdkTracerProvider.builder()
98+
.addSpanProcessor(spanProcessor)
99+
.setResource(resource)
100+
.build())
101+
.setMeterProvider(
102+
SdkMeterProvider.builder()
103+
.registerMetricReader(metricReader)
104+
.setResource(resource)
105+
.build())
106+
.build();
107+
108+
tracer = openTelemetrySdk.getTracer("testing-compute");
109+
110+
// Configure TraceServiceClient with retry settings
111+
TraceServiceSettings.Builder settingsBuilder = TraceServiceSettings.newBuilder();
112+
settingsBuilder.getTraceSettings()
113+
.setRetrySettings(
114+
RetrySettings.newBuilder()
115+
.setTotalTimeoutDuration(Duration.ofSeconds(60))
116+
.setInitialRpcTimeoutDuration(Duration.ofSeconds(5))
117+
.setMaxRpcTimeoutDuration(Duration.ofSeconds(10))
118+
.build())
119+
.setRetryableCodes(StatusCode.Code.NOT_FOUND);
120+
121+
settingsBuilder.getStubSettingsBuilder().setTracerFactory(com.google.api.gax.tracing.BaseApiTracerFactory.getInstance());
122+
123+
traceClient = TraceServiceClient.create(settingsBuilder.build());
124+
125+
// Combine tracers using CompositeTracerFactory
126+
List<ApiTracerFactory> factories = Arrays.asList(
127+
new OpenTelemetryTracingFactory(openTelemetrySdk),
128+
new OpenTelemetryMetricsFactory(openTelemetrySdk),
129+
new LoggingTracerFactory()
130+
);
131+
compositeFactory = new CompositeTracerFactory(factories);
132+
133+
// Initialize and attach TestAppender
134+
testAppender = new TestAppender();
135+
testAppender.start();
136+
Logger loggingTracerLogger =
137+
(Logger) LoggerFactory.getLogger("com.google.api.gax.tracing.LoggingTracer");
138+
loggingTracerLogger.addAppender(testAppender);
139+
loggingTracerLogger.setLevel(ch.qos.logback.classic.Level.DEBUG);
140+
}
141+
142+
@After
143+
public void tearDown() throws Exception {
144+
if (traceClient != null) {
145+
traceClient.close();
146+
}
147+
if (openTelemetrySdk != null) {
148+
openTelemetrySdk.close();
149+
}
150+
if (testAppender != null) {
151+
((Logger) LoggerFactory.getLogger("ROOT"))
152+
.detachAppender(testAppender);
153+
}
154+
}
155+
156+
/**
157+
* Creates a root span with a specific trace ID to simulate an external parent context.
158+
* This helps verify that the library correctly creates child spans that inherit the parent's trace ID.
159+
*
160+
* @param traceId The trace ID to use for the root span.
161+
* @return The created root span.
162+
*/
163+
private Span createRootSpan(String traceId) {
164+
SpanContext customSpanContext = SpanContext.create(traceId, generateRandomHexString(16), TraceFlags.getSampled(), TraceState.getDefault());
165+
return tracer.spanBuilder(rootSpanName)
166+
.setParent(Context.root().with(Span.wrap(customSpanContext)))
167+
.startSpan();
168+
}
169+
170+
/**
171+
* Tests that a successful compute operation generates traces that are correctly exported to Cloud Trace.
172+
*/
173+
@Test
174+
public void testComputeOperationTracing() throws Exception {
175+
String localTraceId = generateRandomHexString(32);
176+
Span rootSpan = createRootSpan(localTraceId);
177+
178+
try (Scope scope = rootSpan.makeCurrent()) {
179+
InstancesSettings.Builder settingsBuilder = InstancesSettings.newBuilder();
180+
settingsBuilder.getStubSettingsBuilder().setTracerFactory(compositeFactory);
181+
182+
try (InstancesClient client = InstancesClient.create(settingsBuilder.build())) {
183+
logger.info("Listing instances in project: " + DEFAULT_PROJECT + " zone: " + DEFAULT_ZONE);
184+
client.list(DEFAULT_PROJECT, DEFAULT_ZONE);
185+
}
186+
} finally {
187+
rootSpan.end();
188+
}
189+
190+
openTelemetrySdk.getSdkTracerProvider().forceFlush();
191+
fetchAndValidateTrace(localTraceId, false);
192+
validateMetrics();
193+
validateLogging(false);
194+
}
195+
196+
/**
197+
* Tests that a failed compute operation generates traces with error attributes.
198+
*/
199+
@Test
200+
public void testComputeOperationTracing_Error() throws Exception {
201+
String localTraceId = generateRandomHexString(32);
202+
Span rootSpan = createRootSpan(localTraceId);
203+
204+
try (Scope scope = rootSpan.makeCurrent()) {
205+
InstancesSettings.Builder settingsBuilder = InstancesSettings.newBuilder();
206+
settingsBuilder.getStubSettingsBuilder().setTracerFactory(compositeFactory);
207+
208+
try (InstancesClient client = InstancesClient.create(settingsBuilder.build())) {
209+
logger.info("Triggering error by listing instances in invalid project...");
210+
client.list("invalid-project-" + UUID.randomUUID().toString(), DEFAULT_ZONE);
211+
fail("Expected exception not thrown");
212+
} catch (Exception e) {
213+
logger.info("Caught expected exception: " + e.getMessage());
214+
}
215+
} finally {
216+
rootSpan.end();
217+
}
218+
219+
openTelemetrySdk.getSdkTracerProvider().forceFlush();
220+
fetchAndValidateTrace(localTraceId, true);
221+
validateMetrics();
222+
validateLogging(true);
223+
}
224+
225+
private void fetchAndValidateTrace(String traceId, boolean expectError) throws Exception {
226+
Trace trace = traceClient.getTrace(DEFAULT_PROJECT, traceId);
227+
assertThat(trace).isNotNull();
228+
229+
for (TraceSpan span : trace.getSpansList()) {
230+
logger.info("Verifying attributes for span: " + span.getName());
231+
232+
// Skip root span as it's manually created and doesn't have RPC attributes.
233+
if (span.getName().contains("ComputeRootSpan")) {
234+
continue;
235+
}
236+
237+
// Assert RPC span name pattern {method} {url template}
238+
assertThat(span.getName()).isEqualTo("GET compute/v1/projects/{project=*}/zones/{zone=*}/instances");
239+
240+
// Compute uses HTTP/REST, so we check for rpc.system.name and other HTTP attributes
241+
assertThat(span.getLabelsMap().get(ObservabilityAttributes.RPC_SYSTEM_NAME_ATTRIBUTE)).isEqualTo("http");
242+
assertThat(span.getLabelsMap().get(ObservabilityAttributes.URL_DOMAIN_ATTRIBUTE)).isEqualTo("compute.googleapis.com");
243+
assertThat(span.getLabelsMap().get(ObservabilityAttributes.HTTP_METHOD_ATTRIBUTE)).isEqualTo("GET");
244+
assertThat(span.getLabelsMap().get(ObservabilityAttributes.GCP_CLIENT_SERVICE_ATTRIBUTE)).isEqualTo("compute");
245+
assertThat(span.getLabelsMap().get(ObservabilityAttributes.REPO_ATTRIBUTE)).isEqualTo("googleapis/google-cloud-java");
246+
247+
if (expectError) {
248+
assertThat(span.getLabelsMap().get(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE)).isEqualTo("404");
249+
} else {
250+
assertThat(span.getLabelsMap().get(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE)).isEqualTo("200");
251+
}
252+
253+
if (expectError) {
254+
// Verify error attributes
255+
assertThat(span.getLabelsMap()).containsKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE);
256+
assertThat(span.getLabelsMap()).containsKey(ObservabilityAttributes.EXCEPTION_TYPE_ATTRIBUTE);
257+
assertThat(span.getLabelsMap()).containsKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE);
258+
}
259+
}
260+
}
261+
262+
private void validateMetrics() {
263+
Collection<MetricData> metrics = metricReader.collectAllMetrics();
264+
logger.info("Collected " + metrics.size() + " metrics");
265+
266+
// GoldenSignalsMetricsRecorder.CLIENT_REQUEST_DURATION_METRIC_NAME is package-private
267+
String expectedMetricName = "gcp.client.request.duration";
268+
269+
MetricData durationMetric =
270+
metrics.stream()
271+
.filter(m -> m.getName().equals(expectedMetricName))
272+
.findFirst()
273+
.orElseThrow(() -> new AssertionError("Duration metric not found: " + expectedMetricName));
274+
275+
logger.info("Found duration metric: " + durationMetric.getName());
276+
277+
// Assert that we have at least one point
278+
assertThat(durationMetric.getHistogramData().getPoints()).isNotEmpty();
279+
280+
io.opentelemetry.api.common.Attributes attributes = durationMetric.getHistogramData().getPoints().iterator().next().getAttributes();
281+
assertThat(attributes.get(AttributeKey.stringKey(ObservabilityAttributes.RPC_SYSTEM_NAME_ATTRIBUTE))).isEqualTo("http");
282+
assertThat(attributes.get(AttributeKey.stringKey(ObservabilityAttributes.GCP_CLIENT_SERVICE_ATTRIBUTE))).isEqualTo("compute");
283+
assertThat(attributes.get(AttributeKey.stringKey(ObservabilityAttributes.URL_TEMPLATE_ATTRIBUTE))).isEqualTo("compute/v1/projects/{project=*}/zones/{zone=*}/instances");
284+
}
285+
286+
private void validateLogging(boolean expectError) {
287+
List<ILoggingEvent> computeEvents = new ArrayList<>();
288+
for (ILoggingEvent event : testAppender.events) {
289+
if (event.getKeyValuePairs() == null) {
290+
continue;
291+
}
292+
Map<String, String> mdc = new HashMap<>();
293+
for (KeyValuePair kvp : event.getKeyValuePairs()) {
294+
mdc.put(kvp.key, String.valueOf(kvp.value));
295+
}
296+
if (!"compute".equals(mdc.get("gcp.client.service"))) {
297+
continue;
298+
}
299+
computeEvents.add(event);
300+
}
301+
302+
if (expectError) {
303+
assertThat(computeEvents).isNotEmpty();
304+
ILoggingEvent event = computeEvents.get(computeEvents.size() - 1);
305+
if (event.getKeyValuePairs() == null) {
306+
fail("Expected log event to have key value pairs");
307+
}
308+
Map<String, String> mdc = new HashMap<>();
309+
for (KeyValuePair kvp : event.getKeyValuePairs()) {
310+
mdc.put(kvp.key, String.valueOf(kvp.value));
311+
}
312+
313+
assertThat(mdc).containsEntry(ObservabilityAttributes.RPC_SYSTEM_NAME_ATTRIBUTE, "http");
314+
assertThat(mdc).containsEntry(ObservabilityAttributes.GCP_CLIENT_SERVICE_ATTRIBUTE, "compute");
315+
assertThat(mdc).containsEntry(ObservabilityAttributes.REPO_ATTRIBUTE, "googleapis/google-cloud-java");
316+
assertThat(mdc).containsEntry(ObservabilityAttributes.HTTP_METHOD_ATTRIBUTE, "GET");
317+
assertThat(mdc).containsKey("url.template");
318+
assertThat(mdc).containsKey(ObservabilityAttributes.EXCEPTION_MESSAGE_ATTRIBUTE);
319+
} else {
320+
if (!computeEvents.isEmpty()) {
321+
logger.info("Captured " + computeEvents.size() + " unexpected compute log events:");
322+
for (ILoggingEvent event : computeEvents) {
323+
logger.info("Event: " + event.getMessage() + ", Extracted: " + event.getKeyValuePairs());
324+
}
325+
}
326+
assertThat(computeEvents).isEmpty();
327+
}
328+
}
329+
330+
public static class TestAppender extends AppenderBase<ILoggingEvent> {
331+
public List<ILoggingEvent> events = new ArrayList<>();
332+
333+
@Override
334+
protected void append(ILoggingEvent eventObject) {
335+
eventObject.getMDCPropertyMap();
336+
events.add(eventObject);
337+
}
338+
339+
public void clearEvents() {
340+
events.clear();
341+
}
342+
}
343+
344+
private String generateRandomHexString(int length) {
345+
return UUID.randomUUID().toString().replace("-", "").substring(0, length);
346+
}
347+
}

0 commit comments

Comments
 (0)