Add shutdown signal and full example using axum

Author: davidsteiner

.gitignore

@@ -1,3 +1,3 @@
-/target
-/Cargo.lock
+target/
+Cargo.lock
 .idea/

Cargo.toml

@@ -1,15 +1,14 @@
 [package]
 name = "ddtrace"
-version = "0.0.2"
+version = "0.0.3"
 authors = ["David Steiner <david_j_steiner@yahoo.co.nz", "Fergus Strangways-Dixon <fergusdixon101@gmail.com>"]
 edition = "2021"
 license = "MIT"
 description = "utilities for integrating Datadog with tracing"
 readme = "README.md"
 
 [features]
-axum = ["dep:axum", "dep:axum-tracing-opentelemetry"]
-reqwest = ["dep:reqwest"]
+axum = ["dep:axum", "dep:tokio", "dep:axum-tracing-opentelemetry"]
 
 [dependencies]
 axum = { version = "^0.6.10", optional = true }
@@ -18,9 +17,9 @@ chrono = "^0.4.24"
 opentelemetry = { version = "^0.18.0", features = ["rt-tokio"] }
 opentelemetry-datadog = "^0.6.0"
 opentelemetry-otlp = { version = "^0.11.0" }
-reqwest = { version = "^0.11.16", optional = true }
 serde = { version = "^1.0.156", features = ["derive"] }
 serde_json = "^1.0.95"
+tokio = { version = "^1.26.0", features = ["signal"], optional = true }
 tracing = "^0.1.37"
 tracing-opentelemetry = "^0.18.0"
 tracing-serde = "^0.1.3"

README.md

@@ -1,7 +1,4 @@
-This repository is home to a Rust crate `ddtrace` with various Datadog
-utilities for tracing and logging in Rust.
-
-# Background
+Datadog tracing and log correlation for Rust services.
 
 Datadog has official support for Python, which includes various SDKs and
 other utilities (such as the Python `ddtrace` library)
@@ -12,32 +9,126 @@ They don't have similar support for Rust. However, they do support the
 This crate contains the necessary glue to bridge the gap between OpenTelemetry
 and Datadog.
 
-# Further Information and Rationale
-## Tracing
+# Features
+
+`ddtrace` has the following features:
+1. tracing: utilities for building an OpenTelemetry tracer/layer that sends traces to the Datadog agent
+2. log correlation: a log formatter that converts the trace ID and span ID to the Datadog native format and injects them into the `dd.trace_id` and `dd.span_id` fields
+   (more information: https://docs.datadoghq.com/tracing/other_telemetry/connect_logs_and_traces/opentelemetry/)
+3. propagation: a utility function to set the Datadog propagator as the global propagator
+4. axum (enabled via the `axum` feature): re-exposing the functionality of [axum-tracing-opentelemetry](https://github.com/davidB/axum-tracing-opentelemetry)
+
+# A Complete Example
+
+The following is an example for using `ddtrace` with the `axum` feature enabled
+to set up an `axum` service with traces and logs sent to Datadog.
+
+```rust
+use std::net::SocketAddr;
+use std::time::Duration;
+
+use axum::{routing::get, Router};
+use ddtrace::axum::opentelemetry_tracing_layer;
+use ddtrace::formatter::DatadogFormatter;
+use ddtrace::set_global_propagator;
+use ddtrace::tracer::{build_layer, TraceResult};
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+
+#[tokio::main]
+async fn main() -> TraceResult<()> {
+    let service_name = std::env::var("DD_SERVICE").unwrap_or("my-service".to_string());
+    let tracing_layer = build_layer(&service_name)?;
+    tracing_subscriber::registry()
+        .with(tracing_subscriber::EnvFilter::new(
+            std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()),
+        ))
+        .with(
+            tracing_subscriber::fmt::layer()
+                .json()
+                .event_format(DatadogFormatter),
+        )
+        .with(tracing_layer)
+        .init();
+    set_global_propagator();
+
+    let app = Router::new()
+        .route("/", get(root))
+        .layer(opentelemetry_tracing_layer())
+        .route("/health", get(health));
+
+    let addr = SocketAddr::from(([0, 0, 0, 0], 3025));
+    tracing::info!("listening on {}", addr);
+    axum::Server::bind(&addr)
+        .serve(app.into_make_service())
+        .with_graceful_shutdown(ddtrace::axum::shutdown_signal())
+        .await
+        .unwrap();
+
+    Ok(())
+}
+
+async fn root() -> &'static str {
+    do_something().await;
+    "Hello, World!"
+}
+
+#[tracing::instrument]
+async fn do_something() {
+    tokio::time::sleep(Duration::from_millis(120)).await;
+    do_something_else().await;
+    tracing::info!("in the middle of doing something");
+    tokio::time::sleep(Duration::from_millis(10)).await;
+    do_something_else().await;
+    tokio::time::sleep(Duration::from_millis(20)).await;
+}
+
+#[tracing::instrument]
+async fn do_something_else() {
+    tokio::time::sleep(Duration::from_millis(40)).await;
+}
+
+async fn health() -> &'static str {
+    "healthy"
+}
+```
+
+# Datadog Agent Setup
+
+The Datadog agent needs to be configured to receive OTel traces over gRPC.
+Please [refer to the Datadog documentation](https://docs.datadoghq.com/opentelemetry/otlp_ingest_in_the_agent/?tab=docker)
+to set up the agent.
+
+# Further Context and Rationale
+
+## Exporting Traces
 For traces, the official Datadog agent
 [can ingest OTel trace data](https://docs.datadoghq.com/opentelemetry/)
 with the correct environment variable settings. The traces can be sent 
-via either HTTP or gRPC. More information on this can be found here:
-https://docs.datadoghq.com/opentelemetry/otlp_ingest_in_the_agent/?tab=docker
+via either HTTP or gRPC. More information on this can be found
+[here](https://docs.datadoghq.com/opentelemetry/otlp_ingest_in_the_agent/?tab=docker).
 
 OpenTelemetry has an official Rust crate with extensions for major 
 formats/providers. This includes a Datadog exporter. We have found
 this exporter to be less reliable than the standard OTel exporter
 sending data to the OTel endpoint of the Datadog agent, though.
+This crate builds on the OTel exporter.
 
-This library provides utilities to set up the Rust `tracing` crate
-for sending data to the agent in the correct way.
+## Propagation
 
-## Logging
-Datadog can ingest OpenTelemetry logs with two caveats - 
-it expects the `dd.trace_id` and `dd.span_id` attributes
-to be set, and it expects a slightly different format for
-trace ID.
+Two commonly used propagation standards are `B3` (OpenZipkin's propagation style)
+and Jaeger. OpenTelemetry [supports both](https://opentelemetry.io/docs/reference/specification/context/api-propagators/#propagators-distribution).
 
-This crate contains a JSON formatter layer that also correctly
-transform the trace ID to the Datadog native format.
+Most Datadog SDK's support both `B3` and the Datadog native propagation style.
+For example, the Python `ddtrace` library supports `B3` but it
+[needs to be explicitly enabled](https://ddtrace.readthedocs.io/en/stable/configuration.html#DD_TRACE_PROPAGATION_STYLE).
 
-## Propagation
+For ease of integration with services written in other languages that use the official Datadog SDK,
+we opted for sticking with Datadog-style propagation over `B3`. This is set via the
+`set_global_propagator` function.
+
+
+# Reqwest Propagation
 The Python library takes care of propagation of the trace context automatically.
 Unfortunately, we need to do this manually in Rust.
 
@@ -70,15 +161,3 @@ fn get_http_client() -> ClientWithMiddleware {
         .build()
 }
 ```
-
-
-## Axum Support
-The trace context propagated from other services needs to be extracted and injected
-into the propagator. For `axum`, our choice of HTTP API framework, a third-party crate
-exists that supports this: https://github.com/davidB/axum-tracing-opentelemetry.
-
-We re-expose this library for convenience.
-
-# Full Example
-
-TODO

examples/axum/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "axum-full-example"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+axum = "^0.6.12"
+ddtrace = { path = "../..", features = ["axum"] }
+tokio = { version = "^1.26.0", features = ["macros", "rt-multi-thread"] }
+tracing = "^0.1.37"
+tracing-subscriber = { version = "^0.3.16", features = ["env-filter", "json"] }

examples/axum/src/main.rs

@@ -0,0 +1,67 @@
+use std::net::SocketAddr;
+use std::time::Duration;
+
+use axum::{routing::get, Router};
+use ddtrace::axum::opentelemetry_tracing_layer;
+use ddtrace::formatter::DatadogFormatter;
+use ddtrace::set_global_propagator;
+use ddtrace::tracer::{build_layer, TraceResult};
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+
+#[tokio::main]
+async fn main() -> TraceResult<()> {
+    let service_name = std::env::var("DD_SERVICE").unwrap_or("my-service".to_string());
+    let tracing_layer = build_layer(&service_name)?;
+    tracing_subscriber::registry()
+        .with(tracing_subscriber::EnvFilter::new(
+            std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()),
+        ))
+        .with(
+            tracing_subscriber::fmt::layer()
+                .json()
+                .event_format(DatadogFormatter),
+        )
+        .with(tracing_layer)
+        .init();
+    set_global_propagator();
+
+    let app = Router::new()
+        .route("/", get(root))
+        .layer(opentelemetry_tracing_layer())
+        .route("/health", get(health));
+
+    let addr = SocketAddr::from(([0, 0, 0, 0], 3025));
+    tracing::info!("listening on {}", addr);
+    axum::Server::bind(&addr)
+        .serve(app.into_make_service())
+        .with_graceful_shutdown(ddtrace::axum::shutdown_signal())
+        .await
+        .unwrap();
+
+    Ok(())
+}
+
+async fn root() -> &'static str {
+    do_something().await;
+    "Hello, World!"
+}
+
+#[tracing::instrument]
+async fn do_something() {
+    tokio::time::sleep(Duration::from_millis(120)).await;
+    do_something_else().await;
+    tracing::info!("in the middle of doing something");
+    tokio::time::sleep(Duration::from_millis(10)).await;
+    do_something_else().await;
+    tokio::time::sleep(Duration::from_millis(20)).await;
+}
+
+#[tracing::instrument]
+async fn do_something_else() {
+    tokio::time::sleep(Duration::from_millis(40)).await;
+}
+
+async fn health() -> &'static str {
+    "healthy"
+}

src/axum.rs

@@ -7,3 +7,12 @@
 
 pub use axum_tracing_opentelemetry::opentelemetry_tracing_layer;
 pub use axum_tracing_opentelemetry::opentelemetry_tracing_layer_grpc;
+
+pub async fn shutdown_signal() {
+    tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
+        .expect("failed to install signal handler")
+        .recv()
+        .await;
+
+    opentelemetry::global::shutdown_tracer_provider();
+}