feat(vrl): add native parse_ddtags function

Add a VRL function `parse_ddtags(value, multivalue: true)` that parses
Datadog tag strings (comma-separated key:value pairs) into objects.

In multivalue mode (default), values are arrays so duplicate keys are
preserved. In single-value mode, the first occurrence wins.

Handles standalone keys (no colon), colons embedded in values (splits on
first only), whitespace trimming, and empty segments.

Benchmarked against the equivalent pure-VRL implementation (for_each +
split + get/set/push) on a realistic 50-tag input:

  native:   10.1 µs/iter
  pure VRL: 320.8 µs/iter  (~32x slower)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: eliebleton-manomano

benches/remap.rs

@@ -12,6 +12,59 @@ use vector::{
 };
 use vrl::{event_path, prelude::*};
 
+// ~50 realistic Datadog tags with long values and duplicate keys.
+const DDTAGS_BENCH_INPUT: &str = "\
+env:production,\
+service:payment-gateway-service,\
+version:4.12.7-rc3,\
+host:ip-10-42-137-29.us-east-1.compute.internal,\
+instance-type:m5.2xlarge,\
+availability-zone:us-east-1c,\
+region:us-east-1,\
+cluster:eks-prod-main-useast1-2024,\
+namespace:payments,\
+pod_name:payment-gateway-service-7f8b9c6d4f-x2k9m,\
+container_name:payment-gateway,\
+image_tag:registry.internal.example.com/payments/gateway:4.12.7-rc3-sha-a1b2c3d4,\
+team:platform-payments,\
+cost_center:cc-payments-12345,\
+owner:payments-oncall@example.com,\
+pagerduty:payments-p1,\
+slo:payments-availability-99.99,\
+tier:tier-0-critical,\
+compliance:pci-dss-v4,\
+compliance:soc2-type2,\
+compliance:gdpr,\
+datacenter:us-east-1-primary,\
+network:vpc-0a1b2c3d4e5f67890,\
+subnet:subnet-private-us-east-1c-payments,\
+security_group:sg-payment-gateway-prod,\
+load_balancer:arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/payment-gw-prod/50dc6c495c0c9188,\
+target_group:arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/payment-gw-tg/73e2d6bc24d8a067,\
+dns:payment-gateway.internal.prod.example.com,\
+port:8443,\
+protocol:https,\
+framework:spring-boot-3.2.1,\
+runtime:openjdk-21.0.2+13,\
+orchestrator:kubernetes-1.29,\
+deploy_pipeline:argo-cd,\
+deploy_sha:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0,\
+deploy_timestamp:2024-11-15T14:32:07Z,\
+canary:false,\
+feature_flag:new-checkout-flow-v2,\
+feature_flag:payment-retry-logic-v3,\
+feature_flag:fraud-detection-ml-model-2024q4,\
+circuit_breaker:downstream-bank-api,\
+rate_limit_tier:premium,\
+db_pool:payments-primary-rds-cluster.cluster-abc123def456.us-east-1.rds.amazonaws.com,\
+cache_cluster:payments-redis-prod-001.abc123.0001.use1.cache.amazonaws.com,\
+message_queue:arn:aws:sqs:us-east-1:123456789012:payment-events-prod,\
+trace_sample_rate:0.15,\
+log_level:info,\
+custom_metric_prefix:payments.gateway,\
+git_repository:github.com/example-org/payment-gateway-service,\
+oncall_schedule:payments-primary-rotation-2024";
+
 criterion_group!(
     name = benches;
     // encapsulates CI noise we saw in
@@ -205,4 +258,107 @@ fn benchmark_remap(c: &mut Criterion) {
             BatchSize::SmallInput,
         );
     });
+
+    let parse_ddtags_runner = |tform: &mut Box<dyn SyncTransform>, event: Event| {
+        let mut outputs = TransformOutputsBuf::new_with_capacity(
+            vec![TransformOutput::new(DataType::all_bits(), HashMap::new())],
+            1,
+        );
+        tform.transform(event, &mut outputs);
+        let result = outputs.take_primary();
+        let output_1 = result.first().unwrap().as_log();
+
+        debug_assert!(output_1.get(event_path!("parsed")).is_some());
+
+        result
+    };
+
+    group.bench_function("parse_ddtags/native", |b| {
+        let mut tform: Box<dyn SyncTransform> = Box::new(
+            Remap::new_ast(
+                RemapConfig {
+                    source: Some(
+                        r#".parsed = parse_ddtags!(string!(.ddtags))"#.to_string(),
+                    ),
+                    file: None,
+                    timezone: None,
+                    drop_on_error: true,
+                    drop_on_abort: true,
+                    ..Default::default()
+                },
+                &Default::default(),
+            )
+            .unwrap()
+            .0,
+        );
+
+        let event = {
+            let mut event = Event::Log(LogEvent::from("parse ddtags"));
+            event
+                .as_mut_log()
+                .insert(event_path!("ddtags"), DDTAGS_BENCH_INPUT.to_owned());
+            event
+        };
+
+        b.iter_batched(
+            || event.clone(),
+            |event| parse_ddtags_runner(&mut tform, event),
+            BatchSize::SmallInput,
+        );
+    });
+
+    group.bench_function("parse_ddtags/pure_vrl", |b| {
+        let mut tform: Box<dyn SyncTransform> = Box::new(
+            Remap::new_ast(
+                RemapConfig {
+                    source: Some(
+                        indoc! {r#"
+                            tags = split!(string!(.ddtags), ",")
+                            result = {}
+                            for_each(tags) -> |_i, tag| {
+                                parts = split(tag, ":", limit: 2)
+                                key = strip_whitespace!(to_string!(get!(parts, [0])))
+                                val_raw = get(parts, [1]) ?? null
+                                val = if val_raw != null {
+                                    strip_whitespace!(to_string!(val_raw))
+                                } else {
+                                    true
+                                }
+                                existing = get(result, [key]) ?? null
+                                if existing == null {
+                                    result = set!(result, [key], [val])
+                                } else {
+                                    result = set!(result, [key], push!(array!(existing), val))
+                                }
+                            }
+                            .parsed = result
+                        "#}
+                        .to_string(),
+                    ),
+                    file: None,
+                    timezone: None,
+                    drop_on_error: true,
+                    drop_on_abort: true,
+                    ..Default::default()
+                },
+                &Default::default(),
+            )
+            .unwrap()
+            .0,
+        );
+
+        let event = {
+            let mut event = Event::Log(LogEvent::from("parse ddtags"));
+            event
+                .as_mut_log()
+                .insert(event_path!("ddtags"), DDTAGS_BENCH_INPUT.to_owned());
+            event
+        };
+
+        b.iter_batched(
+            || event.clone(),
+            |event| parse_ddtags_runner(&mut tform, event),
+            BatchSize::SmallInput,
+        );
+    });
 }

changelog.d/parse_ddtags_vrl_function.feature.md

@@ -0,0 +1 @@
+Added a `parse_ddtags` VRL function that parses Datadog tag strings (comma-separated `key:value` pairs) into objects. The `multivalue` parameter controls duplicate key handling.

lib/vector-vrl/functions/src/lib.rs

@@ -12,6 +12,7 @@
 use vrl::{compiler::Function, path::OwnedTargetPath};
 
 pub mod get_secret;
+pub mod parse_ddtags;
 pub mod remove_secret;
 pub mod set_secret;
 pub mod set_semantic_meaning;
@@ -30,6 +31,7 @@ pub fn secret_functions() -> Vec<Box<dyn Function>> {
     vec![
         Box::new(set_semantic_meaning::SetSemanticMeaning) as _,
         Box::new(get_secret::GetSecret) as _,
+        Box::new(parse_ddtags::ParseDdtags) as _,
         Box::new(remove_secret::RemoveSecret) as _,
         Box::new(set_secret::SetSecret) as _,
     ]