diff --git a/braintrust-java-agent/smoke-test/otel-agent/src/main/java/dev/braintrust/smoketest/otelagent/OtelAgentSmokeTest.java b/braintrust-java-agent/smoke-test/otel-agent/src/main/java/dev/braintrust/smoketest/otelagent/OtelAgentSmokeTest.java index 6050488f..eb54391a 100644 --- a/braintrust-java-agent/smoke-test/otel-agent/src/main/java/dev/braintrust/smoketest/otelagent/OtelAgentSmokeTest.java +++ b/braintrust-java-agent/smoke-test/otel-agent/src/main/java/dev/braintrust/smoketest/otelagent/OtelAgentSmokeTest.java @@ -113,6 +113,11 @@ private static void assertOtelCollector(MockOtlpCollector collector) throws Exce assertTrue( span.endTimeUnixNano > span.startTimeUnixNano, "span endTimeUnixNano <= startTimeUnixNano"); + if (checkIsRunningWithBraintrustJavaagent()) { + assertNotNull( + span.stringAttr("braintrust.context_json"), + "OTel collector span missing Braintrust provenance"); + } System.out.println("[otel-agent-smoke-test] OTel collector assertions: OK"); } @@ -138,6 +143,9 @@ private static void assertBraintrustBackend(MockOtlpCollector backend) throws Ex var span = backend.findSpanByName(SPAN_NAME); assertNotNull(span, "Braintrust backend missing span with name '" + SPAN_NAME + "'"); + assertNotNull( + span.stringAttr("braintrust.context_json"), + "Braintrust backend span missing provenance"); System.out.println("[otel-agent-smoke-test] Braintrust backend assertions: OK"); } diff --git a/braintrust-sdk/build.gradle b/braintrust-sdk/build.gradle index f3f85ec6..df39a31e 100644 --- a/braintrust-sdk/build.gradle +++ b/braintrust-sdk/build.gradle @@ -145,6 +145,7 @@ dependencies { testImplementation "org.slf4j:slf4j-simple:${slf4jVersion}" testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${otelVersion}" + testImplementation 'io.opentelemetry.proto:opentelemetry-proto:1.11.0-alpha' testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}" testImplementation 'org.wiremock:wiremock:3.13.1' diff --git a/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java b/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java index 5b05962d..9541c824 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java @@ -2,13 +2,17 @@ import dev.braintrust.Braintrust; import dev.braintrust.api.BraintrustOpenApiClient; +import dev.braintrust.trace.SpanCustomizer; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import javax.annotation.Nullable; import javax.net.ssl.SSLContext; @@ -64,6 +68,9 @@ public final class BraintrustConfig extends BaseConfig { /** Custom X509 trust manager for OTLP exporter. Builder-only field, not backed by envars. */ private final X509TrustManager x509TrustManager; + /** Immutable export customizers in registration order. Builder-only, not backed by envars. */ + private final List spanCustomizers; + /** * Sizes the thread pools the SDK creates for batch work (e.g. eval execution), and so how much * of that work runs at once. Used only when the caller has not supplied an executor of their @@ -123,14 +130,15 @@ public static BraintrustConfig of(String... envOverrides) { for (int i = 0; i < envOverrides.length - 1; i = i + 2) { overridesMap.put(envOverrides[i], envOverrides[i + 1]); } - return new BraintrustConfig(overridesMap, null, null); + return new BraintrustConfig(overridesMap, null, null, List.of()); } @SneakyThrows private BraintrustConfig( Map envOverrides, SSLContext sslContext, - X509TrustManager x509TrustManager) { + X509TrustManager x509TrustManager, + List spanCustomizers) { super(envOverrides); if (defaultProjectId.isEmpty() && defaultProjectName.isEmpty()) { // should never happen @@ -145,6 +153,7 @@ private BraintrustConfig( + otelMaxQueueSize); } + this.spanCustomizers = List.copyOf(spanCustomizers); this.sslContext = sslContext != null ? sslContext : SSLContext.getDefault(); if (x509TrustManager != null) { this.x509TrustManager = x509TrustManager; @@ -191,6 +200,7 @@ public static class Builder { private final Map envOverrides = new HashMap<>(); private SSLContext sslContext; private X509TrustManager x509TrustManager; + private final List spanCustomizers = new ArrayList<>(); public Builder apiKey(String value) { envOverrides.put("BRAINTRUST_API_KEY", value); @@ -325,8 +335,20 @@ public Builder otelExportIntervalMillis(int exportIntervalMillis) { return this; } + /** + * Adds a span customizer and run its hooks over all spans which pass through braintrust + * export + * + * @throws NullPointerException if {@code customizer} is null + */ + public Builder addSpanCustomizer(SpanCustomizer customizer) { + spanCustomizers.add(Objects.requireNonNull(customizer, "customizer")); + return this; + } + public BraintrustConfig build() { - return new BraintrustConfig(envOverrides, sslContext, x509TrustManager); + return new BraintrustConfig( + envOverrides, sslContext, x509TrustManager, spanCustomizers); } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSampler.java b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSampler.java index 450bb407..2fad23c7 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSampler.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSampler.java @@ -4,11 +4,14 @@ import java.util.List; /** - * A filter that decides whether a finalized span should be exported. Samplers are evaluated in - * {@link BraintrustSpanProcessor#onEnd} after all attributes have been set on the span. + * A filter that decides whether a span should be enriched and exported. Samplers are evaluated once + * in {@link BraintrustSpanProcessor#onEnding}, before enrichment and delegated ending hooks. */ interface BraintrustSampler { - /** Returns {@code true} if the span should be exported, {@code false} to discard it. */ + /** + * Returns {@code true} to enrich and export the span, {@code false} to discard it. The span is + * still ending; implementations must only read it, not downcast it to mutate it. + */ boolean sample(ReadableSpan span); /** Keeps only spans that have at least one attribute with a known AI instrumentation prefix. */ diff --git a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java index 2a431f74..fd4b62d4 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java @@ -33,8 +33,18 @@ public CompletableResultCode export(Collection spans) { return CompletableResultCode.ofSuccess(); } - // Group spans by their parent (project or experiment) - var spansByParent = spans.stream().collect(Collectors.groupingBy(this::getParentFromSpan)); + // Finish customization before sending any group, so failed redaction cannot leak a batch. + Map> spansByParent; + try { + var exportSpans = spans.stream(); + if (!config.spanCustomizers().isEmpty()) { + exportSpans = exportSpans.map(this::customizeSpan); + } + spansByParent = exportSpans.collect(Collectors.groupingBy(this::getParentFromSpan)); + } catch (Exception e) { + log.error("Failed to customize spans for export", e); + return CompletableResultCode.ofFailure(); + } // Export each group with the appropriate x-bt-parent header var results = @@ -49,6 +59,30 @@ public CompletableResultCode export(Collection spans) { return combined; } + private SpanData customizeSpan(SpanData span) { + var traceId = span.getTraceId(); + var spanId = span.getSpanId(); + var parentSpanId = span.getParentSpanId(); + var current = span; + for (var customizer : config.spanCustomizers()) { + current = customizer.onSpanExport(current); + if (current == null) { + throw new IllegalStateException("SpanCustomizer.onSpanExport must not return null"); + } + if (!traceId.equals(current.getTraceId()) + || !spanId.equals(current.getSpanId()) + || !parentSpanId.equals(current.getParentSpanId()) + || !traceId.equals(current.getSpanContext().getTraceId()) + || !spanId.equals(current.getSpanContext().getSpanId()) + || !parentSpanId.equals(current.getParentSpanContext().getSpanId())) { + throw new IllegalStateException( + "SpanCustomizer.onSpanExport must not change trace ID, span ID, or parent" + + " span ID"); + } + } + return current; + } + private String getParentFromSpan(SpanData span) { var parent = span.getAttributes().get(BraintrustSpanProcessor.PARENT); if (parent != null) { diff --git a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSpanProcessor.java b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSpanProcessor.java index c243bb36..485104a4 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSpanProcessor.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustSpanProcessor.java @@ -5,20 +5,20 @@ import dev.braintrust.config.SpanOriginEnvironment; import dev.braintrust.json.BraintrustJsonMapper; import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.common.Attributes; import io.opentelemetry.context.Context; import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.common.InstrumentationLibraryInfo; import io.opentelemetry.sdk.trace.ReadWriteSpan; import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.SpanProcessor; -import io.opentelemetry.sdk.trace.data.DelegatingSpanData; -import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.internal.ExtendedSpanProcessor; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.WeakHashMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import javax.annotation.Nonnull; @@ -28,9 +28,13 @@ /** * Custom span processor that enriches spans with Braintrust-specific attributes. Supports parent * assignment to projects or experiments. + * + *

Uses OpenTelemetry's internal, unstable ending hook to mutate the real span before it becomes + * immutable. Enrichment is visible to all processors exporting that span and obeys the SDK's + * configured attribute count and value-length limits. */ @Slf4j -public class BraintrustSpanProcessor implements SpanProcessor { +public class BraintrustSpanProcessor implements ExtendedSpanProcessor { static final AttributeKey INPUT_JSON = AttributeKey.stringKey("braintrust.input_json"); static final AttributeKey OUTPUT_JSON = AttributeKey.stringKey("braintrust.output_json"); @@ -43,6 +47,9 @@ public class BraintrustSpanProcessor implements SpanProcessor { private final BraintrustConfig config; private final SpanProcessor delegate; private final List samplers; + // Keep accepted decisions until onEnd without retaining spans if another ending hook throws. + private final Set sampledSpans = + Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<>())); private final ConcurrentMap parentContexts = new ConcurrentHashMap<>(); private final AttachmentProcessor attachmentProcessor; @@ -157,10 +164,7 @@ public boolean isStartRequired() { } @Override - public void onEnd(ReadableSpan span) { - if (config.debug()) { - logSpanDetails(span); - } + public void onEnding(ReadWriteSpan span) { for (var sampler : samplers) { if (!sampler.sample(span)) { log.debug( @@ -170,34 +174,56 @@ public void onEnd(ReadableSpan span) { return; } } + if (!samplers.isEmpty()) { + sampledSpans.add(span); + } - var spanData = span.toSpanData(); - @Nullable String inputJson = spanData.getAttributes().get(INPUT_JSON); - @Nullable String outputJson = spanData.getAttributes().get(OUTPUT_JSON); + try { + var inputJson = span.getAttribute(INPUT_JSON); + var newInputJson = attachmentProcessor.processAndUpload(inputJson); + if (newInputJson != null && !Objects.equals(newInputJson, inputJson)) { + span.setAttribute(INPUT_JSON, newInputJson); + } - @Nullable String newInputJson = attachmentProcessor.processAndUpload(inputJson); - @Nullable String newOutputJson = attachmentProcessor.processAndUpload(outputJson); - var spanOriginEnvironment = config.spanOriginEnvironment().orElse(null); - var newContextJson = - mergedContextJson( - spanData.getAttributes().get(CONTEXT_JSON), spanOriginEnvironment); + var outputJson = span.getAttribute(OUTPUT_JSON); + var newOutputJson = attachmentProcessor.processAndUpload(outputJson); + if (newOutputJson != null && !Objects.equals(newOutputJson, outputJson)) { + span.setAttribute(OUTPUT_JSON, newOutputJson); + } - if (!Objects.equals(newInputJson, inputJson) - || !Objects.equals(newOutputJson, outputJson)) { - delegate.onEnd( - new TransformedReadableSpan( - span, - newInputJson, - newOutputJson, - newContextJson, - spanOriginEnvironment)); - } else { - delegate.onEnd( - new TransformedReadableSpan( - span, inputJson, outputJson, newContextJson, spanOriginEnvironment)); + var contextJson = span.getAttribute(CONTEXT_JSON); + var newContextJson = + mergedContextJson(contextJson, config.spanOriginEnvironment().orElse(null)); + if (!Objects.equals(newContextJson, contextJson)) { + span.setAttribute(CONTEXT_JSON, newContextJson); + } + } catch (Exception e) { + log.warn("Failed to enrich ending span; exporting available attributes", e); + } + + if (delegate instanceof ExtendedSpanProcessor extendedDelegate + && extendedDelegate.isOnEndingRequired()) { + extendedDelegate.onEnding(span); } } + @Override + public boolean isOnEndingRequired() { + return true; + } + + @Override + public void onEnd(ReadableSpan span) { + if (config.debug()) { + logSpanDetails(span); + } + if (!samplers.isEmpty() && !sampledSpans.remove(span)) { + return; + } + + delegate.onEnd(span); + } + @Override public boolean isEndRequired() { return true; @@ -252,106 +278,4 @@ public static ParentContext experiment(String experimentId) { return new ParentContext(null, experimentId, ParentType.EXPERIMENT); } } - - /** - * otel java does not implement onEnding, so this is the most idiomatic way to mutate a span - * once it ends - */ - private static class TransformedReadableSpan implements ReadableSpan { - private final ReadableSpan delegate; - private final Attributes attributes; - - TransformedReadableSpan( - ReadableSpan delegate, - String inputJson, - String outputJson, - String contextJson, - @Nullable SpanOriginEnvironment environment) { - this.delegate = delegate; - var builder = delegate.getAttributes().toBuilder(); - if (inputJson != null) { - builder.put(INPUT_JSON, inputJson); - } - if (outputJson != null) { - builder.put(OUTPUT_JSON, outputJson); - } - builder.put(CONTEXT_JSON, contextJson); - attributes = builder.build(); - } - - @Override - public Attributes getAttributes() { - return attributes; - } - - @Override - @SuppressWarnings("unchecked") - public T getAttribute(AttributeKey key) { - if (key.equals(INPUT_JSON)) { - return (T) attributes.get(INPUT_JSON); - } - if (key.equals(OUTPUT_JSON)) { - return (T) attributes.get(OUTPUT_JSON); - } - if (key.equals(CONTEXT_JSON)) { - return (T) attributes.get(CONTEXT_JSON); - } - return delegate.getAttribute(key); - } - - @Override - public SpanData toSpanData() { - return new DelegatingSpanData(delegate.toSpanData()) { - @Override - public io.opentelemetry.api.common.Attributes getAttributes() { - return TransformedReadableSpan.this.getAttributes(); - } - - @Override - public int getTotalAttributeCount() { - return getAttributes().size(); - } - }; - } - - @Override - public String getName() { - return delegate.getName(); - } - - @Override - public io.opentelemetry.api.trace.SpanContext getSpanContext() { - return delegate.getSpanContext(); - } - - @Override - public boolean hasEnded() { - return delegate.hasEnded(); - } - - @Override - public io.opentelemetry.sdk.common.InstrumentationScopeInfo getInstrumentationScopeInfo() { - return delegate.getInstrumentationScopeInfo(); - } - - @Override - public InstrumentationLibraryInfo getInstrumentationLibraryInfo() { - return delegate.getInstrumentationLibraryInfo(); - } - - @Override - public long getLatencyNanos() { - return delegate.getLatencyNanos(); - } - - @Override - public io.opentelemetry.api.trace.SpanContext getParentSpanContext() { - return delegate.getParentSpanContext(); - } - - @Override - public io.opentelemetry.api.trace.SpanKind getKind() { - return delegate.getKind(); - } - } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustTracing.java b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustTracing.java index 99964e5b..1126299f 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustTracing.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/trace/BraintrustTracing.java @@ -113,7 +113,9 @@ public static void enable( * etc.). * *

The additional processors are composited into the {@link BraintrustSpanProcessor}'s - * delegate, so they see the transformed span data rather than the raw original. + * delegate. Accepted spans reach their ending callbacks after Braintrust enrichment, including + * {@code onEnding} if they implement OpenTelemetry's extended processor interface. Spans + * rejected by Braintrust sampling skip enrichment and both delegate ending callbacks. * * @param additionalDelegates extra span processors that receive post-processed spans alongside * the Braintrust exporter. Pass {@code List.of()} if none are needed. diff --git a/braintrust-sdk/src/main/java/dev/braintrust/trace/SpanCustomizer.java b/braintrust-sdk/src/main/java/dev/braintrust/trace/SpanCustomizer.java new file mode 100644 index 00000000..8095dcce --- /dev/null +++ b/braintrust-sdk/src/main/java/dev/braintrust/trace/SpanCustomizer.java @@ -0,0 +1,18 @@ +package dev.braintrust.trace; + +import io.opentelemetry.sdk.trace.data.SpanData; + +/** + * A callback hook for altering span data. This is primarly used by SDK users who wish to customize + * auto-instrumented spans. + */ +public interface SpanCustomizer { + /** + * called just before a span is exported. You may add/remove/delete most fields on the span. + * + *

The follow fields may NOT be altered: trace id, span id, parent id + */ + default SpanData onSpanExport(SpanData span) { + return span; + } +} diff --git a/braintrust-sdk/src/test/java/dev/braintrust/config/BraintrustConfigTest.java b/braintrust-sdk/src/test/java/dev/braintrust/config/BraintrustConfigTest.java index bf7a9b8a..66482fc2 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/config/BraintrustConfigTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/config/BraintrustConfigTest.java @@ -6,6 +6,7 @@ import java.lang.reflect.Method; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import javax.net.ssl.SSLContext; @@ -52,25 +53,48 @@ public void testBuilderEqualsEnv() { @Test public void testBuilderHasMethodForEveryField() { - List fieldsToSkip = List.of("envOverrides"); + final Map> builderMethodOverrides = + Map.of("spanCustomizers", List.of("addSpanCustomizer")); // Get all fields from BraintrustConfig Field[] configFields = BraintrustConfig.class.getDeclaredFields(); + Set configFieldNames = + Arrays.stream(configFields).map(Field::getName).collect(Collectors.toSet()); // Get all methods from Builder Method[] builderMethods = BraintrustConfig.Builder.class.getDeclaredMethods(); Set builderMethodNames = Arrays.stream(builderMethods).map(Method::getName).collect(Collectors.toSet()); + // Validate overrides even when the field is skipped or its method list is empty. + builderMethodOverrides.forEach( + (fieldName, methodNames) -> { + assertTrue( + configFieldNames.contains(fieldName), + "Builder override references unknown field: " + fieldName); + for (String methodName : methodNames) { + assertTrue( + builderMethodNames.contains(methodName), + "Builder override for field " + + fieldName + + " references unknown method: " + + methodName); + } + }); + // For each field, verify there's a corresponding builder method for (Field field : configFields) { String configFieldName = field.getName(); - // Skip internal fields - if (fieldsToSkip.contains(configFieldName)) { - continue; + // An explicit empty list exempts a field from requiring builder methods. + for (String builderMethodName : + builderMethodOverrides.getOrDefault( + configFieldName, List.of(configFieldName))) { + assertTrue( + builderMethodNames.contains(builderMethodName), + "Builder is missing method " + + builderMethodName + + " for field: " + + configFieldName); } - assertTrue( - builderMethodNames.contains(configFieldName), - "Builder is missing method for field: " + configFieldName); } } diff --git a/braintrust-sdk/src/test/java/dev/braintrust/trace/AttachmentProcessorTest.java b/braintrust-sdk/src/test/java/dev/braintrust/trace/AttachmentProcessorTest.java index 1f2adb45..03477ecb 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/trace/AttachmentProcessorTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/trace/AttachmentProcessorTest.java @@ -7,6 +7,7 @@ import dev.braintrust.json.BraintrustJsonMapper; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.trace.ReadableSpan; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Stream; @@ -174,10 +175,6 @@ static Stream attachmentFormatCases() { @MethodSource("attachmentFormatCases") @SneakyThrows void attachmentFormatReplacesBase64WithRef(FormatTestCase testCase) { - assertTrue( - AttachmentProcessor.BASE64_HEURISTIC.matcher(testCase.inputJson).find(), - "BASE64_HEURISTIC should match test data"); - String spanName = "fmt-" + testCase.name + "-" + spanCounter.incrementAndGet(); var span = tracer.spanBuilder(spanName).startSpan(); span.setAttribute("braintrust.input_json", testCase.inputJson); @@ -196,6 +193,9 @@ void attachmentFormatReplacesBase64WithRef(FormatTestCase testCase) { assertNotEquals(testCase.inputJson, exportedJson, "base64 data should have been replaced"); testCase.assertions.accept(BraintrustJsonMapper.get().readTree(exportedJson)); + testCase.assertions.accept( + BraintrustJsonMapper.get() + .readTree(((ReadableSpan) span).getAttribute(INPUT_JSON))); } // ── Negative cases ──────────────────────────────────────────────── diff --git a/braintrust-sdk/src/test/java/dev/braintrust/trace/BraintrustSpanExporterTest.java b/braintrust-sdk/src/test/java/dev/braintrust/trace/BraintrustSpanExporterTest.java index 0ad34c4b..ec07a5ff 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/trace/BraintrustSpanExporterTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/trace/BraintrustSpanExporterTest.java @@ -7,31 +7,58 @@ import dev.braintrust.Braintrust; import dev.braintrust.config.BraintrustConfig; import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.logs.SdkLoggerProvider; import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.DelegatingSpanData; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.io.ByteArrayInputStream; import java.io.FileInputStream; import java.net.InetSocketAddress; import java.nio.file.Files; import java.security.*; import java.time.Duration; +import java.util.HexFormat; +import java.util.List; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.GZIPInputStream; import javax.net.ssl.*; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; public class BraintrustSpanExporterTest { + private static final AttributeKey KEEP = AttributeKey.stringKey("custom.keep"); + private static final AttributeKey PARENT = + AttributeKey.stringKey(BraintrustTracing.PARENT_KEY); + private static final SpanContext PARENT_CONTEXT = + SpanContext.create( + "1234567890abcdef1234567890abcdef", + "1234567890abcdef", + TraceFlags.getSampled(), + TraceState.getDefault()); + private HttpsServer server; private SSLContext serverSslContext; private SSLContext clientSslContext; private X509TrustManager clientTrustManager; private int port; - private AtomicBoolean requestReceived; + private BlockingQueue requests; private CountDownLatch requestLatch; private java.nio.file.Path keystoreFile; @@ -39,7 +66,7 @@ public class BraintrustSpanExporterTest { void setUp() throws Exception { GlobalOpenTelemetry.resetForTest(); - requestReceived = new AtomicBoolean(false); + requests = new LinkedBlockingQueue<>(); requestLatch = new CountDownLatch(1); // Generate self-signed certificate using keytool @@ -75,7 +102,11 @@ void setUp() throws Exception { "-storetype", "JKS") .start(); - int exitCode = keytoolProcess.waitFor(); + if (!keytoolProcess.waitFor(30, TimeUnit.SECONDS)) { + keytoolProcess.destroyForcibly(); + fail("keytool did not finish within 30 seconds"); + } + int exitCode = keytoolProcess.exitValue(); if (exitCode != 0) { throw new RuntimeException("keytool failed with exit code " + exitCode); } @@ -114,11 +145,16 @@ void setUp() throws Exception { server.createContext( "/otel/v1/traces", exchange -> { - requestReceived.set(true); - requestLatch.countDown(); - // Return 200 OK - exchange.sendResponseHeaders(200, 0); - exchange.getResponseBody().close(); + try (exchange) { + var body = exchange.getRequestBody().readAllBytes(); + requests.add( + new CapturedRequest( + body, + exchange.getRequestHeaders().getFirst("Content-Encoding"), + exchange.getRequestHeaders().getFirst("x-bt-parent"))); + requestLatch.countDown(); + exchange.sendResponseHeaders(200, 0); + } }); server.start(); @@ -137,57 +173,413 @@ void tearDown() throws Exception { @Test void testCustomSslContextAndTrustManager() throws Exception { - // Create config with custom SSL context + var exported = + exportSpan( + configBuilder().build(), + Attributes.builder().put("test-attr", "test-value").build()); + + assertStringAttribute(exported.span(), "test-attr", "test-value"); + } + + @Test + void spanExportAddsAttribute() throws Exception { + var config = + configBuilder() + .addSpanCustomizer( + new SpanCustomizer() { + @Override + public SpanData onSpanExport(SpanData span) { + var attributes = + span.getAttributes().toBuilder() + .put("custom.added", "new-value") + .build(); + return withAttributes(span, attributes); + } + }) + .build(); + + var exported = exportSpan(config, Attributes.of(KEEP, "unchanged")); + var protoSpan = exported.span(); + assertStringAttribute(protoSpan, "custom.added", "new-value"); + assertStringAttribute(protoSpan, KEEP.getKey(), "unchanged"); + assertEquals( + exported.originalContext().getTraceId(), + HexFormat.of().formatHex(protoSpan.getTraceId().toByteArray())); + assertEquals( + exported.originalContext().getSpanId(), + HexFormat.of().formatHex(protoSpan.getSpanId().toByteArray())); + assertEquals( + PARENT_CONTEXT.getSpanId(), + HexFormat.of().formatHex(protoSpan.getParentSpanId().toByteArray())); + } + + @Test + void spanExportMutatesExistingAttribute() throws Exception { + var config = + configBuilder() + .addSpanCustomizer( + new SpanCustomizer() { + @Override + public SpanData onSpanExport(SpanData span) { + var attributes = + span.getAttributes().toBuilder() + .put("custom.label", "after") + .build(); + return withAttributes(span, attributes); + } + }) + .build(); + + var exported = + exportSpan( + config, + Attributes.builder() + .put("custom.label", "before") + .put(KEEP, "unchanged") + .build()); + assertStringAttribute(exported.span(), "custom.label", "after"); + assertStringAttribute(exported.span(), KEEP.getKey(), "unchanged"); + } + + @Test + void spanExportDeletesAttribute() throws Exception { var config = - BraintrustConfig.builder() - .apiKey("test-key") - .apiUrl("https://localhost:" + port) - .defaultProjectName("test-project") - .requestTimeout(Duration.ofSeconds(5)) - .sslContext(clientSslContext) - .x509TrustManager(clientTrustManager) + configBuilder() + .addSpanCustomizer( + new SpanCustomizer() { + @Override + public SpanData onSpanExport(SpanData span) { + var attributes = + span.getAttributes().toBuilder() + .remove( + AttributeKey.stringKey( + "custom.secret")) + .build(); + return withAttributes(span, attributes); + } + }) .build(); - // Verify config stores the SSL context and trust manager - assertNotNull(config.sslContext()); - assertNotNull(config.x509TrustManager()); - assertSame(clientSslContext, config.sslContext()); - assertSame(clientTrustManager, config.x509TrustManager()); + var exported = + exportSpan( + config, + Attributes.builder() + .put("custom.secret", "remove-me") + .put(KEEP, "unchanged") + .build()); + assertNoAttribute(exported.span(), "custom.secret"); + assertStringAttribute(exported.span(), KEEP.getKey(), "unchanged"); + assertEquals(0, exported.span().getDroppedAttributesCount()); + } + + @Test + void spanExportComposesCustomizersWithoutChangingBuiltConfigs() throws Exception { + var chain = AttributeKey.stringKey("custom.chain"); + var builder = + configBuilder() + .addSpanCustomizer( + new SpanCustomizer() { + @Override + public SpanData onSpanExport(SpanData span) { + return withAttributes( + span, + span.getAttributes().toBuilder() + .put(chain, "A") + .build()); + } + }); + var configA = builder.build(); + var configB = + builder.addSpanCustomizer( + new SpanCustomizer() { + @Override + public SpanData onSpanExport(SpanData span) { + return withAttributes( + span, + span.getAttributes().toBuilder() + .put( + chain, + span.getAttributes().get(chain) + + "B") + .build()); + } + }) + .build(); + + assertStringAttribute(exportSpan(configA, Attributes.empty()).span(), chain.getKey(), "A"); + assertStringAttribute(exportSpan(configB, Attributes.empty()).span(), chain.getKey(), "AB"); + } + + @Test + void spanExportRoutesUsingCustomizedParent() throws Exception { + var config = + configBuilder() + .addSpanCustomizer( + new SpanCustomizer() { + @Override + public SpanData onSpanExport(SpanData span) { + return withAttributes( + span, + span.getAttributes().toBuilder() + .put(PARENT, "project_name:customized") + .build()); + } + }) + .build(); + + var exported = exportSpan(config, Attributes.of(PARENT, "project_name:original")); + assertStringAttribute(exported.span(), PARENT.getKey(), "project_name:customized"); + assertEquals("project_name:customized", exported.parentHeader()); + } + + @Test + void spanExportUsesConfiguredParentWhenCustomizerDeletesRoutingAttribute() throws Exception { + var config = + configBuilder() + .addSpanCustomizer( + new SpanCustomizer() { + @Override + public SpanData onSpanExport(SpanData span) { + return withAttributes( + span, + span.getAttributes().toBuilder() + .remove(PARENT) + .build()); + } + }) + .build(); - // Set up OpenTelemetry with Braintrust + var exported = exportSpan(config, Attributes.of(PARENT, "project_name:original")); + assertNoAttribute(exported.span(), PARENT.getKey()); + assertEquals("project_name:test-project", exported.parentHeader()); + } + + @ParameterizedTest + @EnumSource(InvalidCustomization.class) + void invalidCustomizerFailsEntireBatchBeforeTransport(InvalidCustomization invalid) + throws Exception { + var config = + configBuilder() + .addSpanCustomizer( + new SpanCustomizer() { + @Override + public SpanData onSpanExport(SpanData span) { + if (span.getName().equals("invalid")) { + return invalid.customize(span); + } + return withAttributes( + span, + span.getAttributes().toBuilder() + .put(PARENT, "project_name:customized") + .build()); + } + }) + .build(); + + try (var provider = SdkTracerProvider.builder().build(); + var exporter = new BraintrustSpanExporter(config)) { + var tracer = provider.get("test-tracer"); + var first = tracer.spanBuilder("valid").setParent(parentContext()).startSpan(); + var second = tracer.spanBuilder("invalid").setParent(parentContext()).startSpan(); + first.end(); + second.end(); + + var result = + exporter.export( + List.of( + ((ReadableSpan) first).toSpanData(), + ((ReadableSpan) second).toSpanData())) + .join(10, TimeUnit.SECONDS); + assertTrue(result.isDone(), "Export must finish"); + assertFalse(result.isSuccess(), "Invalid customization must fail the entire batch"); + } + assertNull( + requests.poll(300, TimeUnit.MILLISECONDS), + "No span from a failed batch may reach HTTP transport"); + assertTrue(requests.isEmpty(), "No additional export requests are allowed"); + } + + private BraintrustConfig.Builder configBuilder() { + return BraintrustConfig.builder() + .apiKey("test-key") + .apiUrl("https://localhost:" + port) + .filterAISpans(false) + .defaultProjectId(null) + .defaultProjectName("test-project") + .requestTimeout(Duration.ofSeconds(5)) + .sslContext(clientSslContext) + .x509TrustManager(clientTrustManager); + } + + private ExportedSpan exportSpan(BraintrustConfig config, Attributes attributes) + throws Exception { var tracerBuilder = SdkTracerProvider.builder(); var loggerBuilder = SdkLoggerProvider.builder(); var meterBuilder = SdkMeterProvider.builder(); + Braintrust.of(config).openTelemetryEnable(tracerBuilder, loggerBuilder, meterBuilder); - var braintrust = Braintrust.of(config); - braintrust.openTelemetryEnable(tracerBuilder, loggerBuilder, meterBuilder); - - var openTelemetry = + try (var openTelemetry = OpenTelemetrySdk.builder() .setTracerProvider(tracerBuilder.build()) .setLoggerProvider(loggerBuilder.build()) .setMeterProvider(meterBuilder.build()) - .build(); - - // Create and export a span - Tracer tracer = openTelemetry.getTracer("test-tracer"); - var span = tracer.spanBuilder("test-span").startSpan(); - try { - span.setAttribute("test-attr", "test-value"); - } finally { + .build()) { + var span = + openTelemetry + .getTracer("test-tracer") + .spanBuilder("test-span") + .setParent(parentContext()) + .setAllAttributes(attributes) + .startSpan(); + var originalContext = span.getSpanContext(); span.end(); + + var result = + openTelemetry.getSdkTracerProvider().forceFlush().join(10, TimeUnit.SECONDS); + assertTrue(result.isDone(), "Force flush must finish"); + assertTrue(result.isSuccess(), "Force flush must succeed"); + assertTrue( + requestLatch.await(10, TimeUnit.SECONDS), "Expected an HTTPS export request"); + var request = requests.poll(10, TimeUnit.SECONDS); + assertNotNull(request, "Expected a complete OTLP request body"); + var spans = request.decodeSpans(); + assertEquals(1, spans.size(), "Expected exactly the manually created span"); + return new ExportedSpan(spans.get(0), originalContext, request.parentHeader()); } + } - // Force flush to ensure span is exported - openTelemetry.getSdkTracerProvider().forceFlush().join(10, TimeUnit.SECONDS); + private static Context parentContext() { + return Context.root().with(Span.wrap(PARENT_CONTEXT)); + } - // Wait for the request to reach our test server - boolean received = requestLatch.await(10, TimeUnit.SECONDS); - assertTrue( - received, - "Expected span export request to reach test HTTPS server with custom SSL context"); + private static SpanData withAttributes(SpanData span, Attributes attributes) { + var totalAttributeCount = + attributes.size() + + Math.max(0, span.getTotalAttributeCount() - span.getAttributes().size()); + return new DelegatingSpanData(span) { + @Override + public Attributes getAttributes() { + return attributes; + } + + @Override + public int getTotalAttributeCount() { + return totalAttributeCount; + } + }; + } + + private static void assertStringAttribute( + io.opentelemetry.proto.trace.v1.Span span, String key, String expected) { + var values = + span.getAttributesList().stream() + .filter(attribute -> attribute.getKey().equals(key)) + .map(attribute -> attribute.getValue().getStringValue()) + .toList(); + assertEquals(List.of(expected), values, "Unexpected exported attribute " + key); + } + + private static void assertNoAttribute(io.opentelemetry.proto.trace.v1.Span span, String key) { assertTrue( - requestReceived.get(), - "Expected request handler to be invoked on test HTTPS server"); + span.getAttributesList().stream() + .noneMatch(attribute -> attribute.getKey().equals(key)), + "Attribute must not be exported: " + key); + } + + private record ExportedSpan( + io.opentelemetry.proto.trace.v1.Span span, + SpanContext originalContext, + String parentHeader) {} + + private record CapturedRequest(byte[] body, String contentEncoding, String parentHeader) { + List decodeSpans() throws Exception { + byte[] payload = body; + if ("gzip".equalsIgnoreCase(contentEncoding)) { + try (var gzip = new GZIPInputStream(new ByteArrayInputStream(body))) { + payload = gzip.readAllBytes(); + } + } + return ExportTraceServiceRequest.parseFrom(payload).getResourceSpansList().stream() + .flatMap(resource -> resource.getScopeSpansList().stream()) + .flatMap(scope -> scope.getSpansList().stream()) + .toList(); + } + } + + private enum InvalidCustomization { + THROWS, + RETURNS_NULL, + TRACE_ID, + SPAN_ID, + PARENT_SPAN_ID, + CONTEXT_TRACE_ID, + CONTEXT_SPAN_ID, + CONTEXT_PARENT_SPAN_ID; + + SpanData customize(SpanData span) { + return switch (this) { + case THROWS -> throw new IllegalStateException("customizer failed"); + case RETURNS_NULL -> null; + case TRACE_ID -> + new DelegatingSpanData(span) { + @Override + public String getTraceId() { + return "ffffffffffffffffffffffffffffffff"; + } + }; + case SPAN_ID -> + new DelegatingSpanData(span) { + @Override + public String getSpanId() { + return differentSpanId(span.getSpanId()); + } + }; + case PARENT_SPAN_ID -> + new DelegatingSpanData(span) { + @Override + public String getParentSpanId() { + return differentSpanId(span.getParentSpanId()); + } + }; + case CONTEXT_TRACE_ID -> + new DelegatingSpanData(span) { + @Override + public SpanContext getSpanContext() { + return SpanContext.create( + "ffffffffffffffffffffffffffffffff", + span.getSpanId(), + span.getSpanContext().getTraceFlags(), + span.getSpanContext().getTraceState()); + } + }; + case CONTEXT_SPAN_ID -> + new DelegatingSpanData(span) { + @Override + public SpanContext getSpanContext() { + return SpanContext.create( + span.getTraceId(), + differentSpanId(span.getSpanId()), + span.getSpanContext().getTraceFlags(), + span.getSpanContext().getTraceState()); + } + }; + case CONTEXT_PARENT_SPAN_ID -> + new DelegatingSpanData(span) { + @Override + public SpanContext getParentSpanContext() { + var parent = span.getParentSpanContext(); + return SpanContext.create( + parent.getTraceId(), + differentSpanId(parent.getSpanId()), + parent.getTraceFlags(), + parent.getTraceState()); + } + }; + }; + } + + private static String differentSpanId(String original) { + return original.equals("ffffffffffffffff") ? "eeeeeeeeeeeeeeee" : "ffffffffffffffff"; + } } } diff --git a/braintrust-sdk/src/test/java/dev/braintrust/trace/BraintrustTracingTest.java b/braintrust-sdk/src/test/java/dev/braintrust/trace/BraintrustTracingTest.java index a3437458..32dfdd20 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/trace/BraintrustTracingTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/trace/BraintrustTracingTest.java @@ -8,8 +8,14 @@ import dev.braintrust.json.BraintrustJsonMapper; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SpanLimits; +import io.opentelemetry.sdk.trace.SpanProcessor; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.internal.ExtendedSpanProcessor; import java.util.Map; import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.BeforeEach; @@ -73,8 +79,7 @@ void spanProcessorAddsParentFromConfig() { @Test void filterAISpansDropsNonAISpans() { - // Build a tracer where the BraintrustSpanProcessor wraps a test exporter, - // so the filter in onEnd() gates what the exporter receives. + // Build a tracer where the BraintrustSpanProcessor gates what the exporter receives. var config = BraintrustConfig.builder() .apiKey("test-key") @@ -123,9 +128,14 @@ void spanOriginMergesWithContextJsonSetAfterSpanStart() throws Exception { .build(); var spanExporter = new UnitTestSpanExporter(); + var siblingExporter = new UnitTestSpanExporter(); var processor = new BraintrustSpanProcessor(config, SimpleSpanProcessor.create(spanExporter)); - var tracerProvider = SdkTracerProvider.builder().addSpanProcessor(processor).build(); + var tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(siblingExporter)) + .addSpanProcessor(processor) + .build(); var tracer = tracerProvider.get("test"); var span = tracer.spanBuilder("late-context").startSpan(); @@ -140,6 +150,13 @@ void spanOriginMergesWithContextJsonSetAfterSpanStart() throws Exception { var contextJson = spans.get(0).getAttributes().get(AttributeKey.stringKey("braintrust.context_json")); assertNotNull(contextJson); + assertEquals( + contextJson, + siblingExporter + .getFinishedSpanItems() + .get(0) + .getAttributes() + .get(BraintrustSpanProcessor.CONTEXT_JSON)); var context = BraintrustJsonMapper.get().readValue(contextJson, Map.class); assertEquals("late-attribute", ((Map) context.get("metadata")).get("source")); var spanOrigin = (Map) context.get("span_origin"); @@ -147,6 +164,147 @@ void spanOriginMergesWithContextJsonSetAfterSpanStart() throws Exception { assertEquals( BraintrustTracing.INSTRUMENTATION_NAME, ((Map) spanOrigin.get("instrumentation")).get("name")); + tracerProvider.close(); + } + + @Test + void spanOriginRespectsAttributeCapacity() throws Exception { + var config = + BraintrustConfig.builder() + .apiKey("test-key") + .apiUrl("http://localhost:1234") + .filterAISpans(false) + .defaultProjectName("capacity-test") + .build(); + var exporter = new UnitTestSpanExporter(); + try (var provider = + SdkTracerProvider.builder() + .setSpanLimits(SpanLimits.builder().setMaxNumberOfAttributes(2).build()) + .addSpanProcessor( + new BraintrustSpanProcessor( + config, SimpleSpanProcessor.create(exporter))) + .build()) { + var tracer = provider.get("test"); + var fullSpan = tracer.spanBuilder("full").startSpan(); + fullSpan.setAttribute("custom.key", "kept"); + fullSpan.end(); + + var existingContextSpan = + tracer.spanBuilder("existing-context") + .setAttribute(BraintrustSpanProcessor.CONTEXT_JSON, "{}") + .startSpan(); + existingContextSpan.end(); + + var spans = exporter.getFinishedSpanItems(); + assertEquals(2, spans.size()); + assertEquals( + "kept", spans.get(0).getAttributes().get(AttributeKey.stringKey("custom.key"))); + assertNull(spans.get(0).getAttributes().get(BraintrustSpanProcessor.CONTEXT_JSON)); + var context = + BraintrustJsonMapper.get() + .readTree( + spans.get(1) + .getAttributes() + .get(BraintrustSpanProcessor.CONTEXT_JSON)); + assertEquals("braintrust.sdk.java", context.path("span_origin").path("name").asText()); + } + } + + @Test + void extendedDelegateMutatesSpanBeforeExport() { + var config = + BraintrustConfig.builder() + .apiKey("test-key") + .apiUrl("http://localhost:1234") + .filterAISpans(true) + .build(); + var exporter = new UnitTestSpanExporter(); + var extendedDelegate = + new ExtendedSpanProcessor() { + @Override + public void onStart(Context parentContext, ReadWriteSpan span) {} + + @Override + public boolean isStartRequired() { + return false; + } + + @Override + public void onEnding(ReadWriteSpan span) { + var context = + BraintrustJsonMapper.fromJson( + span.getAttribute(BraintrustSpanProcessor.CONTEXT_JSON), + Map.class); + if (span.getName().equals("accepted")) { + assertEquals( + "late-attribute", + ((Map) context.get("metadata")).get("source")); + } + assertEquals( + "braintrust.sdk.java", + ((Map) context.get("span_origin")).get("name")); + span.setAttribute("smoke.ending", true); + // The export decision must not be re-evaluated after ending hooks. + span.setAttribute(AttributeKey.stringKey("gen_ai.model"), (String) null); + } + + @Override + public boolean isOnEndingRequired() { + return true; + } + + @Override + public void onEnd(ReadableSpan span) { + assertEquals("accepted", span.getName()); + assertTrue(span.hasEnded()); + } + + @Override + public boolean isEndRequired() { + return true; + } + }; + try (var provider = + SdkTracerProvider.builder() + .addSpanProcessor( + new BraintrustSpanProcessor( + config, + SpanProcessor.composite( + extendedDelegate, + SimpleSpanProcessor.create(exporter)))) + .build()) { + var tracer = provider.get("test"); + var accepted = tracer.spanBuilder("accepted").startSpan(); + accepted.setAttribute("gen_ai.model", "smoke-model"); + accepted.setAttribute( + BraintrustSpanProcessor.CONTEXT_JSON, + "{\"metadata\":{\"source\":\"late-attribute\"}}"); + accepted.end(); + + var rejected = tracer.spanBuilder("rejected").startSpan(); + rejected.setAttribute("http.method", "GET"); + rejected.end(); + assertNull( + ((ReadableSpan) rejected).getAttribute(BraintrustSpanProcessor.CONTEXT_JSON), + "Rejected spans must skip enrichment"); + assertNull( + ((ReadableSpan) rejected).getAttribute(AttributeKey.booleanKey("smoke.ending")), + "Rejected spans must skip delegated ending hooks"); + + var spans = exporter.getFinishedSpanItems(); + assertEquals(1, spans.size()); + assertEquals("accepted", spans.get(0).getName()); + assertEquals( + Boolean.TRUE, + spans.get(0).getAttributes().get(AttributeKey.booleanKey("smoke.ending"))); + var context = + BraintrustJsonMapper.fromJson( + spans.get(0).getAttributes().get(BraintrustSpanProcessor.CONTEXT_JSON), + Map.class); + assertEquals("late-attribute", ((Map) context.get("metadata")).get("source")); + assertEquals( + "braintrust.sdk.java", ((Map) context.get("span_origin")).get("name")); + } } private void doSimpleOtelTrace(Tracer tracer) {