From f1d46cfe83047e788b9e22612c0a5a2e38c96b62 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Thu, 10 Sep 2026 11:00:34 -0400 Subject: [PATCH 1/7] Emit gen_ai.* attributes on APM spans Write the scalar gen_ai.* subset of an LLM Observability span onto the APM span, so model, provider, application, conversation and token usage become indexed, searchable APM tags. Co-Authored-By: Claude Opus 5 --- .../trace/llmobs/domain/DDLLMObsSpan.java | 8 + .../domain/DDLLMObsSpanGenAiApmTagsTest.java | 128 ++++++++++++++ .../openai_java/OpenAiDecorator.java | 3 + .../groovy/ChatCompletionServiceTest.groovy | 10 ++ .../test/groovy/CompletionServiceTest.groovy | 7 + .../test/groovy/EmbeddingServiceTest.groovy | 6 + .../test/groovy/ResponseServiceTest.groovy | 9 + .../trace/api/llmobs/GenAiApmTags.java | 118 +++++++++++++ .../trace/api/llmobs/GenAiApmTagsTest.java | 164 ++++++++++++++++++ 9 files changed, 453 insertions(+) create mode 100644 dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java create mode 100644 internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java create mode 100644 internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 473d118cdc3..a096a7d532a 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -6,6 +6,7 @@ import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.DDTraceId; import datadog.trace.api.WellKnownTags; +import datadog.trace.api.llmobs.GenAiApmTags; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.llmobs.LLMObsSampler; @@ -691,6 +692,13 @@ public void finish() { if (finished) { return; } + // Ahead of span.finish() while the span is still mutable, and in its own try/catch so a + // failure here cannot cost the LLM Observability event. + try { + GenAiApmTags.apply(span); + } catch (Throwable t) { + LOGGER.debug("failed to set gen_ai APM tags", t); + } span.finish(); if (standaloneApmScope != null) { standaloneApmScope.close(); diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java new file mode 100644 index 00000000000..00680cb083c --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java @@ -0,0 +1,128 @@ +package datadog.trace.llmobs.domain; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.trace.agent.tooling.TracerInstaller; +import datadog.trace.api.WellKnownTags; +import datadog.trace.api.llmobs.GenAiApmTags; +import datadog.trace.api.llmobs.LLMObsTags; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.core.CoreTracer; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Covers the scalar {@code gen_ai.*} attributes emitted onto the APM span at finish, which make + * model, provider, application, conversation and token usage searchable in APM. + */ +class DDLLMObsSpanGenAiApmTagsTest { + private static final Field SPAN_FIELD; + + private static CoreTracer tracer; + + static { + try { + SPAN_FIELD = DDLLMObsSpan.class.getDeclaredField("span"); + SPAN_FIELD.setAccessible(true); + } catch (ReflectiveOperationException error) { + throw new ExceptionInInitializerError(error); + } + } + + @BeforeAll + static void installTracer() { + tracer = CoreTracer.builder().build(); + TracerInstaller.forceInstallGlobalTracer(tracer); + } + + @AfterAll + static void closeTracer() { + TracerInstaller.forceInstallGlobalTracer(null); + tracer.close(); + } + + @Test + void llmSpanEmitsEveryScalarAndTokenUsage() { + DDLLMObsSpan llm = llmObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "chat", "sess-1"); + llm.setTag(LLMObsTags.MODEL_NAME, "gpt-4"); + llm.setTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + Map metrics = new HashMap<>(); + metrics.put("input_tokens", 10); + metrics.put("output_tokens", 20); + metrics.put("total_tokens", 30); + metrics.put("cache_read_input_tokens", 4); + metrics.put("cache_write_input_tokens", 5); + metrics.put("reasoning_output_tokens", 6); + llm.setMetrics(metrics); + llm.finish(); + + AgentSpan span = spanOf(llm); + assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, span.getTag(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", span.getTag(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", span.getTag(GenAiApmTags.PROVIDER_NAME)); + assertEquals("test-ml-app", span.getTag(GenAiApmTags.APPLICATION_NAME)); + assertEquals("sess-1", span.getTag(GenAiApmTags.CONVERSATION_ID)); + assertEquals(10.0, span.getTag(GenAiApmTags.USAGE_INPUT_TOKENS)); + assertEquals(20.0, span.getTag(GenAiApmTags.USAGE_OUTPUT_TOKENS)); + assertEquals(30.0, span.getTag(GenAiApmTags.USAGE_TOTAL_TOKENS)); + assertEquals(4.0, span.getTag(GenAiApmTags.USAGE_CACHE_READ_INPUT_TOKENS)); + assertEquals(5.0, span.getTag(GenAiApmTags.USAGE_CACHE_WRITE_INPUT_TOKENS)); + assertEquals(6.0, span.getTag(GenAiApmTags.USAGE_REASONING_OUTPUT_TOKENS)); + } + + @Test + void modelBackedSpanWithoutModelFallsBackToCustom() { + DDLLMObsSpan embedding = llmObsSpan(Tags.LLMOBS_EMBEDDING_SPAN_KIND, "embed", null); + embedding.finish(); + + AgentSpan span = spanOf(embedding); + assertEquals("custom", span.getTag(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", span.getTag(GenAiApmTags.PROVIDER_NAME)); + assertNull(span.getTag(GenAiApmTags.CONVERSATION_ID)); + } + + @Test + void nonModelBackedSpanEmitsNoModelFieldsOrTokenUsage() { + DDLLMObsSpan workflow = llmObsSpan(Tags.LLMOBS_WORKFLOW_SPAN_KIND, "flow", null); + workflow.setMetric("input_tokens", 10); + workflow.finish(); + + AgentSpan span = spanOf(workflow); + assertEquals(Tags.LLMOBS_WORKFLOW_SPAN_KIND, span.getTag(GenAiApmTags.OPERATION_NAME)); + assertNull(span.getTag(GenAiApmTags.REQUEST_MODEL)); + assertNull(span.getTag(GenAiApmTags.PROVIDER_NAME)); + assertNull(span.getTag(GenAiApmTags.USAGE_INPUT_TOKENS)); + } + + @Test + void nonModelBackedSpanKeepsExplicitModelFields() { + DDLLMObsSpan agent = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "agent", null); + agent.setTag(LLMObsTags.MODEL_NAME, "gpt-4"); + agent.setTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + agent.finish(); + + AgentSpan span = spanOf(agent); + assertEquals("gpt-4", span.getTag(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", span.getTag(GenAiApmTags.PROVIDER_NAME)); + } + + private static DDLLMObsSpan llmObsSpan(String kind, String name, String sessionId) { + WellKnownTags tags = + new WellKnownTags("runtime-id", "hostname", "test", "service", "version", "java"); + return new DDLLMObsSpan(kind, name, "test-ml-app", sessionId, "service", tags); + } + + private static AgentSpan spanOf(DDLLMObsSpan llmObsSpan) { + try { + return (AgentSpan) SPAN_FIELD.get(llmObsSpan); + } catch (IllegalAccessException error) { + throw new AssertionError(error); + } + } +} diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 796490bdda5..b7b1bff9029 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -9,6 +9,7 @@ import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.WellKnownTags; +import datadog.trace.api.llmobs.GenAiApmTags; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.llmobs.LLMObsSampler; import datadog.trace.api.telemetry.LLMObsMetricCollector; @@ -179,6 +180,8 @@ protected void doBeforeFinish(@Nonnull Context context) { span.setTag(CommonTags.ERROR, span.isError() ? 1 : 0); span.setTag(CommonTags.ERROR_TYPE, span.getTag(DDTags.ERROR_TYPE)); + GenAiApmTags.apply(span); + Object spanKindTag = span.getTag(CommonTags.SPAN_KIND); if (spanKindTag != null) { String spanKind = spanKindTag.toString(); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 0a8553e7930..4a4d06e4784 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -401,6 +401,16 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_tag.sampling_decision" "1" "_ml_obs_tag.sample_rate" "1" "_ml_obs_tag.ml_app" String + "gen_ai.operation.name" "llm" + "gen_ai.request.model" String + "gen_ai.provider.name" "openai" + "gen_ai.application.name" String + if (!isStreaming) { + "gen_ai.usage.input_tokens" Double + "gen_ai.usage.output_tokens" Double + "gen_ai.usage.total_tokens" Double + "gen_ai.usage.cache_read_input_tokens" Double + } "_ml_obs_tag.service" String "$CommonTags.DDTRACE_VERSION" String if (expectToolDefinitions) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index 50529ee02a6..8723549ab00 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -183,6 +183,13 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.sampling_decision" "1" "_ml_obs_tag.sample_rate" "1" "_ml_obs_tag.ml_app" String + "gen_ai.operation.name" "llm" + "gen_ai.request.model" String + "gen_ai.provider.name" "openai" + "gen_ai.application.name" String + "gen_ai.usage.input_tokens" Double + "gen_ai.usage.output_tokens" Double + "gen_ai.usage.total_tokens" Double "_ml_obs_tag.service" String "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 41e341284f3..0d718ee5e3d 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -76,6 +76,12 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.sampling_decision" "1" "_ml_obs_tag.sample_rate" "1" "_ml_obs_tag.ml_app" String + "gen_ai.operation.name" "embedding" + "gen_ai.request.model" "text-embedding-ada-002-v2" + "gen_ai.provider.name" "openai" + "gen_ai.application.name" String + "gen_ai.usage.input_tokens" Double + "gen_ai.usage.total_tokens" Double "_ml_obs_tag.service" String "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index b22389a4d41..c455bffdd96 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -478,6 +478,15 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.sampling_decision" "1" "_ml_obs_tag.sample_rate" "1" "_ml_obs_tag.ml_app" String + "gen_ai.operation.name" "llm" + "gen_ai.request.model" String + "gen_ai.provider.name" "openai" + "gen_ai.application.name" String + "gen_ai.usage.input_tokens" Double + "gen_ai.usage.output_tokens" Double + "gen_ai.usage.total_tokens" Double + "gen_ai.usage.reasoning_output_tokens" Double + "gen_ai.usage.cache_read_input_tokens" Double "$CommonTags.INTEGRATION" "openai" "_ml_obs_tag.service" String "$CommonTags.DDTRACE_VERSION" String diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java new file mode 100644 index 00000000000..076fdd09f42 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java @@ -0,0 +1,118 @@ +package datadog.trace.api.llmobs; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.Locale; + +/** + * Emits the scalar {@code gen_ai.*} attributes of an LLM Observability span onto the APM span, so + * model, provider, application, conversation and token usage are indexed and searchable in APM. + * + *

Message bodies (input, output, tool definitions, retrieval documents) are deliberately left + * off the APM span and keep coming from the LLM Observability track. + * + *

Values are read back from the {@code _ml_obs_tag.} / {@code _ml_obs_metric.} tags rather than + * passed in, because they are stamped by several decorators over a span's lifetime and are only all + * present at finish time. + */ +public final class GenAiApmTags { + public static final String OPERATION_NAME = "gen_ai.operation.name"; + public static final String REQUEST_MODEL = "gen_ai.request.model"; + public static final String PROVIDER_NAME = "gen_ai.provider.name"; + public static final String APPLICATION_NAME = "gen_ai.application.name"; + public static final String CONVERSATION_ID = "gen_ai.conversation.id"; + + public static final String USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; + public static final String USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; + public static final String USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens"; + public static final String USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read_input_tokens"; + public static final String USAGE_CACHE_WRITE_INPUT_TOKENS = + "gen_ai.usage.cache_write_input_tokens"; + public static final String USAGE_REASONING_OUTPUT_TOKENS = "gen_ai.usage.reasoning_output_tokens"; + + private static final String LLMOBS_TAG_PREFIX = "_ml_obs_tag."; + private static final String LLMOBS_METRIC_PREFIX = "_ml_obs_metric."; + + private static final String SPAN_KIND_TAG = LLMOBS_TAG_PREFIX + Tags.SPAN_KIND; + private static final String MODEL_NAME_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_NAME; + private static final String MODEL_PROVIDER_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_PROVIDER; + private static final String ML_APP_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.ML_APP; + private static final String SESSION_ID_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID; + + /** Matches the fallback the LLM Observability event uses for model-backed spans. */ + private static final String DEFAULT_MODEL = "custom"; + + /** LLM Observability metric name paired with the {@code gen_ai.usage.*} key it maps to. */ + private static final String[][] TOKEN_METRICS = { + {LLMOBS_METRIC_PREFIX + "input_tokens", USAGE_INPUT_TOKENS}, + {LLMOBS_METRIC_PREFIX + "output_tokens", USAGE_OUTPUT_TOKENS}, + {LLMOBS_METRIC_PREFIX + "total_tokens", USAGE_TOTAL_TOKENS}, + {LLMOBS_METRIC_PREFIX + "cache_read_input_tokens", USAGE_CACHE_READ_INPUT_TOKENS}, + {LLMOBS_METRIC_PREFIX + "cache_write_input_tokens", USAGE_CACHE_WRITE_INPUT_TOKENS}, + {LLMOBS_METRIC_PREFIX + "reasoning_output_tokens", USAGE_REASONING_OUTPUT_TOKENS}, + }; + + /** + * Writes the {@code gen_ai.*} attributes onto the given span. Must be called before the span is + * finished, and is a no-op for a span that carries no LLM Observability span kind. + */ + public static void apply(AgentSpan span) { + if (span == null) { + return; + } + String spanKind = stringTag(span, SPAN_KIND_TAG); + if (spanKind == null) { + return; + } + span.setTag(OPERATION_NAME, spanKind); + + String modelName = stringTag(span, MODEL_NAME_TAG); + String modelProvider = stringTag(span, MODEL_PROVIDER_TAG); + boolean modelBacked = + Tags.LLMOBS_LLM_SPAN_KIND.equals(spanKind) + || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(spanKind); + if (modelBacked) { + span.setTag(REQUEST_MODEL, modelName == null ? DEFAULT_MODEL : modelName); + span.setTag( + PROVIDER_NAME, + (modelProvider == null ? DEFAULT_MODEL : modelProvider).toLowerCase(Locale.ROOT)); + } else { + if (modelName != null) { + span.setTag(REQUEST_MODEL, modelName); + } + if (modelProvider != null) { + span.setTag(PROVIDER_NAME, modelProvider.toLowerCase(Locale.ROOT)); + } + } + + String mlApp = stringTag(span, ML_APP_TAG); + if (mlApp != null) { + span.setTag(APPLICATION_NAME, mlApp); + } + String sessionId = stringTag(span, SESSION_ID_TAG); + if (sessionId != null) { + span.setTag(CONVERSATION_ID, sessionId); + } + + // Other span kinds carry unrelated metrics that would be misleading under a gen_ai.usage.* key. + if (modelBacked) { + for (String[] metric : TOKEN_METRICS) { + Object value = span.getTag(metric[0]); + if (value instanceof Number) { + span.setMetric(metric[1], ((Number) value).doubleValue()); + } + } + } + } + + private static String stringTag(AgentSpan span, String key) { + Object value = span.getTag(key); + if (value == null) { + return null; + } + String string = String.valueOf(value); + return string.isEmpty() ? null : string; + } + + private GenAiApmTags() {} +} diff --git a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java new file mode 100644 index 00000000000..c257168ee1e --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java @@ -0,0 +1,164 @@ +package datadog.trace.api.llmobs; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class GenAiApmTagsTest { + private static final String LLMOBS_TAG_PREFIX = "_ml_obs_tag."; + private static final String LLMOBS_METRIC_PREFIX = "_ml_obs_metric."; + + private final Map tags = new HashMap<>(); + private AgentSpan span; + + @BeforeEach + void setUp() { + span = mock(AgentSpan.class); + when(span.getTag(anyString())).thenAnswer(call -> tags.get(call.getArgument(0))); + when(span.setTag(anyString(), anyString())) + .thenAnswer( + call -> { + tags.put(call.getArgument(0), call.getArgument(1)); + return span; + }); + when(span.setMetric(any(CharSequence.class), anyDouble())) + .thenAnswer( + call -> { + tags.put(call.getArgument(0).toString(), call.getArgument(1)); + return span; + }); + } + + @Test + void llmSpanEmitsEveryScalarAndTokenUsage() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + llmObsTag(LLMObsTags.MODEL_NAME, "gpt-4"); + llmObsTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + llmObsTag(LLMObsTags.ML_APP, "my-app"); + llmObsTag(LLMObsTags.SESSION_ID, "sess-1"); + llmObsMetric("input_tokens", 10); + llmObsMetric("output_tokens", 20); + llmObsMetric("total_tokens", 30); + llmObsMetric("cache_read_input_tokens", 4); + llmObsMetric("cache_write_input_tokens", 5); + llmObsMetric("reasoning_output_tokens", 6); + + GenAiApmTags.apply(span); + + assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", tags.get(GenAiApmTags.PROVIDER_NAME)); + assertEquals("my-app", tags.get(GenAiApmTags.APPLICATION_NAME)); + assertEquals("sess-1", tags.get(GenAiApmTags.CONVERSATION_ID)); + assertEquals(10.0, tags.get(GenAiApmTags.USAGE_INPUT_TOKENS)); + assertEquals(20.0, tags.get(GenAiApmTags.USAGE_OUTPUT_TOKENS)); + assertEquals(30.0, tags.get(GenAiApmTags.USAGE_TOTAL_TOKENS)); + assertEquals(4.0, tags.get(GenAiApmTags.USAGE_CACHE_READ_INPUT_TOKENS)); + assertEquals(5.0, tags.get(GenAiApmTags.USAGE_CACHE_WRITE_INPUT_TOKENS)); + assertEquals(6.0, tags.get(GenAiApmTags.USAGE_REASONING_OUTPUT_TOKENS)); + } + + @Test + void modelBackedSpanWithoutModelFallsBackToCustom() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); + + GenAiApmTags.apply(span); + + assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); + } + + @Test + void emptyModelValuesAreTreatedAsAbsent() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + llmObsTag(LLMObsTags.MODEL_NAME, ""); + llmObsTag(LLMObsTags.MODEL_PROVIDER, ""); + llmObsTag(LLMObsTags.ML_APP, ""); + llmObsTag(LLMObsTags.SESSION_ID, ""); + + GenAiApmTags.apply(span); + + assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); + assertNull(tags.get(GenAiApmTags.APPLICATION_NAME)); + assertNull(tags.get(GenAiApmTags.CONVERSATION_ID)); + } + + @Test + void nonModelBackedSpanKeepsExplicitModelFieldsWithoutFallbacks() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_AGENT_SPAN_KIND); + llmObsTag(LLMObsTags.MODEL_NAME, "gpt-4"); + llmObsTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + + GenAiApmTags.apply(span); + + assertEquals(Tags.LLMOBS_AGENT_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", tags.get(GenAiApmTags.PROVIDER_NAME)); + } + + @Test + void nonModelBackedSpanWithoutModelEmitsNoModelFields() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_WORKFLOW_SPAN_KIND); + + GenAiApmTags.apply(span); + + assertFalse(tags.containsKey(GenAiApmTags.REQUEST_MODEL)); + assertFalse(tags.containsKey(GenAiApmTags.PROVIDER_NAME)); + } + + @Test + void nonModelBackedSpanDropsTokenUsage() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_TASK_SPAN_KIND); + llmObsMetric("input_tokens", 10); + + GenAiApmTags.apply(span); + + assertFalse(tags.containsKey(GenAiApmTags.USAGE_INPUT_TOKENS)); + } + + @Test + void nonNumericTokenMetricIsSkipped() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + tags.put(LLMOBS_METRIC_PREFIX + "input_tokens", "not-a-number"); + + GenAiApmTags.apply(span); + + assertFalse(tags.containsKey(GenAiApmTags.USAGE_INPUT_TOKENS)); + } + + @Test + void spanWithoutLlmObsKindEmitsNothing() { + llmObsTag(LLMObsTags.MODEL_NAME, "gpt-4"); + + GenAiApmTags.apply(span); + + assertTrue(tags.keySet().stream().noneMatch(key -> key.startsWith("gen_ai."))); + } + + @Test + void nullSpanIsANoOp() { + GenAiApmTags.apply(null); + } + + private void llmObsTag(String key, String value) { + tags.put(LLMOBS_TAG_PREFIX + key, value); + } + + private void llmObsMetric(String key, Number value) { + tags.put(LLMOBS_METRIC_PREFIX + key, value); + } +} From abae3521358f7c01163d2824a16d6658f74d9d27 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Fri, 11 Sep 2026 10:26:20 -0400 Subject: [PATCH 2/7] Emit available gen_ai.* attributes with LLM Observability off The openai-java instrumentation keeps tracing when LLM Observability is disabled, and the operation, model, provider and ml_app are all known on that path. Token usage and conversation id are not computed there, so they stay unreported. Co-Authored-By: Claude Opus 5 --- .../openai_java/OpenAiDecorator.java | 40 +++++++++ .../groovy/LlmObsDisabledForkedTest.groovy | 49 +++++++++++ .../trace/api/llmobs/GenAiApmTags.java | 84 ++++++++++++++----- .../trace/api/llmobs/GenAiApmTagsTest.java | 32 +++++++ 4 files changed, 183 insertions(+), 22 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index b7b1bff9029..8da3a318621 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -17,6 +17,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.ClientDecorator; import java.util.List; @@ -38,6 +39,8 @@ public class OpenAiDecorator extends ClientDecorator { private static final String TOKENS_LIMIT_METRIC = METRIC_PREFIX + "tokens.limit"; private static final String TOKENS_REMAINING_METRIC = METRIC_PREFIX + "tokens.remaining"; + private static final String EMBEDDINGS_ENDPOINT = "/v1/embeddings"; + private static final String HEADER_PREFIX = "x-ratelimit-"; private static final String LIMIT_REQUESTS_HEADER = HEADER_PREFIX + "limit-requests"; private static final String REMAINING_REQUESTS_HEADER = HEADER_PREFIX + "remaining-requests"; @@ -189,10 +192,47 @@ protected void doBeforeFinish(@Nonnull Context context) { LLMObsMetricCollector.get() .recordSpanFinished(INTEGRATION, spanKind, isRootSpan, true, span.isError(), false); } + } else if (span != null) { + // Tracing still runs with LLM Observability off, and model, provider and ml_app are known + // here. Token usage and session id are not computed on this path, so they stay unreported. + GenAiApmTags.applyWithoutLlmObs( + span, + operationName(span), + requestedModel(span), + stringTag(span, CommonTags.MODEL_PROVIDER), + Config.get().getLlmObsMlApp()); } super.doBeforeFinish(context); } + /** + * The LLM Observability span kind the endpoint maps to, or null if the span traced no request. + */ + private static String operationName(AgentSpan span) { + String endpoint = stringTag(span, CommonTags.OPENAI_REQUEST_ENDPOINT); + if (endpoint == null) { + return null; + } + return EMBEDDINGS_ENDPOINT.equals(endpoint) + ? Tags.LLMOBS_EMBEDDING_SPAN_KIND + : Tags.LLMOBS_LLM_SPAN_KIND; + } + + /** Prefers the model the response reports, which resolves aliases the request used. */ + private static String requestedModel(AgentSpan span) { + String model = stringTag(span, CommonTags.OPENAI_RESPONSE_MODEL); + return model != null ? model : stringTag(span, CommonTags.OPENAI_REQUEST_MODEL); + } + + private static String stringTag(AgentSpan span, String key) { + Object value = span.getTag(key); + if (value == null) { + return null; + } + String string = value.toString(); + return string.isEmpty() ? null : string; + } + public void withHttpResponse(AgentSpan span, Headers headers) { if (!llmObsEnabled) { return; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy new file mode 100644 index 00000000000..1cc9d30fa29 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy @@ -0,0 +1,49 @@ +import datadog.trace.api.config.LlmObsConfig + +/** + * With LLM Observability off the instrumentation still traces, so the gen_ai.* attributes it can + * resolve without the LLMObs pipeline are emitted. Forked because OpenAiDecorator reads the + * llmobs.enabled flag once, into a final field, when the class is loaded. + */ +class LlmObsDisabledForkedTest extends OpenAiTest { + + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig(LlmObsConfig.LLMOBS_ENABLED, "false") + } + + def "chat completion emits the gen_ai attributes available without LLMObs"() { + when: + openAiClient.chat().completions().create(chatCompletionCreateParams(false)) + TEST_WRITER.waitForTraces(1) + def span = TEST_WRITER.flatten().find { it.operationName.toString() == "openai.request" } + + then: + span.getTag("gen_ai.operation.name") == "llm" + span.getTag("gen_ai.request.model") == span.getTag("openai.response.model") + span.getTag("gen_ai.provider.name") == "openai" + span.getTag("gen_ai.application.name") != null + + and: "neither is computed with LLMObs off" + span.getTag("gen_ai.conversation.id") == null + span.getTag("gen_ai.usage.input_tokens") == null + span.getTag("gen_ai.usage.output_tokens") == null + span.getTag("gen_ai.usage.total_tokens") == null + + and: "the LLMObs track stays off the span" + span.getTag("_ml_obs_tag.span.kind") == null + } + + def "embedding maps to the embedding operation"() { + when: + openAiClient.embeddings().create(embeddingCreateParams(false)) + TEST_WRITER.waitForTraces(1) + def span = TEST_WRITER.flatten().find { it.operationName.toString() == "openai.request" } + + then: + span.getTag("gen_ai.operation.name") == "embedding" + span.getTag("gen_ai.request.model") == span.getTag("openai.response.model") + span.getTag("gen_ai.provider.name") == "openai" + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java index 076fdd09f42..46824594234 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java @@ -11,9 +11,10 @@ *

Message bodies (input, output, tool definitions, retrieval documents) are deliberately left * off the APM span and keep coming from the LLM Observability track. * - *

Values are read back from the {@code _ml_obs_tag.} / {@code _ml_obs_metric.} tags rather than - * passed in, because they are stamped by several decorators over a span's lifetime and are only all - * present at finish time. + *

{@link #apply} reads the values back from the {@code _ml_obs_tag.} / {@code _ml_obs_metric.} + * tags rather than taking them as arguments, because they are stamped by several decorators over a + * span's lifetime and are only all present at finish time. {@link #applyWithoutLlmObs} covers the + * instrumentation that still traces with LLM Observability off, where those tags do not exist. */ public final class GenAiApmTags { public static final String OPERATION_NAME = "gen_ai.operation.name"; @@ -64,14 +65,56 @@ public static void apply(AgentSpan span) { if (spanKind == null) { return; } - span.setTag(OPERATION_NAME, spanKind); - - String modelName = stringTag(span, MODEL_NAME_TAG); - String modelProvider = stringTag(span, MODEL_PROVIDER_TAG); - boolean modelBacked = - Tags.LLMOBS_LLM_SPAN_KIND.equals(spanKind) - || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(spanKind); - if (modelBacked) { + setScalars( + span, + spanKind, + stringTag(span, MODEL_NAME_TAG), + stringTag(span, MODEL_PROVIDER_TAG), + stringTag(span, ML_APP_TAG), + stringTag(span, SESSION_ID_TAG)); + + // Other span kinds carry unrelated metrics that would be misleading under a gen_ai.usage.* key. + if (isModelBacked(spanKind)) { + for (String[] metric : TOKEN_METRICS) { + Object value = span.getTag(metric[0]); + if (value instanceof Number) { + span.setMetric(metric[1], ((Number) value).doubleValue()); + } + } + } + } + + /** + * Writes the subset of {@code gen_ai.*} attributes that instrumentation can supply with LLM + * Observability disabled. Token usage and conversation id are left out: neither is computed on + * that path, so there is nothing to report. + * + * @param operationName the LLM Observability span kind this operation maps to + */ + public static void applyWithoutLlmObs( + AgentSpan span, String operationName, String modelName, String modelProvider, String mlApp) { + if (span == null || operationName == null) { + return; + } + setScalars( + span, + operationName, + emptyToNull(modelName), + emptyToNull(modelProvider), + emptyToNull(mlApp), + null); + } + + private static void setScalars( + AgentSpan span, + String operationName, + String modelName, + String modelProvider, + String mlApp, + String sessionId) { + span.setTag(OPERATION_NAME, operationName); + + if (isModelBacked(operationName)) { span.setTag(REQUEST_MODEL, modelName == null ? DEFAULT_MODEL : modelName); span.setTag( PROVIDER_NAME, @@ -85,24 +128,21 @@ public static void apply(AgentSpan span) { } } - String mlApp = stringTag(span, ML_APP_TAG); if (mlApp != null) { span.setTag(APPLICATION_NAME, mlApp); } - String sessionId = stringTag(span, SESSION_ID_TAG); if (sessionId != null) { span.setTag(CONVERSATION_ID, sessionId); } + } - // Other span kinds carry unrelated metrics that would be misleading under a gen_ai.usage.* key. - if (modelBacked) { - for (String[] metric : TOKEN_METRICS) { - Object value = span.getTag(metric[0]); - if (value instanceof Number) { - span.setMetric(metric[1], ((Number) value).doubleValue()); - } - } - } + private static boolean isModelBacked(String spanKind) { + return Tags.LLMOBS_LLM_SPAN_KIND.equals(spanKind) + || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(spanKind); + } + + private static String emptyToNull(String value) { + return value == null || value.isEmpty() ? null : value; } private static String stringTag(AgentSpan span, String key) { diff --git a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java index c257168ee1e..81fa87c0c6c 100644 --- a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java +++ b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java @@ -154,6 +154,38 @@ void nullSpanIsANoOp() { GenAiApmTags.apply(null); } + @Test + void withoutLlmObsEmitsScalarsButNoUsageOrConversation() { + GenAiApmTags.applyWithoutLlmObs(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "OpenAI", "my-app"); + + assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", tags.get(GenAiApmTags.PROVIDER_NAME)); + assertEquals("my-app", tags.get(GenAiApmTags.APPLICATION_NAME)); + assertFalse(tags.containsKey(GenAiApmTags.CONVERSATION_ID)); + assertFalse(tags.containsKey(GenAiApmTags.USAGE_INPUT_TOKENS)); + } + + @Test + void withoutLlmObsFallsBackToCustomForModelBackedKinds() { + GenAiApmTags.applyWithoutLlmObs(span, Tags.LLMOBS_EMBEDDING_SPAN_KIND, null, "", "app"); + + assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); + } + + @Test + void withoutLlmObsIgnoresSpanWithNoOperationName() { + GenAiApmTags.applyWithoutLlmObs(span, null, "gpt-4", "openai", "my-app"); + + assertTrue(tags.keySet().stream().noneMatch(key -> key.startsWith("gen_ai."))); + } + + @Test + void withoutLlmObsNullSpanIsANoOp() { + GenAiApmTags.applyWithoutLlmObs(null, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "openai", "app"); + } + private void llmObsTag(String key, String value) { tags.put(LLMOBS_TAG_PREFIX + key, value); } From 1527dca26575f4748d1f76ef7620d0a50d27df18 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Fri, 11 Sep 2026 11:01:46 -0400 Subject: [PATCH 3/7] Rewrite the LLMObs-disabled openai test in Java The repository does not accept new .groovy files. The module already has a JUnit forked-test base for LLMObs configurations, so the coverage moves there. Co-Authored-By: Claude Opus 5 --- .../groovy/LlmObsDisabledForkedTest.groovy | 49 --------------- .../LlmObsContextPropagationForkedTest.java | 63 +++++++++++++++++++ 2 files changed, 63 insertions(+), 49 deletions(-) delete mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy deleted file mode 100644 index 1cc9d30fa29..00000000000 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy +++ /dev/null @@ -1,49 +0,0 @@ -import datadog.trace.api.config.LlmObsConfig - -/** - * With LLM Observability off the instrumentation still traces, so the gen_ai.* attributes it can - * resolve without the LLMObs pipeline are emitted. Forked because OpenAiDecorator reads the - * llmobs.enabled flag once, into a final field, when the class is loaded. - */ -class LlmObsDisabledForkedTest extends OpenAiTest { - - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig(LlmObsConfig.LLMOBS_ENABLED, "false") - } - - def "chat completion emits the gen_ai attributes available without LLMObs"() { - when: - openAiClient.chat().completions().create(chatCompletionCreateParams(false)) - TEST_WRITER.waitForTraces(1) - def span = TEST_WRITER.flatten().find { it.operationName.toString() == "openai.request" } - - then: - span.getTag("gen_ai.operation.name") == "llm" - span.getTag("gen_ai.request.model") == span.getTag("openai.response.model") - span.getTag("gen_ai.provider.name") == "openai" - span.getTag("gen_ai.application.name") != null - - and: "neither is computed with LLMObs off" - span.getTag("gen_ai.conversation.id") == null - span.getTag("gen_ai.usage.input_tokens") == null - span.getTag("gen_ai.usage.output_tokens") == null - span.getTag("gen_ai.usage.total_tokens") == null - - and: "the LLMObs track stays off the span" - span.getTag("_ml_obs_tag.span.kind") == null - } - - def "embedding maps to the embedding operation"() { - when: - openAiClient.embeddings().create(embeddingCreateParams(false)) - TEST_WRITER.waitForTraces(1) - def span = TEST_WRITER.flatten().find { it.operationName.toString() == "openai.request" } - - then: - span.getTag("gen_ai.operation.name") == "embedding" - span.getTag("gen_ai.request.model") == span.getTag("openai.response.model") - span.getTag("gen_ai.provider.name") == "openai" - } -} diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java index 66d4f6aa285..840426bd7c8 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java @@ -9,6 +9,8 @@ import com.openai.credential.BearerTokenCredential; import com.openai.models.ChatModel; import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.embeddings.EmbeddingCreateParams; +import com.openai.models.embeddings.EmbeddingModel; import com.sun.net.httpserver.HttpServer; import datadog.context.ContextScope; import datadog.trace.agent.test.AbstractInstrumentationTest; @@ -76,6 +78,13 @@ protected static ChatCompletionCreateParams buildMinimalChatParams() { .build(); } + protected static EmbeddingCreateParams buildMinimalEmbeddingParams() { + return EmbeddingCreateParams.builder() + .model(EmbeddingModel.TEXT_EMBEDDING_ADA_002) + .input("") + .build(); + } + protected static DDSpan findSpanByOperationName(List> traces, String operationName) { return traces.stream() .flatMap(List::stream) @@ -318,3 +327,57 @@ void parentlessOpenAiRequestSpanIsDroppedAtZeroSampleRate() throws Exception { assertEquals("0", openAiSpan.getTag("_ml_obs_tag.sample_rate")); } } + +/** + * Verifies the gen_ai.* attributes an openai.request span carries with LLM Observability disabled. + * The instrumentation still traces on that path, so operation, model, provider and application are + * resolvable, while token usage and conversation id are never computed and must stay absent. + */ +@WithConfig(key = "llmobs.enabled", value = "false") +class LlmObsDisabledForkedTest extends AbstractLlmObsOpenAiForkedTest { + + @Test + void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() throws Exception { + try { + openAiClient.chat().completions().create(buildMinimalChatParams()); + } catch (Exception ignored) { + // Mock server returns no body — the SDK may throw on parse. The span is already created. + } + + writer.waitForTraces(1); + DDSpan openAiSpan = findSpanByOperationName(writer, "openai.request"); + assertNotNull(openAiSpan, "openai.request span should have been created"); + + assertEquals("llm", openAiSpan.getTag("gen_ai.operation.name")); + // The mock returns no body, so there is no response model and the request model stands in. + assertEquals( + openAiSpan.getTag("openai.request.model"), openAiSpan.getTag("gen_ai.request.model")); + assertEquals("openai", openAiSpan.getTag("gen_ai.provider.name")); + assertNotNull(openAiSpan.getTag("gen_ai.application.name")); + + assertNull(openAiSpan.getTag("gen_ai.conversation.id")); + assertNull(openAiSpan.getTag("gen_ai.usage.input_tokens")); + assertNull(openAiSpan.getTag("gen_ai.usage.output_tokens")); + assertNull(openAiSpan.getTag("gen_ai.usage.total_tokens")); + + assertNull(openAiSpan.getTag("_ml_obs_tag.span.kind")); + } + + @Test + void embeddingMapsToTheEmbeddingOperation() throws Exception { + try { + openAiClient.embeddings().create(buildMinimalEmbeddingParams()); + } catch (Exception ignored) { + // Mock server returns no body — the SDK may throw on parse. The span is already created. + } + + writer.waitForTraces(1); + DDSpan openAiSpan = findSpanByOperationName(writer, "openai.request"); + assertNotNull(openAiSpan, "openai.request span should have been created"); + + assertEquals("embedding", openAiSpan.getTag("gen_ai.operation.name")); + assertEquals( + openAiSpan.getTag("openai.request.model"), openAiSpan.getTag("gen_ai.request.model")); + assertEquals("openai", openAiSpan.getTag("gen_ai.provider.name")); + } +} From faac2ec6f81098d36e1d5f50964c7bc59b1adb23 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Fri, 11 Sep 2026 13:35:41 -0400 Subject: [PATCH 4/7] Trim comments and docstrings Co-Authored-By: Claude Opus 5 --- .../trace/llmobs/domain/DDLLMObsSpan.java | 3 +-- .../domain/DDLLMObsSpanGenAiApmTagsTest.java | 5 +--- .../openai_java/OpenAiDecorator.java | 8 ++---- .../LlmObsContextPropagationForkedTest.java | 11 ++++---- .../trace/api/llmobs/GenAiApmTags.java | 26 ++++++------------- 5 files changed, 17 insertions(+), 36 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index a096a7d532a..3aaac795186 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -692,8 +692,7 @@ public void finish() { if (finished) { return; } - // Ahead of span.finish() while the span is still mutable, and in its own try/catch so a - // failure here cannot cost the LLM Observability event. + // While the span is still mutable, and isolated so a failure cannot cost the event. try { GenAiApmTags.apply(span); } catch (Throwable t) { diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java index 00680cb083c..7b4720699ae 100644 --- a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java @@ -17,10 +17,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -/** - * Covers the scalar {@code gen_ai.*} attributes emitted onto the APM span at finish, which make - * model, provider, application, conversation and token usage searchable in APM. - */ +/** Covers the {@code gen_ai.*} attributes a manual LLM Observability span emits at finish. */ class DDLLMObsSpanGenAiApmTagsTest { private static final Field SPAN_FIELD; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 8da3a318621..9ae3f33466f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -193,8 +193,7 @@ protected void doBeforeFinish(@Nonnull Context context) { .recordSpanFinished(INTEGRATION, spanKind, isRootSpan, true, span.isError(), false); } } else if (span != null) { - // Tracing still runs with LLM Observability off, and model, provider and ml_app are known - // here. Token usage and session id are not computed on this path, so they stay unreported. + // Tracing still runs with LLM Observability off, where these four remain resolvable. GenAiApmTags.applyWithoutLlmObs( span, operationName(span), @@ -205,9 +204,6 @@ protected void doBeforeFinish(@Nonnull Context context) { super.doBeforeFinish(context); } - /** - * The LLM Observability span kind the endpoint maps to, or null if the span traced no request. - */ private static String operationName(AgentSpan span) { String endpoint = stringTag(span, CommonTags.OPENAI_REQUEST_ENDPOINT); if (endpoint == null) { @@ -218,7 +214,7 @@ private static String operationName(AgentSpan span) { : Tags.LLMOBS_LLM_SPAN_KIND; } - /** Prefers the model the response reports, which resolves aliases the request used. */ + /** The response model resolves aliases the request used, so it wins. */ private static String requestedModel(AgentSpan span) { String model = stringTag(span, CommonTags.OPENAI_RESPONSE_MODEL); return model != null ? model : stringTag(span, CommonTags.OPENAI_REQUEST_MODEL); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java index 840426bd7c8..16830ca0d2a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java @@ -329,9 +329,8 @@ void parentlessOpenAiRequestSpanIsDroppedAtZeroSampleRate() throws Exception { } /** - * Verifies the gen_ai.* attributes an openai.request span carries with LLM Observability disabled. - * The instrumentation still traces on that path, so operation, model, provider and application are - * resolvable, while token usage and conversation id are never computed and must stay absent. + * Verifies the gen_ai.* attributes an openai.request span carries with LLM Observability disabled: + * operation, model, provider and application, but never token usage or conversation id. */ @WithConfig(key = "llmobs.enabled", value = "false") class LlmObsDisabledForkedTest extends AbstractLlmObsOpenAiForkedTest { @@ -341,7 +340,7 @@ void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() throws Except try { openAiClient.chat().completions().create(buildMinimalChatParams()); } catch (Exception ignored) { - // Mock server returns no body — the SDK may throw on parse. The span is already created. + // The mock server returns no body, so the SDK may throw while parsing the response. } writer.waitForTraces(1); @@ -349,7 +348,7 @@ void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() throws Except assertNotNull(openAiSpan, "openai.request span should have been created"); assertEquals("llm", openAiSpan.getTag("gen_ai.operation.name")); - // The mock returns no body, so there is no response model and the request model stands in. + // The mock returns no body, so the request model stands in for the absent response model. assertEquals( openAiSpan.getTag("openai.request.model"), openAiSpan.getTag("gen_ai.request.model")); assertEquals("openai", openAiSpan.getTag("gen_ai.provider.name")); @@ -368,7 +367,7 @@ void embeddingMapsToTheEmbeddingOperation() throws Exception { try { openAiClient.embeddings().create(buildMinimalEmbeddingParams()); } catch (Exception ignored) { - // Mock server returns no body — the SDK may throw on parse. The span is already created. + // The mock server returns no body, so the SDK may throw while parsing the response. } writer.waitForTraces(1); diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java index 46824594234..9a4aabe331e 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java @@ -6,15 +6,8 @@ /** * Emits the scalar {@code gen_ai.*} attributes of an LLM Observability span onto the APM span, so - * model, provider, application, conversation and token usage are indexed and searchable in APM. - * - *

Message bodies (input, output, tool definitions, retrieval documents) are deliberately left - * off the APM span and keep coming from the LLM Observability track. - * - *

{@link #apply} reads the values back from the {@code _ml_obs_tag.} / {@code _ml_obs_metric.} - * tags rather than taking them as arguments, because they are stamped by several decorators over a - * span's lifetime and are only all present at finish time. {@link #applyWithoutLlmObs} covers the - * instrumentation that still traces with LLM Observability off, where those tags do not exist. + * model, provider, application, conversation and token usage are searchable in APM. Message bodies + * stay off the APM span and keep coming from the LLM Observability track. */ public final class GenAiApmTags { public static final String OPERATION_NAME = "gen_ai.operation.name"; @@ -40,7 +33,7 @@ public final class GenAiApmTags { private static final String ML_APP_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.ML_APP; private static final String SESSION_ID_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID; - /** Matches the fallback the LLM Observability event uses for model-backed spans. */ + /** Matches the fallback the LLM Observability event uses. */ private static final String DEFAULT_MODEL = "custom"; /** LLM Observability metric name paired with the {@code gen_ai.usage.*} key it maps to. */ @@ -54,8 +47,8 @@ public final class GenAiApmTags { }; /** - * Writes the {@code gen_ai.*} attributes onto the given span. Must be called before the span is - * finished, and is a no-op for a span that carries no LLM Observability span kind. + * Writes the attributes onto a span that is not yet finished, reading them back from its {@code + * _ml_obs_tag.} / {@code _ml_obs_metric.} tags. No-op for a span with no LLM Observability kind. */ public static void apply(AgentSpan span) { if (span == null) { @@ -73,7 +66,7 @@ public static void apply(AgentSpan span) { stringTag(span, ML_APP_TAG), stringTag(span, SESSION_ID_TAG)); - // Other span kinds carry unrelated metrics that would be misleading under a gen_ai.usage.* key. + // Other kinds carry unrelated metrics that a gen_ai.usage.* key would misrepresent. if (isModelBacked(spanKind)) { for (String[] metric : TOKEN_METRICS) { Object value = span.getTag(metric[0]); @@ -85,11 +78,8 @@ public static void apply(AgentSpan span) { } /** - * Writes the subset of {@code gen_ai.*} attributes that instrumentation can supply with LLM - * Observability disabled. Token usage and conversation id are left out: neither is computed on - * that path, so there is nothing to report. - * - * @param operationName the LLM Observability span kind this operation maps to + * Writes the subset available with LLM Observability disabled. Token usage and conversation id + * are never computed on that path, so they are left out. */ public static void applyWithoutLlmObs( AgentSpan span, String operationName, String modelName, String modelProvider, String mlApp) { From 324b5bbd7fd30def57807b1df8a16dfe14834ba0 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Fri, 11 Sep 2026 16:32:02 -0400 Subject: [PATCH 5/7] Address review: single apply method and shared tag reader Co-Authored-By: Claude Opus 5 --- .../openai_java/OpenAiDecorator.java | 20 +--- .../groovy/ChatCompletionServiceTest.groovy | 1 + .../trace/api/llmobs/GenAiApmTags.java | 111 +++++++----------- .../trace/api/llmobs/GenAiApmTagsTest.java | 33 ++++-- 4 files changed, 67 insertions(+), 98 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 9ae3f33466f..a31cf5c95b3 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.api.llmobs.GenAiApmTags.stringTag; import static datadog.trace.bootstrap.instrumentation.api.AgentSpan.fromContext; import com.openai.core.ClientOptions; @@ -193,13 +194,9 @@ protected void doBeforeFinish(@Nonnull Context context) { .recordSpanFinished(INTEGRATION, spanKind, isRootSpan, true, span.isError(), false); } } else if (span != null) { - // Tracing still runs with LLM Observability off, where these four remain resolvable. - GenAiApmTags.applyWithoutLlmObs( - span, - operationName(span), - requestedModel(span), - stringTag(span, CommonTags.MODEL_PROVIDER), - Config.get().getLlmObsMlApp()); + // Tracing still runs with LLM Observability off, where these remain resolvable. + GenAiApmTags.apply( + span, operationName(span), requestedModel(span), Config.get().getLlmObsMlApp()); } super.doBeforeFinish(context); } @@ -220,15 +217,6 @@ private static String requestedModel(AgentSpan span) { return model != null ? model : stringTag(span, CommonTags.OPENAI_REQUEST_MODEL); } - private static String stringTag(AgentSpan span, String key) { - Object value = span.getTag(key); - if (value == null) { - return null; - } - String string = value.toString(); - return string.isEmpty() ? null : string; - } - public void withHttpResponse(AgentSpan span, Headers headers) { if (!llmObsEnabled) { return; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 4a4d06e4784..37ba7178988 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -406,6 +406,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "gen_ai.provider.name" "openai" "gen_ai.application.name" String if (!isStreaming) { + // streamed completions missing usage data "gen_ai.usage.input_tokens" Double "gen_ai.usage.output_tokens" Double "gen_ai.usage.total_tokens" Double diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java index 9a4aabe331e..1324ec98a7a 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java @@ -46,96 +46,61 @@ public final class GenAiApmTags { {LLMOBS_METRIC_PREFIX + "reasoning_output_tokens", USAGE_REASONING_OUTPUT_TOKENS}, }; + public static void apply(AgentSpan span) { + apply(span, null, null, null); + } + /** - * Writes the attributes onto a span that is not yet finished, reading them back from its {@code - * _ml_obs_tag.} / {@code _ml_obs_metric.} tags. No-op for a span with no LLM Observability kind. + * Writes the attributes onto a span that is not yet finished, reading them from its {@code + * _ml_obs_tag.} / {@code _ml_obs_metric.} tags. The arguments take precedence over those tags and + * cover instrumentation that traces with LLM Observability disabled, where they are not all set. + * No-op for a span with no resolvable operation. */ - public static void apply(AgentSpan span) { + public static void apply(AgentSpan span, String operationName, String modelName, String mlApp) { if (span == null) { return; } - String spanKind = stringTag(span, SPAN_KIND_TAG); - if (spanKind == null) { + String operation = firstNonEmpty(operationName, stringTag(span, SPAN_KIND_TAG)); + if (operation == null) { return; } - setScalars( - span, - spanKind, - stringTag(span, MODEL_NAME_TAG), - stringTag(span, MODEL_PROVIDER_TAG), - stringTag(span, ML_APP_TAG), - stringTag(span, SESSION_ID_TAG)); + span.setTag(OPERATION_NAME, operation); - // Other kinds carry unrelated metrics that a gen_ai.usage.* key would misrepresent. - if (isModelBacked(spanKind)) { - for (String[] metric : TOKEN_METRICS) { - Object value = span.getTag(metric[0]); - if (value instanceof Number) { - span.setMetric(metric[1], ((Number) value).doubleValue()); - } - } - } - } + boolean modelBacked = + Tags.LLMOBS_LLM_SPAN_KIND.equals(operation) + || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(operation); - /** - * Writes the subset available with LLM Observability disabled. Token usage and conversation id - * are never computed on that path, so they are left out. - */ - public static void applyWithoutLlmObs( - AgentSpan span, String operationName, String modelName, String modelProvider, String mlApp) { - if (span == null || operationName == null) { - return; + String model = firstNonEmpty(modelName, stringTag(span, MODEL_NAME_TAG)); + if (model != null || modelBacked) { + span.setTag(REQUEST_MODEL, model == null ? DEFAULT_MODEL : model); } - setScalars( - span, - operationName, - emptyToNull(modelName), - emptyToNull(modelProvider), - emptyToNull(mlApp), - null); - } - - private static void setScalars( - AgentSpan span, - String operationName, - String modelName, - String modelProvider, - String mlApp, - String sessionId) { - span.setTag(OPERATION_NAME, operationName); - - if (isModelBacked(operationName)) { - span.setTag(REQUEST_MODEL, modelName == null ? DEFAULT_MODEL : modelName); + String provider = stringTag(span, MODEL_PROVIDER_TAG); + if (provider != null || modelBacked) { span.setTag( - PROVIDER_NAME, - (modelProvider == null ? DEFAULT_MODEL : modelProvider).toLowerCase(Locale.ROOT)); - } else { - if (modelName != null) { - span.setTag(REQUEST_MODEL, modelName); - } - if (modelProvider != null) { - span.setTag(PROVIDER_NAME, modelProvider.toLowerCase(Locale.ROOT)); - } + PROVIDER_NAME, (provider == null ? DEFAULT_MODEL : provider).toLowerCase(Locale.ROOT)); } - - if (mlApp != null) { - span.setTag(APPLICATION_NAME, mlApp); + String application = firstNonEmpty(mlApp, stringTag(span, ML_APP_TAG)); + if (application != null) { + span.setTag(APPLICATION_NAME, application); } + String sessionId = stringTag(span, SESSION_ID_TAG); if (sessionId != null) { span.setTag(CONVERSATION_ID, sessionId); } - } - private static boolean isModelBacked(String spanKind) { - return Tags.LLMOBS_LLM_SPAN_KIND.equals(spanKind) - || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(spanKind); - } - - private static String emptyToNull(String value) { - return value == null || value.isEmpty() ? null : value; + // Other kinds carry unrelated metrics that a gen_ai.usage.* key would misrepresent. + if (modelBacked) { + for (String[] metric : TOKEN_METRICS) { + Object value = span.getTag(metric[0]); + if (value instanceof Number) { + span.setMetric(metric[1], ((Number) value).doubleValue()); + } + } + } } - private static String stringTag(AgentSpan span, String key) { + /** The value of {@code key} as a non-empty string, or null. */ + public static String stringTag(AgentSpan span, String key) { Object value = span.getTag(key); if (value == null) { return null; @@ -144,5 +109,9 @@ private static String stringTag(AgentSpan span, String key) { return string.isEmpty() ? null : string; } + private static String firstNonEmpty(String preferred, String fallback) { + return preferred == null || preferred.isEmpty() ? fallback : preferred; + } + private GenAiApmTags() {} } diff --git a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java index 81fa87c0c6c..98783a6badd 100644 --- a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java +++ b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java @@ -152,11 +152,14 @@ void spanWithoutLlmObsKindEmitsNothing() { @Test void nullSpanIsANoOp() { GenAiApmTags.apply(null); + GenAiApmTags.apply(null, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "app"); } @Test - void withoutLlmObsEmitsScalarsButNoUsageOrConversation() { - GenAiApmTags.applyWithoutLlmObs(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "OpenAI", "my-app"); + void argumentsStandInForMissingLlmObsTags() { + llmObsTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + + GenAiApmTags.apply(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "my-app"); assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); @@ -167,23 +170,31 @@ void withoutLlmObsEmitsScalarsButNoUsageOrConversation() { } @Test - void withoutLlmObsFallsBackToCustomForModelBackedKinds() { - GenAiApmTags.applyWithoutLlmObs(span, Tags.LLMOBS_EMBEDDING_SPAN_KIND, null, "", "app"); + void argumentsTakePrecedenceOverLlmObsTags() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_WORKFLOW_SPAN_KIND); + llmObsTag(LLMObsTags.MODEL_NAME, "gpt-3.5"); + llmObsTag(LLMObsTags.ML_APP, "tagged-app"); - assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); - assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); + GenAiApmTags.apply(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "my-app"); + + assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("my-app", tags.get(GenAiApmTags.APPLICATION_NAME)); } @Test - void withoutLlmObsIgnoresSpanWithNoOperationName() { - GenAiApmTags.applyWithoutLlmObs(span, null, "gpt-4", "openai", "my-app"); + void modelBackedArgumentsFallBackToCustom() { + GenAiApmTags.apply(span, Tags.LLMOBS_EMBEDDING_SPAN_KIND, null, "app"); - assertTrue(tags.keySet().stream().noneMatch(key -> key.startsWith("gen_ai."))); + assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); } @Test - void withoutLlmObsNullSpanIsANoOp() { - GenAiApmTags.applyWithoutLlmObs(null, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "openai", "app"); + void spanWithNoResolvableOperationEmitsNothing() { + GenAiApmTags.apply(span, null, "gpt-4", "my-app"); + + assertTrue(tags.keySet().stream().noneMatch(key -> key.startsWith("gen_ai."))); } private void llmObsTag(String key, String value) { From a33c1c94852d9a2574ff2a67805be5f49460c557 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Mon, 14 Sep 2026 11:14:57 -0400 Subject: [PATCH 6/7] Select the openai.request span by endpoint in the LLMObs-disabled test Both tests in the class produce an openai.request span, so taking the first one written let a late trace from the sibling test be asserted on. Co-Authored-By: Claude Opus 5 --- .../LlmObsContextPropagationForkedTest.java | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java index 16830ca0d2a..88fdec874ba 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java @@ -336,16 +336,14 @@ void parentlessOpenAiRequestSpanIsDroppedAtZeroSampleRate() throws Exception { class LlmObsDisabledForkedTest extends AbstractLlmObsOpenAiForkedTest { @Test - void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() throws Exception { + void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() { try { openAiClient.chat().completions().create(buildMinimalChatParams()); } catch (Exception ignored) { // The mock server returns no body, so the SDK may throw while parsing the response. } - writer.waitForTraces(1); - DDSpan openAiSpan = findSpanByOperationName(writer, "openai.request"); - assertNotNull(openAiSpan, "openai.request span should have been created"); + DDSpan openAiSpan = awaitOpenAiSpan("/v1/chat/completions"); assertEquals("llm", openAiSpan.getTag("gen_ai.operation.name")); // The mock returns no body, so the request model stands in for the absent response model. @@ -363,20 +361,36 @@ void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() throws Except } @Test - void embeddingMapsToTheEmbeddingOperation() throws Exception { + void embeddingMapsToTheEmbeddingOperation() { try { openAiClient.embeddings().create(buildMinimalEmbeddingParams()); } catch (Exception ignored) { // The mock server returns no body, so the SDK may throw while parsing the response. } - writer.waitForTraces(1); - DDSpan openAiSpan = findSpanByOperationName(writer, "openai.request"); - assertNotNull(openAiSpan, "openai.request span should have been created"); + DDSpan openAiSpan = awaitOpenAiSpan("/v1/embeddings"); assertEquals("embedding", openAiSpan.getTag("gen_ai.operation.name")); assertEquals( openAiSpan.getTag("openai.request.model"), openAiSpan.getTag("gen_ai.request.model")); assertEquals("openai", openAiSpan.getTag("gen_ai.provider.name")); } + + // Both tests here produce an openai.request span, so match on the endpoint rather than take the + // first one written: a trace arriving late from the sibling test would otherwise be picked up. + private DDSpan awaitOpenAiSpan(String endpoint) { + blockUntilTracesMatch(traces -> findOpenAiSpan(traces, endpoint) != null); + DDSpan span = findOpenAiSpan(writer, endpoint); + assertNotNull(span, "openai.request span for " + endpoint + " should have been created"); + return span; + } + + private static DDSpan findOpenAiSpan(List> traces, String endpoint) { + return traces.stream() + .flatMap(List::stream) + .filter(span -> "openai.request".equals(span.getOperationName().toString())) + .filter(span -> endpoint.equals(span.getTag("openai.request.endpoint"))) + .findFirst() + .orElse(null); + } } From eeeafa8a75709c15d43a2f3a9040e6891fb4eed2 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Wed, 16 Sep 2026 16:31:37 -0400 Subject: [PATCH 7/7] Emit no gen_ai.* attributes when LLM Observability is disabled The attributes are derived from the LLMObs tag set, which the openai decorators do not build with LLM Observability off. Drop the fallback that resolved operation, model and ml_app from APM tags instead, and collapse GenAiApmTags back to a single apply(span). Co-Authored-By: Claude Opus 5 --- .../openai_java/OpenAiDecorator.java | 24 ----------- .../LlmObsContextPropagationForkedTest.java | 40 +++++++---------- .../trace/api/llmobs/GenAiApmTags.java | 23 +++------- .../trace/api/llmobs/GenAiApmTagsTest.java | 43 ------------------- 4 files changed, 23 insertions(+), 107 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index a31cf5c95b3..b7b1bff9029 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -1,6 +1,5 @@ package datadog.trace.instrumentation.openai_java; -import static datadog.trace.api.llmobs.GenAiApmTags.stringTag; import static datadog.trace.bootstrap.instrumentation.api.AgentSpan.fromContext; import com.openai.core.ClientOptions; @@ -18,7 +17,6 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; -import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.ClientDecorator; import java.util.List; @@ -40,8 +38,6 @@ public class OpenAiDecorator extends ClientDecorator { private static final String TOKENS_LIMIT_METRIC = METRIC_PREFIX + "tokens.limit"; private static final String TOKENS_REMAINING_METRIC = METRIC_PREFIX + "tokens.remaining"; - private static final String EMBEDDINGS_ENDPOINT = "/v1/embeddings"; - private static final String HEADER_PREFIX = "x-ratelimit-"; private static final String LIMIT_REQUESTS_HEADER = HEADER_PREFIX + "limit-requests"; private static final String REMAINING_REQUESTS_HEADER = HEADER_PREFIX + "remaining-requests"; @@ -193,30 +189,10 @@ protected void doBeforeFinish(@Nonnull Context context) { LLMObsMetricCollector.get() .recordSpanFinished(INTEGRATION, spanKind, isRootSpan, true, span.isError(), false); } - } else if (span != null) { - // Tracing still runs with LLM Observability off, where these remain resolvable. - GenAiApmTags.apply( - span, operationName(span), requestedModel(span), Config.get().getLlmObsMlApp()); } super.doBeforeFinish(context); } - private static String operationName(AgentSpan span) { - String endpoint = stringTag(span, CommonTags.OPENAI_REQUEST_ENDPOINT); - if (endpoint == null) { - return null; - } - return EMBEDDINGS_ENDPOINT.equals(endpoint) - ? Tags.LLMOBS_EMBEDDING_SPAN_KIND - : Tags.LLMOBS_LLM_SPAN_KIND; - } - - /** The response model resolves aliases the request used, so it wins. */ - private static String requestedModel(AgentSpan span) { - String model = stringTag(span, CommonTags.OPENAI_RESPONSE_MODEL); - return model != null ? model : stringTag(span, CommonTags.OPENAI_REQUEST_MODEL); - } - public void withHttpResponse(AgentSpan span, Headers headers) { if (!llmObsEnabled) { return; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java index 88fdec874ba..60151bfd608 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.openai.client.OpenAIClient; import com.openai.client.okhttp.OpenAIOkHttpClient; @@ -329,51 +330,42 @@ void parentlessOpenAiRequestSpanIsDroppedAtZeroSampleRate() throws Exception { } /** - * Verifies the gen_ai.* attributes an openai.request span carries with LLM Observability disabled: - * operation, model, provider and application, but never token usage or conversation id. + * Verifies that an openai.request span carries no gen_ai.* attributes at all with LLM Observability + * disabled. The attributes are derived from the LLMObs tag set, which the decorator does not build + * in that configuration. */ @WithConfig(key = "llmobs.enabled", value = "false") class LlmObsDisabledForkedTest extends AbstractLlmObsOpenAiForkedTest { @Test - void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() { + void chatCompletionEmitsNoGenAiAttributes() { try { openAiClient.chat().completions().create(buildMinimalChatParams()); } catch (Exception ignored) { // The mock server returns no body, so the SDK may throw while parsing the response. } - DDSpan openAiSpan = awaitOpenAiSpan("/v1/chat/completions"); - - assertEquals("llm", openAiSpan.getTag("gen_ai.operation.name")); - // The mock returns no body, so the request model stands in for the absent response model. - assertEquals( - openAiSpan.getTag("openai.request.model"), openAiSpan.getTag("gen_ai.request.model")); - assertEquals("openai", openAiSpan.getTag("gen_ai.provider.name")); - assertNotNull(openAiSpan.getTag("gen_ai.application.name")); - - assertNull(openAiSpan.getTag("gen_ai.conversation.id")); - assertNull(openAiSpan.getTag("gen_ai.usage.input_tokens")); - assertNull(openAiSpan.getTag("gen_ai.usage.output_tokens")); - assertNull(openAiSpan.getTag("gen_ai.usage.total_tokens")); - - assertNull(openAiSpan.getTag("_ml_obs_tag.span.kind")); + assertNoGenAiTags(awaitOpenAiSpan("/v1/chat/completions")); } @Test - void embeddingMapsToTheEmbeddingOperation() { + void embeddingEmitsNoGenAiAttributes() { try { openAiClient.embeddings().create(buildMinimalEmbeddingParams()); } catch (Exception ignored) { // The mock server returns no body, so the SDK may throw while parsing the response. } - DDSpan openAiSpan = awaitOpenAiSpan("/v1/embeddings"); + assertNoGenAiTags(awaitOpenAiSpan("/v1/embeddings")); + } - assertEquals("embedding", openAiSpan.getTag("gen_ai.operation.name")); - assertEquals( - openAiSpan.getTag("openai.request.model"), openAiSpan.getTag("gen_ai.request.model")); - assertEquals("openai", openAiSpan.getTag("gen_ai.provider.name")); + private static void assertNoGenAiTags(DDSpan span) { + // The endpoint and model the span was selected by are APM tags of their own, still present. + assertNotNull(span.getTag("openai.request.model")); + assertTrue( + span.getTags().keySet().stream().noneMatch(key -> key.startsWith("gen_ai.")), + "openai.request span should carry no gen_ai.* tags"); + assertNull(span.getTag("_ml_obs_tag.span.kind")); } // Both tests here produce an openai.request span, so match on the endpoint rather than take the diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java index 1324ec98a7a..c3503d06047 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java @@ -46,21 +46,16 @@ public final class GenAiApmTags { {LLMOBS_METRIC_PREFIX + "reasoning_output_tokens", USAGE_REASONING_OUTPUT_TOKENS}, }; - public static void apply(AgentSpan span) { - apply(span, null, null, null); - } - /** * Writes the attributes onto a span that is not yet finished, reading them from its {@code - * _ml_obs_tag.} / {@code _ml_obs_metric.} tags. The arguments take precedence over those tags and - * cover instrumentation that traces with LLM Observability disabled, where they are not all set. - * No-op for a span with no resolvable operation. + * _ml_obs_tag.} / {@code _ml_obs_metric.} tags. No-op for a span with no LLM Observability span + * kind. */ - public static void apply(AgentSpan span, String operationName, String modelName, String mlApp) { + public static void apply(AgentSpan span) { if (span == null) { return; } - String operation = firstNonEmpty(operationName, stringTag(span, SPAN_KIND_TAG)); + String operation = stringTag(span, SPAN_KIND_TAG); if (operation == null) { return; } @@ -70,7 +65,7 @@ public static void apply(AgentSpan span, String operationName, String modelName, Tags.LLMOBS_LLM_SPAN_KIND.equals(operation) || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(operation); - String model = firstNonEmpty(modelName, stringTag(span, MODEL_NAME_TAG)); + String model = stringTag(span, MODEL_NAME_TAG); if (model != null || modelBacked) { span.setTag(REQUEST_MODEL, model == null ? DEFAULT_MODEL : model); } @@ -79,7 +74,7 @@ public static void apply(AgentSpan span, String operationName, String modelName, span.setTag( PROVIDER_NAME, (provider == null ? DEFAULT_MODEL : provider).toLowerCase(Locale.ROOT)); } - String application = firstNonEmpty(mlApp, stringTag(span, ML_APP_TAG)); + String application = stringTag(span, ML_APP_TAG); if (application != null) { span.setTag(APPLICATION_NAME, application); } @@ -100,7 +95,7 @@ public static void apply(AgentSpan span, String operationName, String modelName, } /** The value of {@code key} as a non-empty string, or null. */ - public static String stringTag(AgentSpan span, String key) { + private static String stringTag(AgentSpan span, String key) { Object value = span.getTag(key); if (value == null) { return null; @@ -109,9 +104,5 @@ public static String stringTag(AgentSpan span, String key) { return string.isEmpty() ? null : string; } - private static String firstNonEmpty(String preferred, String fallback) { - return preferred == null || preferred.isEmpty() ? fallback : preferred; - } - private GenAiApmTags() {} } diff --git a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java index 98783a6badd..c257168ee1e 100644 --- a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java +++ b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java @@ -152,49 +152,6 @@ void spanWithoutLlmObsKindEmitsNothing() { @Test void nullSpanIsANoOp() { GenAiApmTags.apply(null); - GenAiApmTags.apply(null, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "app"); - } - - @Test - void argumentsStandInForMissingLlmObsTags() { - llmObsTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); - - GenAiApmTags.apply(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "my-app"); - - assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); - assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); - assertEquals("openai", tags.get(GenAiApmTags.PROVIDER_NAME)); - assertEquals("my-app", tags.get(GenAiApmTags.APPLICATION_NAME)); - assertFalse(tags.containsKey(GenAiApmTags.CONVERSATION_ID)); - assertFalse(tags.containsKey(GenAiApmTags.USAGE_INPUT_TOKENS)); - } - - @Test - void argumentsTakePrecedenceOverLlmObsTags() { - llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_WORKFLOW_SPAN_KIND); - llmObsTag(LLMObsTags.MODEL_NAME, "gpt-3.5"); - llmObsTag(LLMObsTags.ML_APP, "tagged-app"); - - GenAiApmTags.apply(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "my-app"); - - assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); - assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); - assertEquals("my-app", tags.get(GenAiApmTags.APPLICATION_NAME)); - } - - @Test - void modelBackedArgumentsFallBackToCustom() { - GenAiApmTags.apply(span, Tags.LLMOBS_EMBEDDING_SPAN_KIND, null, "app"); - - assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); - assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); - } - - @Test - void spanWithNoResolvableOperationEmitsNothing() { - GenAiApmTags.apply(span, null, "gpt-4", "my-app"); - - assertTrue(tags.keySet().stream().noneMatch(key -> key.startsWith("gen_ai."))); } private void llmObsTag(String key, String value) {