From d0740e5ef8fe372eda231e623bac17aa50fe2da6 Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Tue, 30 Jun 2026 21:58:29 +0200 Subject: [PATCH 01/12] feat(Tracing): add token usage for AI --- composer.json | 2 + .../AIPlatformTracingCompilerPass.php | 34 ++++ src/DependencyInjection/Configuration.php | 6 + src/DependencyInjection/Extension.php | 50 +++++ src/Tracing/AI/Platform/TracingPlatform.php | 58 ++++++ .../AI/Platform/TracingResultConverter.php | 53 ++++++ .../Platform/TracingTokenUsageExtractor.php | 50 +++++ .../AI/Platform/TracingPlatformTest.php | 172 ++++++++++++++++++ 8 files changed, 425 insertions(+) create mode 100644 src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php create mode 100644 src/Tracing/AI/Platform/TracingPlatform.php create mode 100644 src/Tracing/AI/Platform/TracingResultConverter.php create mode 100644 src/Tracing/AI/Platform/TracingTokenUsageExtractor.php create mode 100644 tests/Tracing/AI/Platform/TracingPlatformTest.php diff --git a/composer.json b/composer.json index 446e5b2..57ab7a7 100644 --- a/composer.json +++ b/composer.json @@ -28,6 +28,7 @@ "open-telemetry/gen-otlp-protobuf": "^1.0", "phpstan/phpstan": "^2.0", "phpunit/phpunit": "^10.5", + "symfony/ai-platform": "^0.6", "symfony/framework-bundle": "^7.4 || ^8.0", "symfony/http-client": "^7.4 || ^8.0", "symfony/messenger": "^7.4 || ^8.0", @@ -39,6 +40,7 @@ }, "suggest": { "open-telemetry/exporter-otlp": "To export telemetry data to an OTLP compatible ingester (Jaeger, Tempo, Loki, etc.)", + "symfony/ai-platform": "To enable OpenTelemetry tracing for Symfony AI platform calls (gen_ai.* attributes, token usage)", "symfony/monolog-bundle": "To enable open-telemetry logging" }, "autoload": { diff --git a/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php new file mode 100644 index 0000000..3722219 --- /dev/null +++ b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php @@ -0,0 +1,34 @@ + + */ + +namespace Instrumentation\DependencyInjection\CompilerPass; + +use Instrumentation\Tracing\AI\Platform\TracingPlatform; +use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; +use Symfony\Component\DependencyInjection\ContainerBuilder; +use Symfony\Component\DependencyInjection\Definition; +use Symfony\Component\DependencyInjection\Reference; + +final class AIPlatformTracingCompilerPass implements CompilerPassInterface +{ + public function process(ContainerBuilder $container): void + { + foreach (array_keys($container->findTaggedServiceIds('ai.platform')) as $platformId) { + $system = str_contains($platformId, 'ai.platform.') + ? substr($platformId, \strlen('ai.platform.')) + : $platformId; + + $definition = (new Definition(TracingPlatform::class)) + ->setDecoratedService($platformId, priority: -512) + ->setArguments([new Reference('.inner'), $system]); + + $container->setDefinition('instrumentation.tracing.ai.platform.'.$system, $definition); + } + } +} diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index ac31a37..fa4bb2f 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -156,6 +156,12 @@ public function getConfigTreeBuilder(): TreeBuilder ->end() ->end() ->end() + ->arrayNode('ai') + ->addDefaultsIfNotSet() + ->children() + ->booleanNode('enabled')->defaultTrue()->end() + ->end() + ->end() ->arrayNode('doctrine') ->addDefaultsIfNotSet() ->children() diff --git a/src/DependencyInjection/Extension.php b/src/DependencyInjection/Extension.php index 05efc36..0b5af27 100644 --- a/src/DependencyInjection/Extension.php +++ b/src/DependencyInjection/Extension.php @@ -10,12 +10,14 @@ namespace Instrumentation\DependencyInjection; use Instrumentation\DependencyInjection\CompilerPass\DoctrineTracingCompilerPass; +use Instrumentation\DependencyInjection\CompilerPass\AIPlatformTracingCompilerPass; use Instrumentation\Tracing\Bridge\TraceUrlGenerator; use Instrumentation\Tracing\Bridge\TraceUrlGeneratorInterface; use Instrumentation\Tracing\Request\EventListener\AddUserEventSubscriber; use OpenTelemetry\SDK\Trace\SpanLimitsBuilder; use Symfony\Bundle\MonologBundle\MonologBundle; use Symfony\Component\Config\FileLocator; +use Symfony\AI\Platform\PlatformInterface; use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Extension\Extension as BaseExtension; @@ -71,6 +73,52 @@ public function prepend(ContainerBuilder $container): void public function process(ContainerBuilder $container): void { (new DoctrineTracingCompilerPass())->process($container); + if ($container->hasParameter('tracing.ai.enabled') && $container->getParameter('tracing.ai.enabled')) { + (new AIPlatformTracingCompilerPass())->process($container); + } + + if ($container->hasParameter('tracing.doctrine.connections') && $container->hasParameter('doctrine.connections')) { + /** @var array $connectionsToTrace */ + $connectionsToTrace = $container->getParameter('tracing.doctrine.connections'); + + /** @var array $connections */ + $connections = $container->getParameter('doctrine.connections'); + + if (empty($connectionsToTrace)) { + $connectionsToTrace = array_keys($connections); + } + + foreach ($connectionsToTrace as $connection) { + $serviceId = \sprintf('doctrine.dbal.%s_connection', $connection); + + if (!\in_array($serviceId, $connections, true)) { + throw new \InvalidArgumentException(\sprintf('No such connection: "%s".', $connection)); + } + + $configDef = $container->getDefinition(\sprintf('%s.configuration', $serviceId)); + + $middlewares = []; + foreach ($configDef->getMethodCalls() as $call) { + [$method, $arguments] = $call; + if ('setMiddlewares' === $method) { + $middlewares = array_merge($middlewares, $arguments[0]); + } + } + + $addedMiddlewares = []; + + if ($container->getParameter('tracing.doctrine.instrumentation')) { + $addedMiddlewares[] = new Reference(InstrumentationMiddleware::class); + } + if ($container->getParameter('tracing.doctrine.propagation')) { + $addedMiddlewares[] = new Reference(PropagationMiddleware::class); + } + + $configDef + ->removeMethodCall('setMiddlewares') + ->addMethodCall('setMiddlewares', [array_merge($middlewares, $addedMiddlewares)]); + } + } } /** @@ -126,6 +174,8 @@ protected function loadTracing(array $config, ContainerBuilder $container): void $config['doctrine']['enabled'] = $config['doctrine']['instrumentation'] || $config['doctrine']['propagation']; + $container->setParameter('tracing.ai.enabled', $this->isConfigEnabled($container, $config['ai']) && interface_exists(PlatformInterface::class)); + foreach (['request', 'command', 'message', 'doctrine'] as $feature) { if (!$this->isConfigEnabled($container, $config[$feature])) { continue; diff --git a/src/Tracing/AI/Platform/TracingPlatform.php b/src/Tracing/AI/Platform/TracingPlatform.php new file mode 100644 index 0000000..c207651 --- /dev/null +++ b/src/Tracing/AI/Platform/TracingPlatform.php @@ -0,0 +1,58 @@ + + */ + +namespace Instrumentation\Tracing\AI\Platform; + +use Instrumentation\Tracing\Tracing; +use OpenTelemetry\API\Trace\SpanKind; +use OpenTelemetry\API\Trace\StatusCode; +use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; +use Symfony\AI\Platform\PlatformInterface; +use Symfony\AI\Platform\Result\DeferredResult; + +final class TracingPlatform implements PlatformInterface +{ + public function __construct( + private readonly PlatformInterface $platform, + private readonly string $system, + ) { + } + + public function invoke(string $model, array|string|object $input, array $options = []): DeferredResult + { + $span = Tracing::getTracer() + ->spanBuilder('chat '.$this->system) + ->setSpanKind(SpanKind::KIND_CLIENT) + ->startSpan(); + + $span->setAttribute('gen_ai.operation.name', 'chat'); + $span->setAttribute('gen_ai.system', $this->system); + $span->setAttribute('gen_ai.request.model', $model); + + try { + $deferredResult = $this->platform->invoke($model, $input, $options); + } catch (\Throwable $e) { + $span->recordException($e); + $span->setStatus(StatusCode::STATUS_ERROR); + $span->end(); + throw $e; + } + + return new DeferredResult( + new TracingResultConverter($deferredResult->getResultConverter(), $span), + $deferredResult->getRawResult(), + $options, + ); + } + + public function getModelCatalog(): ModelCatalogInterface + { + return $this->platform->getModelCatalog(); + } +} diff --git a/src/Tracing/AI/Platform/TracingResultConverter.php b/src/Tracing/AI/Platform/TracingResultConverter.php new file mode 100644 index 0000000..3749908 --- /dev/null +++ b/src/Tracing/AI/Platform/TracingResultConverter.php @@ -0,0 +1,53 @@ + + */ + +namespace Instrumentation\Tracing\AI\Platform; + +use OpenTelemetry\API\Trace\SpanInterface; +use OpenTelemetry\API\Trace\StatusCode; +use Symfony\AI\Platform\Model; +use Symfony\AI\Platform\Result\RawResultInterface; +use Symfony\AI\Platform\Result\ResultInterface; +use Symfony\AI\Platform\ResultConverterInterface; +use Symfony\AI\Platform\TokenUsage\TokenUsageExtractorInterface; + +final class TracingResultConverter implements ResultConverterInterface +{ + public function __construct( + private readonly ResultConverterInterface $inner, + private readonly SpanInterface $span, + ) { + } + + public function supports(Model $model): bool + { + return $this->inner->supports($model); + } + + public function convert(RawResultInterface $result, array $options = []): ResultInterface + { + try { + $converted = $this->inner->convert($result, $options); + $this->span->setStatus(StatusCode::STATUS_OK); + + return $converted; + } catch (\Throwable $e) { + $this->span->recordException($e); + $this->span->setStatus(StatusCode::STATUS_ERROR); + $this->span->end(); + throw $e; + } + // On success, span is ended by TracingTokenUsageExtractor after token extraction. + } + + public function getTokenUsageExtractor(): ?TokenUsageExtractorInterface + { + return new TracingTokenUsageExtractor($this->inner->getTokenUsageExtractor(), $this->span); + } +} diff --git a/src/Tracing/AI/Platform/TracingTokenUsageExtractor.php b/src/Tracing/AI/Platform/TracingTokenUsageExtractor.php new file mode 100644 index 0000000..5693478 --- /dev/null +++ b/src/Tracing/AI/Platform/TracingTokenUsageExtractor.php @@ -0,0 +1,50 @@ + + */ + +namespace Instrumentation\Tracing\AI\Platform; + +use OpenTelemetry\API\Trace\SpanInterface; +use Symfony\AI\Platform\Result\RawResultInterface; +use Symfony\AI\Platform\TokenUsage\TokenUsageExtractorInterface; +use Symfony\AI\Platform\TokenUsage\TokenUsageInterface; + +final class TracingTokenUsageExtractor implements TokenUsageExtractorInterface +{ + public function __construct( + private readonly ?TokenUsageExtractorInterface $inner, + private readonly SpanInterface $span, + ) { + } + + public function extract(RawResultInterface $rawResult, array $options = []): ?TokenUsageInterface + { + try { + if (null === $this->inner) { + return null; + } + + $tokenUsage = $this->inner->extract($rawResult, $options); + + if (null === $tokenUsage) { + return null; + } + + if (null !== $tokenUsage->getPromptTokens()) { + $this->span->setAttribute('gen_ai.usage.input_tokens', $tokenUsage->getPromptTokens()); + } + if (null !== $tokenUsage->getCompletionTokens()) { + $this->span->setAttribute('gen_ai.usage.output_tokens', $tokenUsage->getCompletionTokens()); + } + + return $tokenUsage; + } finally { + $this->span->end(); + } + } +} diff --git a/tests/Tracing/AI/Platform/TracingPlatformTest.php b/tests/Tracing/AI/Platform/TracingPlatformTest.php new file mode 100644 index 0000000..0bd7023 --- /dev/null +++ b/tests/Tracing/AI/Platform/TracingPlatformTest.php @@ -0,0 +1,172 @@ + + */ + +namespace Tests\Instrumentation\Tracing\AI\Platform; + +use Instrumentation\Tracing\AI\Platform\TracingPlatform; +use Instrumentation\Tracing\Tracing; +use OpenTelemetry\API\Trace\SpanKind; +use OpenTelemetry\API\Trace\StatusCode; +use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; +use OpenTelemetry\SDK\Trace\SpanProcessor\SimpleSpanProcessor; +use OpenTelemetry\SDK\Trace\TracerProvider; +use PHPUnit\Framework\TestCase; +use Symfony\AI\Platform\PlatformInterface; +use Symfony\AI\Platform\Result\DeferredResult; +use Symfony\AI\Platform\Result\InMemoryRawResult; +use Symfony\AI\Platform\Result\TextResult; +use Symfony\AI\Platform\ResultConverterInterface; +use Symfony\AI\Platform\TokenUsage\TokenUsage; +use Symfony\AI\Platform\TokenUsage\TokenUsageExtractorInterface; + +class TracingPlatformTest extends TestCase +{ + private \ArrayObject $spans; + + protected function setUp(): void + { + $this->spans = new \ArrayObject(); + $tracerProvider = new TracerProvider(new SimpleSpanProcessor(new InMemoryExporter($this->spans))); + Tracing::setProvider($tracerProvider); + } + + public function testItCreatesASpanWithCorrectNameAndKind(): void + { + $platform = $this->buildPlatform('gemini'); + + $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); + + $this->assertCount(1, $this->spans); + $this->assertSame('chat gemini', $this->spans[0]->getName()); + $this->assertSame(SpanKind::KIND_CLIENT, $this->spans[0]->getKind()); + } + + public function testItSetsGenAiAttributes(): void + { + $platform = $this->buildPlatform('perplexity'); + + $platform->invoke('sonar', 'Hello')->asText(); + + $attributes = $this->spans[0]->getAttributes()->toArray(); + $this->assertSame('chat', $attributes['gen_ai.operation.name']); + $this->assertSame('perplexity', $attributes['gen_ai.system']); + $this->assertSame('sonar', $attributes['gen_ai.request.model']); + } + + public function testSpanEndsOnlyAfterResultIsConsumed(): void + { + $platform = $this->buildPlatform('gemini'); + + $result = $platform->invoke('gemini-2.5-pro', 'Hello'); + + $this->assertCount(0, $this->spans, 'Span must not end before result is consumed'); + + $result->asText(); + + $this->assertCount(1, $this->spans); + } + + public function testItSetsStatusOkOnSuccess(): void + { + $platform = $this->buildPlatform('gemini'); + + $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); + + $this->assertSame(StatusCode::STATUS_OK, $this->spans[0]->getStatus()->getCode()); + } + + public function testItAddsTokenUsageAttributesFromExtractor(): void + { + $extractor = $this->createMock(TokenUsageExtractorInterface::class); + $extractor->method('extract')->willReturn(new TokenUsage(promptTokens: 150, completionTokens: 42)); + + $platform = $this->buildPlatform('gemini', $extractor); + + $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); + + $attributes = $this->spans[0]->getAttributes()->toArray(); + $this->assertSame(150, $attributes['gen_ai.usage.input_tokens']); + $this->assertSame(42, $attributes['gen_ai.usage.output_tokens']); + } + + public function testItDoesNotSetTokenAttributesWhenExtractorReturnsNull(): void + { + $extractor = $this->createMock(TokenUsageExtractorInterface::class); + $extractor->method('extract')->willReturn(null); + + $platform = $this->buildPlatform('gemini', $extractor); + + $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); + + $attributes = $this->spans[0]->getAttributes()->toArray(); + $this->assertArrayNotHasKey('gen_ai.usage.input_tokens', $attributes); + $this->assertArrayNotHasKey('gen_ai.usage.output_tokens', $attributes); + } + + public function testItDoesNotSetTokenAttributesWhenNoExtractor(): void + { + $platform = $this->buildPlatform('gemini'); + + $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); + + $attributes = $this->spans[0]->getAttributes()->toArray(); + $this->assertArrayNotHasKey('gen_ai.usage.input_tokens', $attributes); + $this->assertArrayNotHasKey('gen_ai.usage.output_tokens', $attributes); + } + + public function testItEndsSpanWithErrorStatusWhenPlatformThrows(): void + { + $inner = $this->createMock(PlatformInterface::class); + $inner->method('invoke')->willThrowException(new \RuntimeException('API unreachable')); + + $platform = new TracingPlatform($inner, 'gemini'); + + try { + $platform->invoke('gemini-2.5-pro', 'Hello'); + $this->fail('Expected RuntimeException'); + } catch (\RuntimeException) { + } + + $this->assertCount(1, $this->spans); + $this->assertSame(StatusCode::STATUS_ERROR, $this->spans[0]->getStatus()->getCode()); + } + + public function testItEndsSpanWithErrorStatusWhenConverterThrows(): void + { + $converter = $this->createMock(ResultConverterInterface::class); + $converter->method('convert')->willThrowException(new \RuntimeException('Bad response')); + $converter->method('getTokenUsageExtractor')->willReturn(null); + + $inner = $this->createMock(PlatformInterface::class); + $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult())); + + $platform = new TracingPlatform($inner, 'gemini'); + + try { + $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); + $this->fail('Expected RuntimeException'); + } catch (\RuntimeException) { + } + + $this->assertCount(1, $this->spans); + $this->assertSame(StatusCode::STATUS_ERROR, $this->spans[0]->getStatus()->getCode()); + } + + private function buildPlatform(string $system, ?TokenUsageExtractorInterface $extractor = null): TracingPlatform + { + $converter = $this->createMock(ResultConverterInterface::class); + $converter->method('convert')->willReturn(new TextResult('response')); + $converter->method('getTokenUsageExtractor')->willReturn($extractor); + + $inner = $this->createMock(PlatformInterface::class); + $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult())); + + return new TracingPlatform($inner, $system); + } +} From af2bb98b102c0216c4e81c61c347db4efaa52dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sanchez=20Cl=C3=A9ment?= Date: Fri, 19 Jun 2026 09:30:48 +0200 Subject: [PATCH 02/12] fix(Tracing): make AI tracing opt-in and align with other features --- src/DependencyInjection/Configuration.php | 2 +- src/DependencyInjection/Extension.php | 5 +---- src/DependencyInjection/config/tracing/ai.php | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 src/DependencyInjection/config/tracing/ai.php diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index fa4bb2f..901dbef 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -159,7 +159,7 @@ public function getConfigTreeBuilder(): TreeBuilder ->arrayNode('ai') ->addDefaultsIfNotSet() ->children() - ->booleanNode('enabled')->defaultTrue()->end() + ->booleanNode('enabled')->defaultFalse()->end() ->end() ->end() ->arrayNode('doctrine') diff --git a/src/DependencyInjection/Extension.php b/src/DependencyInjection/Extension.php index 0b5af27..998630c 100644 --- a/src/DependencyInjection/Extension.php +++ b/src/DependencyInjection/Extension.php @@ -17,7 +17,6 @@ use OpenTelemetry\SDK\Trace\SpanLimitsBuilder; use Symfony\Bundle\MonologBundle\MonologBundle; use Symfony\Component\Config\FileLocator; -use Symfony\AI\Platform\PlatformInterface; use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Extension\Extension as BaseExtension; @@ -174,9 +173,7 @@ protected function loadTracing(array $config, ContainerBuilder $container): void $config['doctrine']['enabled'] = $config['doctrine']['instrumentation'] || $config['doctrine']['propagation']; - $container->setParameter('tracing.ai.enabled', $this->isConfigEnabled($container, $config['ai']) && interface_exists(PlatformInterface::class)); - - foreach (['request', 'command', 'message', 'doctrine'] as $feature) { + foreach (['request', 'command', 'message', 'ai', 'doctrine'] as $feature) { if (!$this->isConfigEnabled($container, $config[$feature])) { continue; } diff --git a/src/DependencyInjection/config/tracing/ai.php b/src/DependencyInjection/config/tracing/ai.php new file mode 100644 index 0000000..e40b6bb --- /dev/null +++ b/src/DependencyInjection/config/tracing/ai.php @@ -0,0 +1,15 @@ + + */ + +use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator; + +return static function (ContainerConfigurator $container) { + $container->parameters() + ->set('tracing.ai.enabled', true); +}; From 4694f73eb9488fb8ea941f29c49c2eaa52b28888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sanchez=20Cl=C3=A9ment?= Date: Fri, 19 Jun 2026 12:22:12 +0200 Subject: [PATCH 03/12] feat(Tracing): cleaning and refactoring --- .../AIPlatformTracingCompilerPass.php | 8 ++-- src/DependencyInjection/Extension.php | 47 ++----------------- src/DependencyInjection/config/tracing/ai.php | 6 ++- src/Tracing/AI/Platform/TracingPlatform.php | 7 ++- .../AI/Platform/TracingResultConverter.php | 7 +-- .../AI/Platform/TracingPlatformTest.php | 4 +- 6 files changed, 22 insertions(+), 57 deletions(-) diff --git a/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php index 3722219..64cfd41 100644 --- a/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php +++ b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php @@ -19,11 +19,11 @@ final class AIPlatformTracingCompilerPass implements CompilerPassInterface { public function process(ContainerBuilder $container): void { - foreach (array_keys($container->findTaggedServiceIds('ai.platform')) as $platformId) { - $system = str_contains($platformId, 'ai.platform.') - ? substr($platformId, \strlen('ai.platform.')) - : $platformId; + foreach ($container->findTaggedServiceIds('ai.platform') as $platformId => $tags) { + $system = $tags[0]['name'] ?? $platformId; + // Low priority ensures TracingPlatform is the outermost decorator, + // so the span covers the full call including any inner decorators (retry, cache, etc.). $definition = (new Definition(TracingPlatform::class)) ->setDecoratedService($platformId, priority: -512) ->setArguments([new Reference('.inner'), $system]); diff --git a/src/DependencyInjection/Extension.php b/src/DependencyInjection/Extension.php index 998630c..b2e8110 100644 --- a/src/DependencyInjection/Extension.php +++ b/src/DependencyInjection/Extension.php @@ -11,6 +11,7 @@ use Instrumentation\DependencyInjection\CompilerPass\DoctrineTracingCompilerPass; use Instrumentation\DependencyInjection\CompilerPass\AIPlatformTracingCompilerPass; +use Instrumentation\Tracing\AI\Platform\TracingPlatform; use Instrumentation\Tracing\Bridge\TraceUrlGenerator; use Instrumentation\Tracing\Bridge\TraceUrlGeneratorInterface; use Instrumentation\Tracing\Request\EventListener\AddUserEventSubscriber; @@ -72,51 +73,9 @@ public function prepend(ContainerBuilder $container): void public function process(ContainerBuilder $container): void { (new DoctrineTracingCompilerPass())->process($container); - if ($container->hasParameter('tracing.ai.enabled') && $container->getParameter('tracing.ai.enabled')) { - (new AIPlatformTracingCompilerPass())->process($container); - } - - if ($container->hasParameter('tracing.doctrine.connections') && $container->hasParameter('doctrine.connections')) { - /** @var array $connectionsToTrace */ - $connectionsToTrace = $container->getParameter('tracing.doctrine.connections'); - - /** @var array $connections */ - $connections = $container->getParameter('doctrine.connections'); - - if (empty($connectionsToTrace)) { - $connectionsToTrace = array_keys($connections); - } - - foreach ($connectionsToTrace as $connection) { - $serviceId = \sprintf('doctrine.dbal.%s_connection', $connection); - - if (!\in_array($serviceId, $connections, true)) { - throw new \InvalidArgumentException(\sprintf('No such connection: "%s".', $connection)); - } - $configDef = $container->getDefinition(\sprintf('%s.configuration', $serviceId)); - - $middlewares = []; - foreach ($configDef->getMethodCalls() as $call) { - [$method, $arguments] = $call; - if ('setMiddlewares' === $method) { - $middlewares = array_merge($middlewares, $arguments[0]); - } - } - - $addedMiddlewares = []; - - if ($container->getParameter('tracing.doctrine.instrumentation')) { - $addedMiddlewares[] = new Reference(InstrumentationMiddleware::class); - } - if ($container->getParameter('tracing.doctrine.propagation')) { - $addedMiddlewares[] = new Reference(PropagationMiddleware::class); - } - - $configDef - ->removeMethodCall('setMiddlewares') - ->addMethodCall('setMiddlewares', [array_merge($middlewares, $addedMiddlewares)]); - } + if ($container->hasDefinition(TracingPlatform::class)) { + (new AIPlatformTracingCompilerPass())->process($container); } } diff --git a/src/DependencyInjection/config/tracing/ai.php b/src/DependencyInjection/config/tracing/ai.php index e40b6bb..c362c9c 100644 --- a/src/DependencyInjection/config/tracing/ai.php +++ b/src/DependencyInjection/config/tracing/ai.php @@ -7,9 +7,11 @@ * (c) Worldia */ +use Instrumentation\Tracing\AI\Platform\TracingPlatform; use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator; return static function (ContainerConfigurator $container) { - $container->parameters() - ->set('tracing.ai.enabled', true); + $container->services() + ->set(TracingPlatform::class) + ->abstract(); }; diff --git a/src/Tracing/AI/Platform/TracingPlatform.php b/src/Tracing/AI/Platform/TracingPlatform.php index c207651..99a964d 100644 --- a/src/Tracing/AI/Platform/TracingPlatform.php +++ b/src/Tracing/AI/Platform/TracingPlatform.php @@ -26,12 +26,14 @@ public function __construct( public function invoke(string $model, array|string|object $input, array $options = []): DeferredResult { + $operationName = $options['extra']['operation_name'] ?? 'symfony_ai'; + $span = Tracing::getTracer() - ->spanBuilder('chat '.$this->system) + ->spanBuilder($operationName) ->setSpanKind(SpanKind::KIND_CLIENT) ->startSpan(); - $span->setAttribute('gen_ai.operation.name', 'chat'); + $span->setAttribute('gen_ai.operation.name', $operationName); $span->setAttribute('gen_ai.system', $this->system); $span->setAttribute('gen_ai.request.model', $model); @@ -49,6 +51,7 @@ public function invoke(string $model, array|string|object $input, array $options $deferredResult->getRawResult(), $options, ); + } public function getModelCatalog(): ModelCatalogInterface diff --git a/src/Tracing/AI/Platform/TracingResultConverter.php b/src/Tracing/AI/Platform/TracingResultConverter.php index 3749908..ad3b7ca 100644 --- a/src/Tracing/AI/Platform/TracingResultConverter.php +++ b/src/Tracing/AI/Platform/TracingResultConverter.php @@ -34,15 +34,16 @@ public function convert(RawResultInterface $result, array $options = []): Result { try { $converted = $this->inner->convert($result, $options); - $this->span->setStatus(StatusCode::STATUS_OK); - - return $converted; } catch (\Throwable $e) { $this->span->recordException($e); $this->span->setStatus(StatusCode::STATUS_ERROR); $this->span->end(); throw $e; } + + $this->span->setStatus(StatusCode::STATUS_OK); + + return $converted; // On success, span is ended by TracingTokenUsageExtractor after token extraction. } diff --git a/tests/Tracing/AI/Platform/TracingPlatformTest.php b/tests/Tracing/AI/Platform/TracingPlatformTest.php index 0bd7023..1d955b6 100644 --- a/tests/Tracing/AI/Platform/TracingPlatformTest.php +++ b/tests/Tracing/AI/Platform/TracingPlatformTest.php @@ -43,7 +43,7 @@ public function testItCreatesASpanWithCorrectNameAndKind(): void $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); $this->assertCount(1, $this->spans); - $this->assertSame('chat gemini', $this->spans[0]->getName()); + $this->assertSame('symfony_ai', $this->spans[0]->getName()); $this->assertSame(SpanKind::KIND_CLIENT, $this->spans[0]->getKind()); } @@ -54,7 +54,7 @@ public function testItSetsGenAiAttributes(): void $platform->invoke('sonar', 'Hello')->asText(); $attributes = $this->spans[0]->getAttributes()->toArray(); - $this->assertSame('chat', $attributes['gen_ai.operation.name']); + $this->assertSame('symfony_ai', $attributes['gen_ai.operation.name']); $this->assertSame('perplexity', $attributes['gen_ai.system']); $this->assertSame('sonar', $attributes['gen_ai.request.model']); } From 2753500879010ca890b0834dd0651343985c1ea0 Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Fri, 19 Jun 2026 12:47:21 +0200 Subject: [PATCH 04/12] feat(Tracing): add agent and tool execution spans for Symfony AI Extends the AI platform tracing with the two higher levels of the Symfony AI stack, following the OTel gen_ai semantic conventions: - TracingAgent decorates ai.agent services, emitting an INTERNAL `invoke_agent` span (gen_ai.agent.name) that parents the platform's CLIENT `chat` spans. - TracingToolbox decorates ai.toolbox services, emitting an INTERNAL `execute_tool` span (gen_ai.tool.name, gen_ai.tool.call.id) per call. Both are wired via AIAgentTracingCompilerPass, mirroring the platform pass: abstract services in config/tracing/ai.php, decoration keyed on the ai.agent / ai.toolbox tags at priority -512 (outermost). Attribute names and values use OpenTelemetry\SemConv constants. symfony/ai-agent added as a dev/suggested optional dependency. Co-Authored-By: Claude Opus 4.8 (1M context) --- composer.json | 2 + .../AIAgentTracingCompilerPass.php | 46 ++++++++ src/DependencyInjection/Extension.php | 8 +- src/DependencyInjection/config/tracing/ai.php | 6 + src/Tracing/AI/Agent/TracingAgent.php | 67 +++++++++++ src/Tracing/AI/Toolbox/TracingToolbox.php | 68 +++++++++++ tests/Tracing/AI/Agent/TracingAgentTest.php | 103 +++++++++++++++++ .../Tracing/AI/Toolbox/TracingToolboxTest.php | 109 ++++++++++++++++++ 8 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php create mode 100644 src/Tracing/AI/Agent/TracingAgent.php create mode 100644 src/Tracing/AI/Toolbox/TracingToolbox.php create mode 100644 tests/Tracing/AI/Agent/TracingAgentTest.php create mode 100644 tests/Tracing/AI/Toolbox/TracingToolboxTest.php diff --git a/composer.json b/composer.json index 57ab7a7..2f2b318 100644 --- a/composer.json +++ b/composer.json @@ -28,6 +28,7 @@ "open-telemetry/gen-otlp-protobuf": "^1.0", "phpstan/phpstan": "^2.0", "phpunit/phpunit": "^10.5", + "symfony/ai-agent": "^0.6", "symfony/ai-platform": "^0.6", "symfony/framework-bundle": "^7.4 || ^8.0", "symfony/http-client": "^7.4 || ^8.0", @@ -40,6 +41,7 @@ }, "suggest": { "open-telemetry/exporter-otlp": "To export telemetry data to an OTLP compatible ingester (Jaeger, Tempo, Loki, etc.)", + "symfony/ai-agent": "To enable OpenTelemetry tracing for Symfony AI agents (gen_ai.* invoke_agent and execute_tool spans)", "symfony/ai-platform": "To enable OpenTelemetry tracing for Symfony AI platform calls (gen_ai.* attributes, token usage)", "symfony/monolog-bundle": "To enable open-telemetry logging" }, diff --git a/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php b/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php new file mode 100644 index 0000000..6cd2ab0 --- /dev/null +++ b/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php @@ -0,0 +1,46 @@ + + */ + +namespace Instrumentation\DependencyInjection\CompilerPass; + +use Instrumentation\Tracing\AI\Agent\TracingAgent; +use Instrumentation\Tracing\AI\Toolbox\TracingToolbox; +use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; +use Symfony\Component\DependencyInjection\ContainerBuilder; +use Symfony\Component\DependencyInjection\Definition; +use Symfony\Component\DependencyInjection\Reference; + +final class AIAgentTracingCompilerPass implements CompilerPassInterface +{ + public function process(ContainerBuilder $container): void + { + // Low priority ensures the tracing decorator is the outermost one, so the + // span covers the full call including any inner decorators (e.g. the + // profiler's traceable decorators registered at -1024). + foreach ($container->findTaggedServiceIds('ai.agent') as $agentId => $tags) { + $name = $tags[0]['name'] ?? $agentId; + + $definition = (new Definition(TracingAgent::class)) + ->setDecoratedService($agentId, priority: -512) + ->setArguments([new Reference('.inner')]); + + $container->setDefinition('instrumentation.tracing.ai.agent.'.$name, $definition); + } + + foreach ($container->findTaggedServiceIds('ai.toolbox') as $toolboxId => $tags) { + $name = $tags[0]['name'] ?? $toolboxId; + + $definition = (new Definition(TracingToolbox::class)) + ->setDecoratedService($toolboxId, priority: -512) + ->setArguments([new Reference('.inner')]); + + $container->setDefinition('instrumentation.tracing.ai.toolbox.'.$name, $definition); + } + } +} diff --git a/src/DependencyInjection/Extension.php b/src/DependencyInjection/Extension.php index b2e8110..9703e6b 100644 --- a/src/DependencyInjection/Extension.php +++ b/src/DependencyInjection/Extension.php @@ -9,8 +9,10 @@ namespace Instrumentation\DependencyInjection; -use Instrumentation\DependencyInjection\CompilerPass\DoctrineTracingCompilerPass; +use Instrumentation\DependencyInjection\CompilerPass\AIAgentTracingCompilerPass; use Instrumentation\DependencyInjection\CompilerPass\AIPlatformTracingCompilerPass; +use Instrumentation\DependencyInjection\CompilerPass\DoctrineTracingCompilerPass; +use Instrumentation\Tracing\AI\Agent\TracingAgent; use Instrumentation\Tracing\AI\Platform\TracingPlatform; use Instrumentation\Tracing\Bridge\TraceUrlGenerator; use Instrumentation\Tracing\Bridge\TraceUrlGeneratorInterface; @@ -77,6 +79,10 @@ public function process(ContainerBuilder $container): void if ($container->hasDefinition(TracingPlatform::class)) { (new AIPlatformTracingCompilerPass())->process($container); } + + if ($container->hasDefinition(TracingAgent::class)) { + (new AIAgentTracingCompilerPass())->process($container); + } } /** diff --git a/src/DependencyInjection/config/tracing/ai.php b/src/DependencyInjection/config/tracing/ai.php index c362c9c..a43760c 100644 --- a/src/DependencyInjection/config/tracing/ai.php +++ b/src/DependencyInjection/config/tracing/ai.php @@ -7,11 +7,17 @@ * (c) Worldia */ +use Instrumentation\Tracing\AI\Agent\TracingAgent; use Instrumentation\Tracing\AI\Platform\TracingPlatform; +use Instrumentation\Tracing\AI\Toolbox\TracingToolbox; use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator; return static function (ContainerConfigurator $container) { $container->services() ->set(TracingPlatform::class) + ->abstract() + ->set(TracingAgent::class) + ->abstract() + ->set(TracingToolbox::class) ->abstract(); }; diff --git a/src/Tracing/AI/Agent/TracingAgent.php b/src/Tracing/AI/Agent/TracingAgent.php new file mode 100644 index 0000000..65ad824 --- /dev/null +++ b/src/Tracing/AI/Agent/TracingAgent.php @@ -0,0 +1,67 @@ + + */ + +namespace Instrumentation\Tracing\AI\Agent; + +use Instrumentation\Tracing\Tracing; +use OpenTelemetry\API\Trace\SpanKind; +use OpenTelemetry\API\Trace\StatusCode; +use OpenTelemetry\SemConv\TraceAttributes; +use OpenTelemetry\SemConv\TraceAttributeValues; +use Symfony\AI\Agent\AgentInterface; +use Symfony\AI\Platform\Message\MessageBag; +use Symfony\AI\Platform\Result\ResultInterface; + +/** + * Decorates an agent to create an `invoke_agent` span around each call. + * + * The agent orchestrates in-process (running input/output processors and the + * tool-calling loop), so the span is INTERNAL and acts as the parent of the + * CLIENT `chat` spans emitted by the underlying platform and the INTERNAL + * `execute_tool` spans emitted by the toolbox. + */ +final class TracingAgent implements AgentInterface +{ + public function __construct( + private readonly AgentInterface $agent, + ) { + } + + public function call(MessageBag $messages, array $options = []): ResultInterface + { + $name = $this->agent->getName(); + + $span = Tracing::getTracer() + ->spanBuilder('invoke_agent '.$name) + ->setSpanKind(SpanKind::KIND_INTERNAL) + ->startSpan(); + + $span->setAttribute(TraceAttributes::GEN_AI_OPERATION_NAME, TraceAttributeValues::GEN_AI_OPERATION_NAME_INVOKE_AGENT); + $span->setAttribute(TraceAttributes::GEN_AI_AGENT_NAME, $name); + + try { + $result = $this->agent->call($messages, $options); + $span->setStatus(StatusCode::STATUS_OK); + + return $result; + } catch (\Throwable $e) { + $span->recordException($e); + $span->setStatus(StatusCode::STATUS_ERROR); + + throw $e; + } finally { + $span->end(); + } + } + + public function getName(): string + { + return $this->agent->getName(); + } +} diff --git a/src/Tracing/AI/Toolbox/TracingToolbox.php b/src/Tracing/AI/Toolbox/TracingToolbox.php new file mode 100644 index 0000000..aa40618 --- /dev/null +++ b/src/Tracing/AI/Toolbox/TracingToolbox.php @@ -0,0 +1,68 @@ + + */ + +namespace Instrumentation\Tracing\AI\Toolbox; + +use Instrumentation\Tracing\Tracing; +use OpenTelemetry\API\Trace\SpanKind; +use OpenTelemetry\API\Trace\StatusCode; +use OpenTelemetry\SemConv\TraceAttributes; +use OpenTelemetry\SemConv\TraceAttributeValues; +use Symfony\AI\Agent\Toolbox\ToolboxInterface; +use Symfony\AI\Agent\Toolbox\ToolResult; +use Symfony\AI\Platform\Result\ToolCall; +use Symfony\AI\Platform\Tool\Tool; + +/** + * Decorates a toolbox to create an `execute_tool` span around each tool call. + * + * Tools run in-process as part of the agent loop, so the span is INTERNAL and + * nests under the `invoke_agent` span created by {@see \Instrumentation\Tracing\AI\Agent\TracingAgent}. + */ +final class TracingToolbox implements ToolboxInterface +{ + public function __construct( + private readonly ToolboxInterface $toolbox, + ) { + } + + /** + * @return Tool[] + */ + public function getTools(): array + { + return $this->toolbox->getTools(); + } + + public function execute(ToolCall $toolCall): ToolResult + { + $span = Tracing::getTracer() + ->spanBuilder('execute_tool '.$toolCall->getName()) + ->setSpanKind(SpanKind::KIND_INTERNAL) + ->startSpan(); + + $span->setAttribute(TraceAttributes::GEN_AI_OPERATION_NAME, TraceAttributeValues::GEN_AI_OPERATION_NAME_EXECUTE_TOOL); + $span->setAttribute(TraceAttributes::GEN_AI_TOOL_NAME, $toolCall->getName()); + $span->setAttribute(TraceAttributes::GEN_AI_TOOL_CALL_ID, $toolCall->getId()); + + try { + $result = $this->toolbox->execute($toolCall); + $span->setStatus(StatusCode::STATUS_OK); + + return $result; + } catch (\Throwable $e) { + $span->recordException($e); + $span->setStatus(StatusCode::STATUS_ERROR); + + throw $e; + } finally { + $span->end(); + } + } +} diff --git a/tests/Tracing/AI/Agent/TracingAgentTest.php b/tests/Tracing/AI/Agent/TracingAgentTest.php new file mode 100644 index 0000000..b311863 --- /dev/null +++ b/tests/Tracing/AI/Agent/TracingAgentTest.php @@ -0,0 +1,103 @@ + + */ + +namespace Tests\Instrumentation\Tracing\AI\Agent; + +use Instrumentation\Tracing\AI\Agent\TracingAgent; +use Instrumentation\Tracing\Tracing; +use OpenTelemetry\API\Trace\SpanKind; +use OpenTelemetry\API\Trace\StatusCode; +use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; +use OpenTelemetry\SDK\Trace\SpanProcessor\SimpleSpanProcessor; +use OpenTelemetry\SDK\Trace\TracerProvider; +use PHPUnit\Framework\TestCase; +use Symfony\AI\Agent\AgentInterface; +use Symfony\AI\Platform\Message\MessageBag; +use Symfony\AI\Platform\Result\ResultInterface; +use Symfony\AI\Platform\Result\TextResult; + +class TracingAgentTest extends TestCase +{ + private \ArrayObject $spans; + + protected function setUp(): void + { + $this->spans = new \ArrayObject(); + $tracerProvider = new TracerProvider(new SimpleSpanProcessor(new InMemoryExporter($this->spans))); + Tracing::setProvider($tracerProvider); + } + + public function testItCreatesAnInvokeAgentSpan(): void + { + $agent = $this->buildAgent('my_agent', new TextResult('Hello')); + + $agent->call(new MessageBag()); + + $this->assertCount(1, $this->spans); + $this->assertSame('invoke_agent my_agent', $this->spans[0]->getName()); + $this->assertSame(SpanKind::KIND_INTERNAL, $this->spans[0]->getKind()); + } + + public function testItSetsGenAiAttributes(): void + { + $agent = $this->buildAgent('my_agent', new TextResult('Hello')); + + $agent->call(new MessageBag()); + + $attributes = $this->spans[0]->getAttributes()->toArray(); + $this->assertSame('invoke_agent', $attributes['gen_ai.operation.name']); + $this->assertSame('my_agent', $attributes['gen_ai.agent.name']); + } + + public function testItSetsStatusOkOnSuccess(): void + { + $agent = $this->buildAgent('my_agent', new TextResult('Hello')); + + $agent->call(new MessageBag()); + + $this->assertSame(StatusCode::STATUS_OK, $this->spans[0]->getStatus()->getCode()); + } + + public function testItRecordsExceptionsAndEndsTheSpan(): void + { + $exception = new \RuntimeException('boom'); + $inner = $this->createMock(AgentInterface::class); + $inner->method('getName')->willReturn('my_agent'); + $inner->method('call')->willThrowException($exception); + + $agent = new TracingAgent($inner); + + try { + $agent->call(new MessageBag()); + $this->fail('Exception should have been rethrown'); + } catch (\RuntimeException $e) { + $this->assertSame($exception, $e); + } + + $this->assertCount(1, $this->spans); + $this->assertSame(StatusCode::STATUS_ERROR, $this->spans[0]->getStatus()->getCode()); + $this->assertSame('exception', $this->spans[0]->getEvents()[0]->getName()); + } + + public function testItDelegatesGetName(): void + { + $agent = $this->buildAgent('my_agent', new TextResult('Hello')); + + $this->assertSame('my_agent', $agent->getName()); + } + + private function buildAgent(string $name, ResultInterface $result): TracingAgent + { + $inner = $this->createMock(AgentInterface::class); + $inner->method('getName')->willReturn($name); + $inner->method('call')->willReturn($result); + + return new TracingAgent($inner); + } +} diff --git a/tests/Tracing/AI/Toolbox/TracingToolboxTest.php b/tests/Tracing/AI/Toolbox/TracingToolboxTest.php new file mode 100644 index 0000000..7d55772 --- /dev/null +++ b/tests/Tracing/AI/Toolbox/TracingToolboxTest.php @@ -0,0 +1,109 @@ + + */ + +namespace Tests\Instrumentation\Tracing\AI\Toolbox; + +use Instrumentation\Tracing\AI\Toolbox\TracingToolbox; +use Instrumentation\Tracing\Tracing; +use OpenTelemetry\API\Trace\SpanKind; +use OpenTelemetry\API\Trace\StatusCode; +use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; +use OpenTelemetry\SDK\Trace\SpanProcessor\SimpleSpanProcessor; +use OpenTelemetry\SDK\Trace\TracerProvider; +use PHPUnit\Framework\TestCase; +use Symfony\AI\Agent\Toolbox\ToolboxInterface; +use Symfony\AI\Agent\Toolbox\ToolResult; +use Symfony\AI\Platform\Result\ToolCall; + +class TracingToolboxTest extends TestCase +{ + private \ArrayObject $spans; + + protected function setUp(): void + { + $this->spans = new \ArrayObject(); + $tracerProvider = new TracerProvider(new SimpleSpanProcessor(new InMemoryExporter($this->spans))); + Tracing::setProvider($tracerProvider); + } + + public function testItCreatesAnExecuteToolSpan(): void + { + $toolCall = new ToolCall('call_123', 'get_weather', ['city' => 'Paris']); + $toolbox = $this->buildToolbox($toolCall, new ToolResult($toolCall, 'sunny')); + + $toolbox->execute($toolCall); + + $this->assertCount(1, $this->spans); + $this->assertSame('execute_tool get_weather', $this->spans[0]->getName()); + $this->assertSame(SpanKind::KIND_INTERNAL, $this->spans[0]->getKind()); + } + + public function testItSetsGenAiAttributes(): void + { + $toolCall = new ToolCall('call_123', 'get_weather', ['city' => 'Paris']); + $toolbox = $this->buildToolbox($toolCall, new ToolResult($toolCall, 'sunny')); + + $toolbox->execute($toolCall); + + $attributes = $this->spans[0]->getAttributes()->toArray(); + $this->assertSame('execute_tool', $attributes['gen_ai.operation.name']); + $this->assertSame('get_weather', $attributes['gen_ai.tool.name']); + $this->assertSame('call_123', $attributes['gen_ai.tool.call.id']); + } + + public function testItSetsStatusOkOnSuccess(): void + { + $toolCall = new ToolCall('call_123', 'get_weather'); + $toolbox = $this->buildToolbox($toolCall, new ToolResult($toolCall, 'sunny')); + + $toolbox->execute($toolCall); + + $this->assertSame(StatusCode::STATUS_OK, $this->spans[0]->getStatus()->getCode()); + } + + public function testItRecordsExceptionsAndEndsTheSpan(): void + { + $toolCall = new ToolCall('call_123', 'get_weather'); + $exception = new \RuntimeException('boom'); + + $inner = $this->createMock(ToolboxInterface::class); + $inner->method('execute')->willThrowException($exception); + + $toolbox = new TracingToolbox($inner); + + try { + $toolbox->execute($toolCall); + $this->fail('Exception should have been rethrown'); + } catch (\RuntimeException $e) { + $this->assertSame($exception, $e); + } + + $this->assertCount(1, $this->spans); + $this->assertSame(StatusCode::STATUS_ERROR, $this->spans[0]->getStatus()->getCode()); + $this->assertSame('exception', $this->spans[0]->getEvents()[0]->getName()); + } + + public function testItDelegatesGetTools(): void + { + $inner = $this->createMock(ToolboxInterface::class); + $inner->method('getTools')->willReturn([]); + + $toolbox = new TracingToolbox($inner); + + $this->assertSame([], $toolbox->getTools()); + } + + private function buildToolbox(ToolCall $expected, ToolResult $result): TracingToolbox + { + $inner = $this->createMock(ToolboxInterface::class); + $inner->method('execute')->with($expected)->willReturn($result); + + return new TracingToolbox($inner); + } +} From a8af2bfbc1b3fac3acbc27bcbb09be5380c950f3 Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Fri, 19 Jun 2026 13:17:14 +0200 Subject: [PATCH 05/12] refactor(Tracing): inject tracer and use swappable gen_ai attribute providers for AI agent/tool Aligns the agent/tool tracing decorators with the bundle's conventions: - Inject TracerProviderInterface (via TracerAwareTrait) instead of the static Tracing facade. The static accessor is intended for application code outside the bundle; in-bundle services use constructor injection. - Extract the inline gen_ai.* attributes into AgentAttributeProvider / ToolAttributeProvider behind interfaces registered in semconv.php, so consumers can override them (matching the other instrumentations). Attribute names/values use OpenTelemetry\SemConv constants. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../AIAgentTracingCompilerPass.php | 15 +++++++++-- .../config/semconv/semconv.php | 6 +++++ .../Attribute/AgentAttributeProvider.php | 25 ++++++++++++++++++ .../AgentAttributeProviderInterface.php | 20 ++++++++++++++ .../Attribute/ToolAttributeProvider.php | 26 +++++++++++++++++++ .../ToolAttributeProviderInterface.php | 20 ++++++++++++++ src/Tracing/AI/Agent/TracingAgent.php | 21 ++++++++------- src/Tracing/AI/Toolbox/TracingToolbox.php | 18 +++++++------ tests/Tracing/AI/Agent/TracingAgentTest.php | 10 +++---- .../Tracing/AI/Toolbox/TracingToolboxTest.php | 12 ++++----- 10 files changed, 142 insertions(+), 31 deletions(-) create mode 100644 src/Semantics/Attribute/AgentAttributeProvider.php create mode 100644 src/Semantics/Attribute/AgentAttributeProviderInterface.php create mode 100644 src/Semantics/Attribute/ToolAttributeProvider.php create mode 100644 src/Semantics/Attribute/ToolAttributeProviderInterface.php diff --git a/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php b/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php index 6cd2ab0..a0bd160 100644 --- a/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php +++ b/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php @@ -9,8 +9,11 @@ namespace Instrumentation\DependencyInjection\CompilerPass; +use Instrumentation\Semantics\Attribute\AgentAttributeProviderInterface; +use Instrumentation\Semantics\Attribute\ToolAttributeProviderInterface; use Instrumentation\Tracing\AI\Agent\TracingAgent; use Instrumentation\Tracing\AI\Toolbox\TracingToolbox; +use OpenTelemetry\API\Trace\TracerProviderInterface; use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Definition; @@ -28,7 +31,11 @@ public function process(ContainerBuilder $container): void $definition = (new Definition(TracingAgent::class)) ->setDecoratedService($agentId, priority: -512) - ->setArguments([new Reference('.inner')]); + ->setArguments([ + new Reference('.inner'), + new Reference(TracerProviderInterface::class), + new Reference(AgentAttributeProviderInterface::class), + ]); $container->setDefinition('instrumentation.tracing.ai.agent.'.$name, $definition); } @@ -38,7 +45,11 @@ public function process(ContainerBuilder $container): void $definition = (new Definition(TracingToolbox::class)) ->setDecoratedService($toolboxId, priority: -512) - ->setArguments([new Reference('.inner')]); + ->setArguments([ + new Reference('.inner'), + new Reference(TracerProviderInterface::class), + new Reference(ToolAttributeProviderInterface::class), + ]); $container->setDefinition('instrumentation.tracing.ai.toolbox.'.$name, $definition); } diff --git a/src/DependencyInjection/config/semconv/semconv.php b/src/DependencyInjection/config/semconv/semconv.php index ea73193..b5ecb53 100644 --- a/src/DependencyInjection/config/semconv/semconv.php +++ b/src/DependencyInjection/config/semconv/semconv.php @@ -7,6 +7,8 @@ * (c) Worldia */ +use Instrumentation\Semantics\Attribute\AgentAttributeProvider; +use Instrumentation\Semantics\Attribute\AgentAttributeProviderInterface; use Instrumentation\Semantics\Attribute\ClientRequestAttributeProvider; use Instrumentation\Semantics\Attribute\ClientRequestAttributeProviderInterface; use Instrumentation\Semantics\Attribute\DoctrineConnectionAttributeProvider; @@ -17,6 +19,8 @@ use Instrumentation\Semantics\Attribute\ServerRequestAttributeProviderInterface; use Instrumentation\Semantics\Attribute\ServerResponseAttributeProvider; use Instrumentation\Semantics\Attribute\ServerResponseAttributeProviderInterface; +use Instrumentation\Semantics\Attribute\ToolAttributeProvider; +use Instrumentation\Semantics\Attribute\ToolAttributeProviderInterface; use Instrumentation\Semantics\OperationName\ClientRequestOperationNameResolver; use Instrumentation\Semantics\OperationName\ClientRequestOperationNameResolverInterface; use Instrumentation\Semantics\OperationName\CommandOperationNameResolver; @@ -59,6 +63,8 @@ param('tracing.http.attributes.request_headers'), ]) ->set(DoctrineConnectionAttributeProviderInterface::class, DoctrineConnectionAttributeProvider::class) + ->set(AgentAttributeProviderInterface::class, AgentAttributeProvider::class) + ->set(ToolAttributeProviderInterface::class, ToolAttributeProvider::class) ->set(ClientRequestOperationNameResolverInterface::class, ClientRequestOperationNameResolver::class) ->set(MessageOperationNameResolverInterface::class, MessageOperationNameResolver::class) diff --git a/src/Semantics/Attribute/AgentAttributeProvider.php b/src/Semantics/Attribute/AgentAttributeProvider.php new file mode 100644 index 0000000..3e9c10c --- /dev/null +++ b/src/Semantics/Attribute/AgentAttributeProvider.php @@ -0,0 +1,25 @@ + + */ + +namespace Instrumentation\Semantics\Attribute; + +use OpenTelemetry\SemConv\TraceAttributes; +use OpenTelemetry\SemConv\TraceAttributeValues; +use Symfony\AI\Agent\AgentInterface; + +class AgentAttributeProvider implements AgentAttributeProviderInterface +{ + public function getAttributes(AgentInterface $agent): array + { + return [ + TraceAttributes::GEN_AI_OPERATION_NAME => TraceAttributeValues::GEN_AI_OPERATION_NAME_INVOKE_AGENT, + TraceAttributes::GEN_AI_AGENT_NAME => $agent->getName(), + ]; + } +} diff --git a/src/Semantics/Attribute/AgentAttributeProviderInterface.php b/src/Semantics/Attribute/AgentAttributeProviderInterface.php new file mode 100644 index 0000000..04b36bd --- /dev/null +++ b/src/Semantics/Attribute/AgentAttributeProviderInterface.php @@ -0,0 +1,20 @@ + + */ + +namespace Instrumentation\Semantics\Attribute; + +use Symfony\AI\Agent\AgentInterface; + +interface AgentAttributeProviderInterface +{ + /** + * @return array + */ + public function getAttributes(AgentInterface $agent): array; +} diff --git a/src/Semantics/Attribute/ToolAttributeProvider.php b/src/Semantics/Attribute/ToolAttributeProvider.php new file mode 100644 index 0000000..10060fd --- /dev/null +++ b/src/Semantics/Attribute/ToolAttributeProvider.php @@ -0,0 +1,26 @@ + + */ + +namespace Instrumentation\Semantics\Attribute; + +use OpenTelemetry\SemConv\TraceAttributes; +use OpenTelemetry\SemConv\TraceAttributeValues; +use Symfony\AI\Platform\Result\ToolCall; + +class ToolAttributeProvider implements ToolAttributeProviderInterface +{ + public function getAttributes(ToolCall $toolCall): array + { + return [ + TraceAttributes::GEN_AI_OPERATION_NAME => TraceAttributeValues::GEN_AI_OPERATION_NAME_EXECUTE_TOOL, + TraceAttributes::GEN_AI_TOOL_NAME => $toolCall->getName(), + TraceAttributes::GEN_AI_TOOL_CALL_ID => $toolCall->getId(), + ]; + } +} diff --git a/src/Semantics/Attribute/ToolAttributeProviderInterface.php b/src/Semantics/Attribute/ToolAttributeProviderInterface.php new file mode 100644 index 0000000..6e4348a --- /dev/null +++ b/src/Semantics/Attribute/ToolAttributeProviderInterface.php @@ -0,0 +1,20 @@ + + */ + +namespace Instrumentation\Semantics\Attribute; + +use Symfony\AI\Platform\Result\ToolCall; + +interface ToolAttributeProviderInterface +{ + /** + * @return array + */ + public function getAttributes(ToolCall $toolCall): array; +} diff --git a/src/Tracing/AI/Agent/TracingAgent.php b/src/Tracing/AI/Agent/TracingAgent.php index 65ad824..daf557d 100644 --- a/src/Tracing/AI/Agent/TracingAgent.php +++ b/src/Tracing/AI/Agent/TracingAgent.php @@ -9,11 +9,11 @@ namespace Instrumentation\Tracing\AI\Agent; -use Instrumentation\Tracing\Tracing; +use Instrumentation\Semantics\Attribute\AgentAttributeProviderInterface; +use Instrumentation\Tracing\TracerAwareTrait; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; -use OpenTelemetry\SemConv\TraceAttributes; -use OpenTelemetry\SemConv\TraceAttributeValues; +use OpenTelemetry\API\Trace\TracerProviderInterface; use Symfony\AI\Agent\AgentInterface; use Symfony\AI\Platform\Message\MessageBag; use Symfony\AI\Platform\Result\ResultInterface; @@ -28,23 +28,24 @@ */ final class TracingAgent implements AgentInterface { + use TracerAwareTrait; + public function __construct( private readonly AgentInterface $agent, + TracerProviderInterface $tracerProvider, + private readonly AgentAttributeProviderInterface $attributeProvider, ) { + $this->tracerProvider = $tracerProvider; } public function call(MessageBag $messages, array $options = []): ResultInterface { - $name = $this->agent->getName(); - - $span = Tracing::getTracer() - ->spanBuilder('invoke_agent '.$name) + $span = $this->getTracer() + ->spanBuilder('invoke_agent '.$this->agent->getName()) ->setSpanKind(SpanKind::KIND_INTERNAL) + ->setAttributes($this->attributeProvider->getAttributes($this->agent)) ->startSpan(); - $span->setAttribute(TraceAttributes::GEN_AI_OPERATION_NAME, TraceAttributeValues::GEN_AI_OPERATION_NAME_INVOKE_AGENT); - $span->setAttribute(TraceAttributes::GEN_AI_AGENT_NAME, $name); - try { $result = $this->agent->call($messages, $options); $span->setStatus(StatusCode::STATUS_OK); diff --git a/src/Tracing/AI/Toolbox/TracingToolbox.php b/src/Tracing/AI/Toolbox/TracingToolbox.php index aa40618..cbc1a63 100644 --- a/src/Tracing/AI/Toolbox/TracingToolbox.php +++ b/src/Tracing/AI/Toolbox/TracingToolbox.php @@ -9,11 +9,11 @@ namespace Instrumentation\Tracing\AI\Toolbox; -use Instrumentation\Tracing\Tracing; +use Instrumentation\Semantics\Attribute\ToolAttributeProviderInterface; +use Instrumentation\Tracing\TracerAwareTrait; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; -use OpenTelemetry\SemConv\TraceAttributes; -use OpenTelemetry\SemConv\TraceAttributeValues; +use OpenTelemetry\API\Trace\TracerProviderInterface; use Symfony\AI\Agent\Toolbox\ToolboxInterface; use Symfony\AI\Agent\Toolbox\ToolResult; use Symfony\AI\Platform\Result\ToolCall; @@ -27,9 +27,14 @@ */ final class TracingToolbox implements ToolboxInterface { + use TracerAwareTrait; + public function __construct( private readonly ToolboxInterface $toolbox, + TracerProviderInterface $tracerProvider, + private readonly ToolAttributeProviderInterface $attributeProvider, ) { + $this->tracerProvider = $tracerProvider; } /** @@ -42,15 +47,12 @@ public function getTools(): array public function execute(ToolCall $toolCall): ToolResult { - $span = Tracing::getTracer() + $span = $this->getTracer() ->spanBuilder('execute_tool '.$toolCall->getName()) ->setSpanKind(SpanKind::KIND_INTERNAL) + ->setAttributes($this->attributeProvider->getAttributes($toolCall)) ->startSpan(); - $span->setAttribute(TraceAttributes::GEN_AI_OPERATION_NAME, TraceAttributeValues::GEN_AI_OPERATION_NAME_EXECUTE_TOOL); - $span->setAttribute(TraceAttributes::GEN_AI_TOOL_NAME, $toolCall->getName()); - $span->setAttribute(TraceAttributes::GEN_AI_TOOL_CALL_ID, $toolCall->getId()); - try { $result = $this->toolbox->execute($toolCall); $span->setStatus(StatusCode::STATUS_OK); diff --git a/tests/Tracing/AI/Agent/TracingAgentTest.php b/tests/Tracing/AI/Agent/TracingAgentTest.php index b311863..44f8410 100644 --- a/tests/Tracing/AI/Agent/TracingAgentTest.php +++ b/tests/Tracing/AI/Agent/TracingAgentTest.php @@ -9,8 +9,8 @@ namespace Tests\Instrumentation\Tracing\AI\Agent; +use Instrumentation\Semantics\Attribute\AgentAttributeProvider; use Instrumentation\Tracing\AI\Agent\TracingAgent; -use Instrumentation\Tracing\Tracing; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; @@ -25,12 +25,12 @@ class TracingAgentTest extends TestCase { private \ArrayObject $spans; + private TracerProvider $tracerProvider; protected function setUp(): void { $this->spans = new \ArrayObject(); - $tracerProvider = new TracerProvider(new SimpleSpanProcessor(new InMemoryExporter($this->spans))); - Tracing::setProvider($tracerProvider); + $this->tracerProvider = new TracerProvider(new SimpleSpanProcessor(new InMemoryExporter($this->spans))); } public function testItCreatesAnInvokeAgentSpan(): void @@ -71,7 +71,7 @@ public function testItRecordsExceptionsAndEndsTheSpan(): void $inner->method('getName')->willReturn('my_agent'); $inner->method('call')->willThrowException($exception); - $agent = new TracingAgent($inner); + $agent = new TracingAgent($inner, $this->tracerProvider, new AgentAttributeProvider()); try { $agent->call(new MessageBag()); @@ -98,6 +98,6 @@ private function buildAgent(string $name, ResultInterface $result): TracingAgent $inner->method('getName')->willReturn($name); $inner->method('call')->willReturn($result); - return new TracingAgent($inner); + return new TracingAgent($inner, $this->tracerProvider, new AgentAttributeProvider()); } } diff --git a/tests/Tracing/AI/Toolbox/TracingToolboxTest.php b/tests/Tracing/AI/Toolbox/TracingToolboxTest.php index 7d55772..fbe4bd0 100644 --- a/tests/Tracing/AI/Toolbox/TracingToolboxTest.php +++ b/tests/Tracing/AI/Toolbox/TracingToolboxTest.php @@ -9,8 +9,8 @@ namespace Tests\Instrumentation\Tracing\AI\Toolbox; +use Instrumentation\Semantics\Attribute\ToolAttributeProvider; use Instrumentation\Tracing\AI\Toolbox\TracingToolbox; -use Instrumentation\Tracing\Tracing; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; @@ -24,12 +24,12 @@ class TracingToolboxTest extends TestCase { private \ArrayObject $spans; + private TracerProvider $tracerProvider; protected function setUp(): void { $this->spans = new \ArrayObject(); - $tracerProvider = new TracerProvider(new SimpleSpanProcessor(new InMemoryExporter($this->spans))); - Tracing::setProvider($tracerProvider); + $this->tracerProvider = new TracerProvider(new SimpleSpanProcessor(new InMemoryExporter($this->spans))); } public function testItCreatesAnExecuteToolSpan(): void @@ -75,7 +75,7 @@ public function testItRecordsExceptionsAndEndsTheSpan(): void $inner = $this->createMock(ToolboxInterface::class); $inner->method('execute')->willThrowException($exception); - $toolbox = new TracingToolbox($inner); + $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolAttributeProvider()); try { $toolbox->execute($toolCall); @@ -94,7 +94,7 @@ public function testItDelegatesGetTools(): void $inner = $this->createMock(ToolboxInterface::class); $inner->method('getTools')->willReturn([]); - $toolbox = new TracingToolbox($inner); + $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolAttributeProvider()); $this->assertSame([], $toolbox->getTools()); } @@ -104,6 +104,6 @@ private function buildToolbox(ToolCall $expected, ToolResult $result): TracingTo $inner = $this->createMock(ToolboxInterface::class); $inner->method('execute')->with($expected)->willReturn($result); - return new TracingToolbox($inner); + return new TracingToolbox($inner, $this->tracerProvider, new ToolAttributeProvider()); } } From 8c4350534d6d19fcaea44bc90a5b74cd9a189403 Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Fri, 19 Jun 2026 13:17:22 +0200 Subject: [PATCH 06/12] feat(Metrics): record gen_ai.client.token.usage histogram for AI platform calls Adds an opt-in (metrics.ai.enabled) MeteringPlatform decorator that records the OTel gen_ai semconv histogram gen_ai.client.token.usage, split by gen_ai.token.type (input/output) with gen_ai.system / gen_ai.request.model / gen_ai.operation.name attributes. Mirrors the tracing platform wiring: an abstract service in config/metrics/ai.php decorated onto ai.platform tagged services by AIPlatformMetricsCompilerPass. Metrics and tracing stay independent (separate toggle, separate decorator), each wrapping the result converter to read token usage; they compose when both are enabled. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../AIPlatformMetricsCompilerPass.php | 37 ++++++ src/DependencyInjection/Configuration.php | 6 + src/DependencyInjection/Extension.php | 9 ++ src/DependencyInjection/config/metrics/ai.php | 17 +++ src/Metrics/AI/Platform/MeteringPlatform.php | 49 ++++++++ .../AI/Platform/MeteringResultConverter.php | 49 ++++++++ .../Platform/MeteringTokenUsageExtractor.php | 65 +++++++++++ .../AI/Platform/MeteringPlatformTest.php | 110 ++++++++++++++++++ 8 files changed, 342 insertions(+) create mode 100644 src/DependencyInjection/CompilerPass/AIPlatformMetricsCompilerPass.php create mode 100644 src/DependencyInjection/config/metrics/ai.php create mode 100644 src/Metrics/AI/Platform/MeteringPlatform.php create mode 100644 src/Metrics/AI/Platform/MeteringResultConverter.php create mode 100644 src/Metrics/AI/Platform/MeteringTokenUsageExtractor.php create mode 100644 tests/Metrics/AI/Platform/MeteringPlatformTest.php diff --git a/src/DependencyInjection/CompilerPass/AIPlatformMetricsCompilerPass.php b/src/DependencyInjection/CompilerPass/AIPlatformMetricsCompilerPass.php new file mode 100644 index 0000000..709f630 --- /dev/null +++ b/src/DependencyInjection/CompilerPass/AIPlatformMetricsCompilerPass.php @@ -0,0 +1,37 @@ + + */ + +namespace Instrumentation\DependencyInjection\CompilerPass; + +use Instrumentation\Metrics\AI\Platform\MeteringPlatform; +use OpenTelemetry\API\Metrics\MeterProviderInterface; +use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; +use Symfony\Component\DependencyInjection\ContainerBuilder; +use Symfony\Component\DependencyInjection\Definition; +use Symfony\Component\DependencyInjection\Reference; + +final class AIPlatformMetricsCompilerPass implements CompilerPassInterface +{ + public function process(ContainerBuilder $container): void + { + foreach ($container->findTaggedServiceIds('ai.platform') as $platformId => $tags) { + $system = $tags[0]['name'] ?? $platformId; + + $definition = (new Definition(MeteringPlatform::class)) + ->setDecoratedService($platformId, priority: -512) + ->setArguments([ + new Reference('.inner'), + new Reference(MeterProviderInterface::class), + $system, + ]); + + $container->setDefinition('instrumentation.metrics.ai.platform.'.$system, $definition); + } + } +} diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 901dbef..f129aad 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -201,6 +201,12 @@ public function getConfigTreeBuilder(): TreeBuilder ->end() ->end() ->end() + ->arrayNode('ai') + ->addDefaultsIfNotSet() + ->children() + ->booleanNode('enabled')->defaultFalse()->end() + ->end() + ->end() ->end() ->end() diff --git a/src/DependencyInjection/Extension.php b/src/DependencyInjection/Extension.php index 9703e6b..432be89 100644 --- a/src/DependencyInjection/Extension.php +++ b/src/DependencyInjection/Extension.php @@ -10,8 +10,10 @@ namespace Instrumentation\DependencyInjection; use Instrumentation\DependencyInjection\CompilerPass\AIAgentTracingCompilerPass; +use Instrumentation\DependencyInjection\CompilerPass\AIPlatformMetricsCompilerPass; use Instrumentation\DependencyInjection\CompilerPass\AIPlatformTracingCompilerPass; use Instrumentation\DependencyInjection\CompilerPass\DoctrineTracingCompilerPass; +use Instrumentation\Metrics\AI\Platform\MeteringPlatform; use Instrumentation\Tracing\AI\Agent\TracingAgent; use Instrumentation\Tracing\AI\Platform\TracingPlatform; use Instrumentation\Tracing\Bridge\TraceUrlGenerator; @@ -83,6 +85,10 @@ public function process(ContainerBuilder $container): void if ($container->hasDefinition(TracingAgent::class)) { (new AIAgentTracingCompilerPass())->process($container); } + + if ($container->hasDefinition(MeteringPlatform::class)) { + (new AIPlatformMetricsCompilerPass())->process($container); + } } /** @@ -194,6 +200,9 @@ protected function loadMetrics(array $config, ContainerBuilder $container): void $container->setParameter('metrics.request.blacklist', $config['request']['blacklist']); $loader->load('request.php'); } + if ($this->isConfigEnabled($container, $config['ai'])) { + $loader->load('ai.php'); + } } private function getLoader(string $component, ContainerBuilder $container): PhpFileLoader diff --git a/src/DependencyInjection/config/metrics/ai.php b/src/DependencyInjection/config/metrics/ai.php new file mode 100644 index 0000000..2d5b3dc --- /dev/null +++ b/src/DependencyInjection/config/metrics/ai.php @@ -0,0 +1,17 @@ + + */ + +use Instrumentation\Metrics\AI\Platform\MeteringPlatform; +use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator; + +return static function (ContainerConfigurator $container) { + $container->services() + ->set(MeteringPlatform::class) + ->abstract(); +}; diff --git a/src/Metrics/AI/Platform/MeteringPlatform.php b/src/Metrics/AI/Platform/MeteringPlatform.php new file mode 100644 index 0000000..a263f50 --- /dev/null +++ b/src/Metrics/AI/Platform/MeteringPlatform.php @@ -0,0 +1,49 @@ + + */ + +namespace Instrumentation\Metrics\AI\Platform; + +use OpenTelemetry\API\Metrics\MeterProviderInterface; +use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; +use Symfony\AI\Platform\PlatformInterface; +use Symfony\AI\Platform\Result\DeferredResult; + +/** + * Decorates a platform to record the `gen_ai.client.token.usage` histogram + * (OTel gen_ai metric semconv) from the token usage of each model call. + * + * Independent from {@see \Instrumentation\Tracing\AI\Platform\TracingPlatform}: + * metrics and tracing are separate, separately-toggled concerns, so each wraps + * the result converter on its own. The two decorators compose when both enabled. + */ +final class MeteringPlatform implements PlatformInterface +{ + public function __construct( + private readonly PlatformInterface $platform, + private readonly MeterProviderInterface $meterProvider, + private readonly string $system, + ) { + } + + public function invoke(string $model, array|string|object $input, array $options = []): DeferredResult + { + $deferredResult = $this->platform->invoke($model, $input, $options); + + return new DeferredResult( + new MeteringResultConverter($deferredResult->getResultConverter(), $this->meterProvider, $this->system, $model), + $deferredResult->getRawResult(), + $options, + ); + } + + public function getModelCatalog(): ModelCatalogInterface + { + return $this->platform->getModelCatalog(); + } +} diff --git a/src/Metrics/AI/Platform/MeteringResultConverter.php b/src/Metrics/AI/Platform/MeteringResultConverter.php new file mode 100644 index 0000000..802fa64 --- /dev/null +++ b/src/Metrics/AI/Platform/MeteringResultConverter.php @@ -0,0 +1,49 @@ + + */ + +namespace Instrumentation\Metrics\AI\Platform; + +use OpenTelemetry\API\Metrics\MeterProviderInterface; +use Symfony\AI\Platform\Model; +use Symfony\AI\Platform\Result\RawResultInterface; +use Symfony\AI\Platform\Result\ResultInterface; +use Symfony\AI\Platform\ResultConverterInterface; +use Symfony\AI\Platform\TokenUsage\TokenUsageExtractorInterface; + +final class MeteringResultConverter implements ResultConverterInterface +{ + public function __construct( + private readonly ResultConverterInterface $inner, + private readonly MeterProviderInterface $meterProvider, + private readonly string $system, + private readonly string $model, + ) { + } + + public function supports(Model $model): bool + { + return $this->inner->supports($model); + } + + public function convert(RawResultInterface $result, array $options = []): ResultInterface + { + return $this->inner->convert($result, $options); + } + + public function getTokenUsageExtractor(): TokenUsageExtractorInterface|null + { + $inner = $this->inner->getTokenUsageExtractor(); + + if (null === $inner) { + return null; + } + + return new MeteringTokenUsageExtractor($inner, $this->meterProvider, $this->system, $this->model); + } +} diff --git a/src/Metrics/AI/Platform/MeteringTokenUsageExtractor.php b/src/Metrics/AI/Platform/MeteringTokenUsageExtractor.php new file mode 100644 index 0000000..c5dd80f --- /dev/null +++ b/src/Metrics/AI/Platform/MeteringTokenUsageExtractor.php @@ -0,0 +1,65 @@ + + */ + +namespace Instrumentation\Metrics\AI\Platform; + +use OpenTelemetry\API\Metrics\MeterProviderInterface; +use OpenTelemetry\SemConv\TraceAttributes; +use OpenTelemetry\SemConv\TraceAttributeValues; +use Symfony\AI\Platform\Result\RawResultInterface; +use Symfony\AI\Platform\TokenUsage\TokenUsageExtractorInterface; +use Symfony\AI\Platform\TokenUsage\TokenUsageInterface; + +/** + * Records the OTel gen_ai `gen_ai.client.token.usage` histogram, split by + * `gen_ai.token.type` (input/output), while delegating to the platform's own + * token usage extractor so the result chain is untouched. + */ +final class MeteringTokenUsageExtractor implements TokenUsageExtractorInterface +{ + public function __construct( + private readonly TokenUsageExtractorInterface $inner, + private readonly MeterProviderInterface $meterProvider, + private readonly string $system, + private readonly string $model, + ) { + } + + public function extract(RawResultInterface $rawResult, array $options = []): TokenUsageInterface|null + { + $tokenUsage = $this->inner->extract($rawResult, $options); + + if (null === $tokenUsage) { + return null; + } + + $histogram = $this->meterProvider + ->getMeter('instrumentation') + ->createHistogram('gen_ai.client.token.usage', '{token}', 'Number of tokens used by the model'); + + $attributes = [ + TraceAttributes::GEN_AI_OPERATION_NAME => TraceAttributeValues::GEN_AI_OPERATION_NAME_CHAT, + TraceAttributes::GEN_AI_SYSTEM => $this->system, + TraceAttributes::GEN_AI_REQUEST_MODEL => $this->model, + ]; + + if (null !== $tokenUsage->getPromptTokens()) { + $histogram->record($tokenUsage->getPromptTokens(), $attributes + [ + TraceAttributes::GEN_AI_TOKEN_TYPE => TraceAttributeValues::GEN_AI_TOKEN_TYPE_INPUT, + ]); + } + if (null !== $tokenUsage->getCompletionTokens()) { + $histogram->record($tokenUsage->getCompletionTokens(), $attributes + [ + TraceAttributes::GEN_AI_TOKEN_TYPE => TraceAttributeValues::GEN_AI_TOKEN_TYPE_OUTPUT, + ]); + } + + return $tokenUsage; + } +} diff --git a/tests/Metrics/AI/Platform/MeteringPlatformTest.php b/tests/Metrics/AI/Platform/MeteringPlatformTest.php new file mode 100644 index 0000000..8dbe159 --- /dev/null +++ b/tests/Metrics/AI/Platform/MeteringPlatformTest.php @@ -0,0 +1,110 @@ + + */ + +namespace Tests\Instrumentation\Metrics\AI\Platform; + +use Instrumentation\Metrics\AI\Platform\MeteringPlatform; +use OpenTelemetry\API\Metrics\MeterProviderInterface; +use OpenTelemetry\SDK\Metrics\Data\Histogram; +use OpenTelemetry\SDK\Metrics\MeterProviderBuilder; +use OpenTelemetry\SDK\Metrics\MetricExporter\InMemoryExporter; +use OpenTelemetry\SDK\Metrics\MetricReader\ExportingReader; +use PHPUnit\Framework\TestCase; +use Symfony\AI\Platform\PlatformInterface; +use Symfony\AI\Platform\Result\DeferredResult; +use Symfony\AI\Platform\Result\InMemoryRawResult; +use Symfony\AI\Platform\Result\TextResult; +use Symfony\AI\Platform\ResultConverterInterface; +use Symfony\AI\Platform\TokenUsage\TokenUsage; +use Symfony\AI\Platform\TokenUsage\TokenUsageExtractorInterface; + +class MeteringPlatformTest extends TestCase +{ + private InMemoryExporter $exporter; + private ExportingReader $reader; + private MeterProviderInterface $meterProvider; + + protected function setUp(): void + { + $this->exporter = new InMemoryExporter(); + $this->reader = new ExportingReader($this->exporter); + $this->meterProvider = (new MeterProviderBuilder())->addReader($this->reader)->build(); + } + + public function testItRecordsTheTokenUsageHistogram(): void + { + $extractor = $this->createMock(TokenUsageExtractorInterface::class); + $extractor->method('extract')->willReturn(new TokenUsage(promptTokens: 150, completionTokens: 42)); + + $platform = $this->buildPlatform('openai', $extractor); + $platform->invoke('gpt-4o', 'Hello')->asText(); + + $dataPoints = $this->collectHistogramDataPoints('gen_ai.client.token.usage'); + $this->assertCount(2, $dataPoints, 'one data point per token type'); + + $byType = []; + foreach ($dataPoints as $point) { + $attributes = $point->attributes->toArray(); + $byType[$attributes['gen_ai.token.type']] = ['sum' => $point->sum, 'attributes' => $attributes]; + } + + $this->assertSame(150, $byType['input']['sum']); + $this->assertSame(42, $byType['output']['sum']); + $this->assertSame('chat', $byType['input']['attributes']['gen_ai.operation.name']); + $this->assertSame('openai', $byType['input']['attributes']['gen_ai.system']); + $this->assertSame('gpt-4o', $byType['input']['attributes']['gen_ai.request.model']); + } + + public function testItRecordsNothingWhenNoExtractorIsAvailable(): void + { + $platform = $this->buildPlatform('openai', null); + $platform->invoke('gpt-4o', 'Hello')->asText(); + + $this->assertSame([], $this->collectHistogramDataPoints('gen_ai.client.token.usage')); + } + + public function testItRecordsNothingWhenExtractorReturnsNull(): void + { + $extractor = $this->createMock(TokenUsageExtractorInterface::class); + $extractor->method('extract')->willReturn(null); + + $platform = $this->buildPlatform('openai', $extractor); + $platform->invoke('gpt-4o', 'Hello')->asText(); + + $this->assertSame([], $this->collectHistogramDataPoints('gen_ai.client.token.usage')); + } + + /** + * @return list<\OpenTelemetry\SDK\Metrics\Data\HistogramDataPoint> + */ + private function collectHistogramDataPoints(string $name): array + { + $this->reader->collect(); + + foreach ($this->exporter->collect(true) as $metric) { + if ($metric->name === $name && $metric->data instanceof Histogram) { + return iterator_to_array($metric->data->dataPoints, false); + } + } + + return []; + } + + private function buildPlatform(string $system, TokenUsageExtractorInterface|null $extractor): MeteringPlatform + { + $converter = $this->createMock(ResultConverterInterface::class); + $converter->method('convert')->willReturn(new TextResult('Hello')); + $converter->method('getTokenUsageExtractor')->willReturn($extractor); + + $inner = $this->createMock(PlatformInterface::class); + $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult([]))); + + return new MeteringPlatform($inner, $this->meterProvider, $system); + } +} From dbf532824367a9b64b7277adb368d0d555035f03 Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Fri, 19 Jun 2026 13:27:26 +0200 Subject: [PATCH 07/12] refactor(Tracing): align AI platform tracing with bundle conventions - Inject TracerProviderInterface (via TracerAwareTrait) into TracingPlatform instead of the static Tracing facade. - Extract span attributes into a swappable PlatformAttributeProvider behind an interface registered in semconv.php, consistent with the agent/tool/request/ doctrine providers. - Use OpenTelemetry\SemConv constants for the gen_ai.* attribute names (including the token usage attributes in TracingTokenUsageExtractor). - Drop the nullable from TracingResultConverter::getTokenUsageExtractor(): it always returns a wrapper, so the previous ?type tripped PHPStan. Also resolves the pre-existing PHPStan (return.unusedType) and cs-fixer failures on the base branch. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../AIPlatformTracingCompilerPass.php | 9 ++++++- .../config/semconv/semconv.php | 3 +++ .../Attribute/PlatformAttributeProvider.php | 24 +++++++++++++++++++ .../PlatformAttributeProviderInterface.php | 18 ++++++++++++++ src/Tracing/AI/Platform/TracingPlatform.php | 17 +++++++------ .../AI/Platform/TracingResultConverter.php | 2 +- .../Platform/TracingTokenUsageExtractor.php | 9 +++---- .../AI/Platform/TracingPlatformTest.php | 14 +++++------ 8 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 src/Semantics/Attribute/PlatformAttributeProvider.php create mode 100644 src/Semantics/Attribute/PlatformAttributeProviderInterface.php diff --git a/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php index 64cfd41..7ea9389 100644 --- a/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php +++ b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php @@ -9,7 +9,9 @@ namespace Instrumentation\DependencyInjection\CompilerPass; +use Instrumentation\Semantics\Attribute\PlatformAttributeProviderInterface; use Instrumentation\Tracing\AI\Platform\TracingPlatform; +use OpenTelemetry\API\Trace\TracerProviderInterface; use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\Definition; @@ -26,7 +28,12 @@ public function process(ContainerBuilder $container): void // so the span covers the full call including any inner decorators (retry, cache, etc.). $definition = (new Definition(TracingPlatform::class)) ->setDecoratedService($platformId, priority: -512) - ->setArguments([new Reference('.inner'), $system]); + ->setArguments([ + new Reference('.inner'), + new Reference(TracerProviderInterface::class), + new Reference(PlatformAttributeProviderInterface::class), + $system, + ]); $container->setDefinition('instrumentation.tracing.ai.platform.'.$system, $definition); } diff --git a/src/DependencyInjection/config/semconv/semconv.php b/src/DependencyInjection/config/semconv/semconv.php index b5ecb53..dbe7c7b 100644 --- a/src/DependencyInjection/config/semconv/semconv.php +++ b/src/DependencyInjection/config/semconv/semconv.php @@ -15,6 +15,8 @@ use Instrumentation\Semantics\Attribute\DoctrineConnectionAttributeProviderInterface; use Instrumentation\Semantics\Attribute\MessageAttributeProvider; use Instrumentation\Semantics\Attribute\MessageAttributeProviderInterface; +use Instrumentation\Semantics\Attribute\PlatformAttributeProvider; +use Instrumentation\Semantics\Attribute\PlatformAttributeProviderInterface; use Instrumentation\Semantics\Attribute\ServerRequestAttributeProvider; use Instrumentation\Semantics\Attribute\ServerRequestAttributeProviderInterface; use Instrumentation\Semantics\Attribute\ServerResponseAttributeProvider; @@ -63,6 +65,7 @@ param('tracing.http.attributes.request_headers'), ]) ->set(DoctrineConnectionAttributeProviderInterface::class, DoctrineConnectionAttributeProvider::class) + ->set(PlatformAttributeProviderInterface::class, PlatformAttributeProvider::class) ->set(AgentAttributeProviderInterface::class, AgentAttributeProvider::class) ->set(ToolAttributeProviderInterface::class, ToolAttributeProvider::class) diff --git a/src/Semantics/Attribute/PlatformAttributeProvider.php b/src/Semantics/Attribute/PlatformAttributeProvider.php new file mode 100644 index 0000000..ceceb6a --- /dev/null +++ b/src/Semantics/Attribute/PlatformAttributeProvider.php @@ -0,0 +1,24 @@ + + */ + +namespace Instrumentation\Semantics\Attribute; + +use OpenTelemetry\SemConv\TraceAttributes; + +class PlatformAttributeProvider implements PlatformAttributeProviderInterface +{ + public function getAttributes(string $system, string $model, string $operationName): array + { + return [ + TraceAttributes::GEN_AI_OPERATION_NAME => $operationName, + TraceAttributes::GEN_AI_SYSTEM => $system, + TraceAttributes::GEN_AI_REQUEST_MODEL => $model, + ]; + } +} diff --git a/src/Semantics/Attribute/PlatformAttributeProviderInterface.php b/src/Semantics/Attribute/PlatformAttributeProviderInterface.php new file mode 100644 index 0000000..228f62d --- /dev/null +++ b/src/Semantics/Attribute/PlatformAttributeProviderInterface.php @@ -0,0 +1,18 @@ + + */ + +namespace Instrumentation\Semantics\Attribute; + +interface PlatformAttributeProviderInterface +{ + /** + * @return array + */ + public function getAttributes(string $system, string $model, string $operationName): array; +} diff --git a/src/Tracing/AI/Platform/TracingPlatform.php b/src/Tracing/AI/Platform/TracingPlatform.php index 99a964d..bfe3e07 100644 --- a/src/Tracing/AI/Platform/TracingPlatform.php +++ b/src/Tracing/AI/Platform/TracingPlatform.php @@ -9,34 +9,38 @@ namespace Instrumentation\Tracing\AI\Platform; -use Instrumentation\Tracing\Tracing; +use Instrumentation\Semantics\Attribute\PlatformAttributeProviderInterface; +use Instrumentation\Tracing\TracerAwareTrait; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; +use OpenTelemetry\API\Trace\TracerProviderInterface; use Symfony\AI\Platform\ModelCatalog\ModelCatalogInterface; use Symfony\AI\Platform\PlatformInterface; use Symfony\AI\Platform\Result\DeferredResult; final class TracingPlatform implements PlatformInterface { + use TracerAwareTrait; + public function __construct( private readonly PlatformInterface $platform, + TracerProviderInterface $tracerProvider, + private readonly PlatformAttributeProviderInterface $attributeProvider, private readonly string $system, ) { + $this->tracerProvider = $tracerProvider; } public function invoke(string $model, array|string|object $input, array $options = []): DeferredResult { $operationName = $options['extra']['operation_name'] ?? 'symfony_ai'; - $span = Tracing::getTracer() + $span = $this->getTracer() ->spanBuilder($operationName) ->setSpanKind(SpanKind::KIND_CLIENT) + ->setAttributes($this->attributeProvider->getAttributes($this->system, $model, $operationName)) ->startSpan(); - $span->setAttribute('gen_ai.operation.name', $operationName); - $span->setAttribute('gen_ai.system', $this->system); - $span->setAttribute('gen_ai.request.model', $model); - try { $deferredResult = $this->platform->invoke($model, $input, $options); } catch (\Throwable $e) { @@ -51,7 +55,6 @@ public function invoke(string $model, array|string|object $input, array $options $deferredResult->getRawResult(), $options, ); - } public function getModelCatalog(): ModelCatalogInterface diff --git a/src/Tracing/AI/Platform/TracingResultConverter.php b/src/Tracing/AI/Platform/TracingResultConverter.php index ad3b7ca..e07b2a3 100644 --- a/src/Tracing/AI/Platform/TracingResultConverter.php +++ b/src/Tracing/AI/Platform/TracingResultConverter.php @@ -47,7 +47,7 @@ public function convert(RawResultInterface $result, array $options = []): Result // On success, span is ended by TracingTokenUsageExtractor after token extraction. } - public function getTokenUsageExtractor(): ?TokenUsageExtractorInterface + public function getTokenUsageExtractor(): TokenUsageExtractorInterface { return new TracingTokenUsageExtractor($this->inner->getTokenUsageExtractor(), $this->span); } diff --git a/src/Tracing/AI/Platform/TracingTokenUsageExtractor.php b/src/Tracing/AI/Platform/TracingTokenUsageExtractor.php index 5693478..ee16d19 100644 --- a/src/Tracing/AI/Platform/TracingTokenUsageExtractor.php +++ b/src/Tracing/AI/Platform/TracingTokenUsageExtractor.php @@ -10,6 +10,7 @@ namespace Instrumentation\Tracing\AI\Platform; use OpenTelemetry\API\Trace\SpanInterface; +use OpenTelemetry\SemConv\TraceAttributes; use Symfony\AI\Platform\Result\RawResultInterface; use Symfony\AI\Platform\TokenUsage\TokenUsageExtractorInterface; use Symfony\AI\Platform\TokenUsage\TokenUsageInterface; @@ -17,12 +18,12 @@ final class TracingTokenUsageExtractor implements TokenUsageExtractorInterface { public function __construct( - private readonly ?TokenUsageExtractorInterface $inner, + private readonly TokenUsageExtractorInterface|null $inner, private readonly SpanInterface $span, ) { } - public function extract(RawResultInterface $rawResult, array $options = []): ?TokenUsageInterface + public function extract(RawResultInterface $rawResult, array $options = []): TokenUsageInterface|null { try { if (null === $this->inner) { @@ -36,10 +37,10 @@ public function extract(RawResultInterface $rawResult, array $options = []): ?To } if (null !== $tokenUsage->getPromptTokens()) { - $this->span->setAttribute('gen_ai.usage.input_tokens', $tokenUsage->getPromptTokens()); + $this->span->setAttribute(TraceAttributes::GEN_AI_USAGE_INPUT_TOKENS, $tokenUsage->getPromptTokens()); } if (null !== $tokenUsage->getCompletionTokens()) { - $this->span->setAttribute('gen_ai.usage.output_tokens', $tokenUsage->getCompletionTokens()); + $this->span->setAttribute(TraceAttributes::GEN_AI_USAGE_OUTPUT_TOKENS, $tokenUsage->getCompletionTokens()); } return $tokenUsage; diff --git a/tests/Tracing/AI/Platform/TracingPlatformTest.php b/tests/Tracing/AI/Platform/TracingPlatformTest.php index 1d955b6..3bd440b 100644 --- a/tests/Tracing/AI/Platform/TracingPlatformTest.php +++ b/tests/Tracing/AI/Platform/TracingPlatformTest.php @@ -9,8 +9,8 @@ namespace Tests\Instrumentation\Tracing\AI\Platform; +use Instrumentation\Semantics\Attribute\PlatformAttributeProvider; use Instrumentation\Tracing\AI\Platform\TracingPlatform; -use Instrumentation\Tracing\Tracing; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; @@ -28,12 +28,12 @@ class TracingPlatformTest extends TestCase { private \ArrayObject $spans; + private TracerProvider $tracerProvider; protected function setUp(): void { $this->spans = new \ArrayObject(); - $tracerProvider = new TracerProvider(new SimpleSpanProcessor(new InMemoryExporter($this->spans))); - Tracing::setProvider($tracerProvider); + $this->tracerProvider = new TracerProvider(new SimpleSpanProcessor(new InMemoryExporter($this->spans))); } public function testItCreatesASpanWithCorrectNameAndKind(): void @@ -125,7 +125,7 @@ public function testItEndsSpanWithErrorStatusWhenPlatformThrows(): void $inner = $this->createMock(PlatformInterface::class); $inner->method('invoke')->willThrowException(new \RuntimeException('API unreachable')); - $platform = new TracingPlatform($inner, 'gemini'); + $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformAttributeProvider(), 'gemini'); try { $platform->invoke('gemini-2.5-pro', 'Hello'); @@ -146,7 +146,7 @@ public function testItEndsSpanWithErrorStatusWhenConverterThrows(): void $inner = $this->createMock(PlatformInterface::class); $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult())); - $platform = new TracingPlatform($inner, 'gemini'); + $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformAttributeProvider(), 'gemini'); try { $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); @@ -158,7 +158,7 @@ public function testItEndsSpanWithErrorStatusWhenConverterThrows(): void $this->assertSame(StatusCode::STATUS_ERROR, $this->spans[0]->getStatus()->getCode()); } - private function buildPlatform(string $system, ?TokenUsageExtractorInterface $extractor = null): TracingPlatform + private function buildPlatform(string $system, TokenUsageExtractorInterface|null $extractor = null): TracingPlatform { $converter = $this->createMock(ResultConverterInterface::class); $converter->method('convert')->willReturn(new TextResult('response')); @@ -167,6 +167,6 @@ private function buildPlatform(string $system, ?TokenUsageExtractorInterface $ex $inner = $this->createMock(PlatformInterface::class); $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult())); - return new TracingPlatform($inner, $system); + return new TracingPlatform($inner, $this->tracerProvider, new PlatformAttributeProvider(), $system); } } From 3f090b3180b630b623745d651447c5df186fe80e Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Fri, 19 Jun 2026 13:59:24 +0200 Subject: [PATCH 08/12] refactor(Tracing): add span-leak backstop and swappable operation-name resolvers for AI - TracingResultConverter gets a __destruct backstop that ends the platform span if the deferred result is never consumed (Span::end is idempotent, so it is a no-op on the normal path). Mirrors HttpClient's TracedResponse and addresses the original review's span-leak concern. - Span naming moves out of the decorators into swappable Platform/Agent/Tool OperationNameResolver services (registered in semconv.php), matching the request/command/message/httpclient resolvers. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../AIAgentTracingCompilerPass.php | 4 ++++ .../AIPlatformTracingCompilerPass.php | 2 ++ .../config/semconv/semconv.php | 9 +++++++++ .../AgentOperationNameResolver.php | 20 +++++++++++++++++++ .../AgentOperationNameResolverInterface.php | 20 +++++++++++++++++++ .../PlatformOperationNameResolver.php | 20 +++++++++++++++++++ ...PlatformOperationNameResolverInterface.php | 20 +++++++++++++++++++ .../ToolOperationNameResolver.php | 20 +++++++++++++++++++ .../ToolOperationNameResolverInterface.php | 20 +++++++++++++++++++ src/Tracing/AI/Agent/TracingAgent.php | 4 +++- src/Tracing/AI/Platform/TracingPlatform.php | 4 +++- .../AI/Platform/TracingResultConverter.php | 11 ++++++++++ src/Tracing/AI/Toolbox/TracingToolbox.php | 4 +++- tests/Tracing/AI/Agent/TracingAgentTest.php | 5 +++-- .../AI/Platform/TracingPlatformTest.php | 7 ++++--- .../Tracing/AI/Toolbox/TracingToolboxTest.php | 7 ++++--- 16 files changed, 166 insertions(+), 11 deletions(-) create mode 100644 src/Semantics/OperationName/AgentOperationNameResolver.php create mode 100644 src/Semantics/OperationName/AgentOperationNameResolverInterface.php create mode 100644 src/Semantics/OperationName/PlatformOperationNameResolver.php create mode 100644 src/Semantics/OperationName/PlatformOperationNameResolverInterface.php create mode 100644 src/Semantics/OperationName/ToolOperationNameResolver.php create mode 100644 src/Semantics/OperationName/ToolOperationNameResolverInterface.php diff --git a/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php b/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php index a0bd160..8a78506 100644 --- a/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php +++ b/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php @@ -11,6 +11,8 @@ use Instrumentation\Semantics\Attribute\AgentAttributeProviderInterface; use Instrumentation\Semantics\Attribute\ToolAttributeProviderInterface; +use Instrumentation\Semantics\OperationName\AgentOperationNameResolverInterface; +use Instrumentation\Semantics\OperationName\ToolOperationNameResolverInterface; use Instrumentation\Tracing\AI\Agent\TracingAgent; use Instrumentation\Tracing\AI\Toolbox\TracingToolbox; use OpenTelemetry\API\Trace\TracerProviderInterface; @@ -34,6 +36,7 @@ public function process(ContainerBuilder $container): void ->setArguments([ new Reference('.inner'), new Reference(TracerProviderInterface::class), + new Reference(AgentOperationNameResolverInterface::class), new Reference(AgentAttributeProviderInterface::class), ]); @@ -48,6 +51,7 @@ public function process(ContainerBuilder $container): void ->setArguments([ new Reference('.inner'), new Reference(TracerProviderInterface::class), + new Reference(ToolOperationNameResolverInterface::class), new Reference(ToolAttributeProviderInterface::class), ]); diff --git a/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php index 7ea9389..1039d8a 100644 --- a/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php +++ b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php @@ -10,6 +10,7 @@ namespace Instrumentation\DependencyInjection\CompilerPass; use Instrumentation\Semantics\Attribute\PlatformAttributeProviderInterface; +use Instrumentation\Semantics\OperationName\PlatformOperationNameResolverInterface; use Instrumentation\Tracing\AI\Platform\TracingPlatform; use OpenTelemetry\API\Trace\TracerProviderInterface; use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; @@ -31,6 +32,7 @@ public function process(ContainerBuilder $container): void ->setArguments([ new Reference('.inner'), new Reference(TracerProviderInterface::class), + new Reference(PlatformOperationNameResolverInterface::class), new Reference(PlatformAttributeProviderInterface::class), $system, ]); diff --git a/src/DependencyInjection/config/semconv/semconv.php b/src/DependencyInjection/config/semconv/semconv.php index dbe7c7b..2f20985 100644 --- a/src/DependencyInjection/config/semconv/semconv.php +++ b/src/DependencyInjection/config/semconv/semconv.php @@ -23,17 +23,23 @@ use Instrumentation\Semantics\Attribute\ServerResponseAttributeProviderInterface; use Instrumentation\Semantics\Attribute\ToolAttributeProvider; use Instrumentation\Semantics\Attribute\ToolAttributeProviderInterface; +use Instrumentation\Semantics\OperationName\AgentOperationNameResolver; +use Instrumentation\Semantics\OperationName\AgentOperationNameResolverInterface; use Instrumentation\Semantics\OperationName\ClientRequestOperationNameResolver; use Instrumentation\Semantics\OperationName\ClientRequestOperationNameResolverInterface; use Instrumentation\Semantics\OperationName\CommandOperationNameResolver; use Instrumentation\Semantics\OperationName\CommandOperationNameResolverInterface; use Instrumentation\Semantics\OperationName\MessageOperationNameResolver; use Instrumentation\Semantics\OperationName\MessageOperationNameResolverInterface; +use Instrumentation\Semantics\OperationName\PlatformOperationNameResolver; +use Instrumentation\Semantics\OperationName\PlatformOperationNameResolverInterface; use Instrumentation\Semantics\OperationName\RoutePath\RouteCacheWarmer; use Instrumentation\Semantics\OperationName\RoutePath\RoutePathResolver; use Instrumentation\Semantics\OperationName\RoutePath\RoutePathResolverInterface; use Instrumentation\Semantics\OperationName\RoutePathServerRequestOperationNameResolver; use Instrumentation\Semantics\OperationName\ServerRequestOperationNameResolverInterface; +use Instrumentation\Semantics\OperationName\ToolOperationNameResolver; +use Instrumentation\Semantics\OperationName\ToolOperationNameResolverInterface; use Instrumentation\Semantics\ResourceInfoProvider; use Instrumentation\Semantics\ResourceInfoProviderInterface; use OpenTelemetry\SDK\Resource\ResourceInfo; @@ -72,6 +78,9 @@ ->set(ClientRequestOperationNameResolverInterface::class, ClientRequestOperationNameResolver::class) ->set(MessageOperationNameResolverInterface::class, MessageOperationNameResolver::class) ->set(CommandOperationNameResolverInterface::class, CommandOperationNameResolver::class) + ->set(PlatformOperationNameResolverInterface::class, PlatformOperationNameResolver::class) + ->set(AgentOperationNameResolverInterface::class, AgentOperationNameResolver::class) + ->set(ToolOperationNameResolverInterface::class, ToolOperationNameResolver::class) ->set(ServerRequestOperationNameResolverInterface::class, RoutePathServerRequestOperationNameResolver::class) ->args([ service(RoutePathResolverInterface::class), diff --git a/src/Semantics/OperationName/AgentOperationNameResolver.php b/src/Semantics/OperationName/AgentOperationNameResolver.php new file mode 100644 index 0000000..5a530a9 --- /dev/null +++ b/src/Semantics/OperationName/AgentOperationNameResolver.php @@ -0,0 +1,20 @@ + + */ + +namespace Instrumentation\Semantics\OperationName; + +use Symfony\AI\Agent\AgentInterface; + +class AgentOperationNameResolver implements AgentOperationNameResolverInterface +{ + public function getOperationName(AgentInterface $agent): string + { + return \sprintf('invoke_agent %s', $agent->getName()); + } +} diff --git a/src/Semantics/OperationName/AgentOperationNameResolverInterface.php b/src/Semantics/OperationName/AgentOperationNameResolverInterface.php new file mode 100644 index 0000000..93efca2 --- /dev/null +++ b/src/Semantics/OperationName/AgentOperationNameResolverInterface.php @@ -0,0 +1,20 @@ + + */ + +namespace Instrumentation\Semantics\OperationName; + +use Symfony\AI\Agent\AgentInterface; + +interface AgentOperationNameResolverInterface +{ + /** + * @return string&non-empty-string + */ + public function getOperationName(AgentInterface $agent): string; +} diff --git a/src/Semantics/OperationName/PlatformOperationNameResolver.php b/src/Semantics/OperationName/PlatformOperationNameResolver.php new file mode 100644 index 0000000..0b74dbb --- /dev/null +++ b/src/Semantics/OperationName/PlatformOperationNameResolver.php @@ -0,0 +1,20 @@ + + */ + +namespace Instrumentation\Semantics\OperationName; + +class PlatformOperationNameResolver implements PlatformOperationNameResolverInterface +{ + public function getOperationName(string $model, array $options): string + { + $operationName = $options['extra']['operation_name'] ?? null; + + return \is_string($operationName) && '' !== $operationName ? $operationName : 'symfony_ai'; + } +} diff --git a/src/Semantics/OperationName/PlatformOperationNameResolverInterface.php b/src/Semantics/OperationName/PlatformOperationNameResolverInterface.php new file mode 100644 index 0000000..b209807 --- /dev/null +++ b/src/Semantics/OperationName/PlatformOperationNameResolverInterface.php @@ -0,0 +1,20 @@ + + */ + +namespace Instrumentation\Semantics\OperationName; + +interface PlatformOperationNameResolverInterface +{ + /** + * @param array $options + * + * @return string&non-empty-string + */ + public function getOperationName(string $model, array $options): string; +} diff --git a/src/Semantics/OperationName/ToolOperationNameResolver.php b/src/Semantics/OperationName/ToolOperationNameResolver.php new file mode 100644 index 0000000..3450c9d --- /dev/null +++ b/src/Semantics/OperationName/ToolOperationNameResolver.php @@ -0,0 +1,20 @@ + + */ + +namespace Instrumentation\Semantics\OperationName; + +use Symfony\AI\Platform\Result\ToolCall; + +class ToolOperationNameResolver implements ToolOperationNameResolverInterface +{ + public function getOperationName(ToolCall $toolCall): string + { + return \sprintf('execute_tool %s', $toolCall->getName()); + } +} diff --git a/src/Semantics/OperationName/ToolOperationNameResolverInterface.php b/src/Semantics/OperationName/ToolOperationNameResolverInterface.php new file mode 100644 index 0000000..f9d356b --- /dev/null +++ b/src/Semantics/OperationName/ToolOperationNameResolverInterface.php @@ -0,0 +1,20 @@ + + */ + +namespace Instrumentation\Semantics\OperationName; + +use Symfony\AI\Platform\Result\ToolCall; + +interface ToolOperationNameResolverInterface +{ + /** + * @return string&non-empty-string + */ + public function getOperationName(ToolCall $toolCall): string; +} diff --git a/src/Tracing/AI/Agent/TracingAgent.php b/src/Tracing/AI/Agent/TracingAgent.php index daf557d..0815cf4 100644 --- a/src/Tracing/AI/Agent/TracingAgent.php +++ b/src/Tracing/AI/Agent/TracingAgent.php @@ -10,6 +10,7 @@ namespace Instrumentation\Tracing\AI\Agent; use Instrumentation\Semantics\Attribute\AgentAttributeProviderInterface; +use Instrumentation\Semantics\OperationName\AgentOperationNameResolverInterface; use Instrumentation\Tracing\TracerAwareTrait; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -33,6 +34,7 @@ final class TracingAgent implements AgentInterface public function __construct( private readonly AgentInterface $agent, TracerProviderInterface $tracerProvider, + private readonly AgentOperationNameResolverInterface $operationNameResolver, private readonly AgentAttributeProviderInterface $attributeProvider, ) { $this->tracerProvider = $tracerProvider; @@ -41,7 +43,7 @@ public function __construct( public function call(MessageBag $messages, array $options = []): ResultInterface { $span = $this->getTracer() - ->spanBuilder('invoke_agent '.$this->agent->getName()) + ->spanBuilder($this->operationNameResolver->getOperationName($this->agent)) ->setSpanKind(SpanKind::KIND_INTERNAL) ->setAttributes($this->attributeProvider->getAttributes($this->agent)) ->startSpan(); diff --git a/src/Tracing/AI/Platform/TracingPlatform.php b/src/Tracing/AI/Platform/TracingPlatform.php index bfe3e07..298c0da 100644 --- a/src/Tracing/AI/Platform/TracingPlatform.php +++ b/src/Tracing/AI/Platform/TracingPlatform.php @@ -10,6 +10,7 @@ namespace Instrumentation\Tracing\AI\Platform; use Instrumentation\Semantics\Attribute\PlatformAttributeProviderInterface; +use Instrumentation\Semantics\OperationName\PlatformOperationNameResolverInterface; use Instrumentation\Tracing\TracerAwareTrait; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -25,6 +26,7 @@ final class TracingPlatform implements PlatformInterface public function __construct( private readonly PlatformInterface $platform, TracerProviderInterface $tracerProvider, + private readonly PlatformOperationNameResolverInterface $operationNameResolver, private readonly PlatformAttributeProviderInterface $attributeProvider, private readonly string $system, ) { @@ -33,7 +35,7 @@ public function __construct( public function invoke(string $model, array|string|object $input, array $options = []): DeferredResult { - $operationName = $options['extra']['operation_name'] ?? 'symfony_ai'; + $operationName = $this->operationNameResolver->getOperationName($model, $options); $span = $this->getTracer() ->spanBuilder($operationName) diff --git a/src/Tracing/AI/Platform/TracingResultConverter.php b/src/Tracing/AI/Platform/TracingResultConverter.php index e07b2a3..0ff8054 100644 --- a/src/Tracing/AI/Platform/TracingResultConverter.php +++ b/src/Tracing/AI/Platform/TracingResultConverter.php @@ -51,4 +51,15 @@ public function getTokenUsageExtractor(): TokenUsageExtractorInterface { return new TracingTokenUsageExtractor($this->inner->getTokenUsageExtractor(), $this->span); } + + /** + * Backstop: if the deferred result is never consumed (so neither convert() + * nor the token usage extractor runs), end the span here so it is not leaked. + * Span::end() is idempotent, so this is a no-op on the normal path. Mirrors + * the destructor in the HttpClient TracedResponse. + */ + public function __destruct() + { + $this->span->end(); + } } diff --git a/src/Tracing/AI/Toolbox/TracingToolbox.php b/src/Tracing/AI/Toolbox/TracingToolbox.php index cbc1a63..14d0f22 100644 --- a/src/Tracing/AI/Toolbox/TracingToolbox.php +++ b/src/Tracing/AI/Toolbox/TracingToolbox.php @@ -10,6 +10,7 @@ namespace Instrumentation\Tracing\AI\Toolbox; use Instrumentation\Semantics\Attribute\ToolAttributeProviderInterface; +use Instrumentation\Semantics\OperationName\ToolOperationNameResolverInterface; use Instrumentation\Tracing\TracerAwareTrait; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -32,6 +33,7 @@ final class TracingToolbox implements ToolboxInterface public function __construct( private readonly ToolboxInterface $toolbox, TracerProviderInterface $tracerProvider, + private readonly ToolOperationNameResolverInterface $operationNameResolver, private readonly ToolAttributeProviderInterface $attributeProvider, ) { $this->tracerProvider = $tracerProvider; @@ -48,7 +50,7 @@ public function getTools(): array public function execute(ToolCall $toolCall): ToolResult { $span = $this->getTracer() - ->spanBuilder('execute_tool '.$toolCall->getName()) + ->spanBuilder($this->operationNameResolver->getOperationName($toolCall)) ->setSpanKind(SpanKind::KIND_INTERNAL) ->setAttributes($this->attributeProvider->getAttributes($toolCall)) ->startSpan(); diff --git a/tests/Tracing/AI/Agent/TracingAgentTest.php b/tests/Tracing/AI/Agent/TracingAgentTest.php index 44f8410..d125660 100644 --- a/tests/Tracing/AI/Agent/TracingAgentTest.php +++ b/tests/Tracing/AI/Agent/TracingAgentTest.php @@ -10,6 +10,7 @@ namespace Tests\Instrumentation\Tracing\AI\Agent; use Instrumentation\Semantics\Attribute\AgentAttributeProvider; +use Instrumentation\Semantics\OperationName\AgentOperationNameResolver; use Instrumentation\Tracing\AI\Agent\TracingAgent; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -71,7 +72,7 @@ public function testItRecordsExceptionsAndEndsTheSpan(): void $inner->method('getName')->willReturn('my_agent'); $inner->method('call')->willThrowException($exception); - $agent = new TracingAgent($inner, $this->tracerProvider, new AgentAttributeProvider()); + $agent = new TracingAgent($inner, $this->tracerProvider, new AgentOperationNameResolver(), new AgentAttributeProvider()); try { $agent->call(new MessageBag()); @@ -98,6 +99,6 @@ private function buildAgent(string $name, ResultInterface $result): TracingAgent $inner->method('getName')->willReturn($name); $inner->method('call')->willReturn($result); - return new TracingAgent($inner, $this->tracerProvider, new AgentAttributeProvider()); + return new TracingAgent($inner, $this->tracerProvider, new AgentOperationNameResolver(), new AgentAttributeProvider()); } } diff --git a/tests/Tracing/AI/Platform/TracingPlatformTest.php b/tests/Tracing/AI/Platform/TracingPlatformTest.php index 3bd440b..0f30e00 100644 --- a/tests/Tracing/AI/Platform/TracingPlatformTest.php +++ b/tests/Tracing/AI/Platform/TracingPlatformTest.php @@ -10,6 +10,7 @@ namespace Tests\Instrumentation\Tracing\AI\Platform; use Instrumentation\Semantics\Attribute\PlatformAttributeProvider; +use Instrumentation\Semantics\OperationName\PlatformOperationNameResolver; use Instrumentation\Tracing\AI\Platform\TracingPlatform; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -125,7 +126,7 @@ public function testItEndsSpanWithErrorStatusWhenPlatformThrows(): void $inner = $this->createMock(PlatformInterface::class); $inner->method('invoke')->willThrowException(new \RuntimeException('API unreachable')); - $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformAttributeProvider(), 'gemini'); + $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), 'gemini'); try { $platform->invoke('gemini-2.5-pro', 'Hello'); @@ -146,7 +147,7 @@ public function testItEndsSpanWithErrorStatusWhenConverterThrows(): void $inner = $this->createMock(PlatformInterface::class); $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult())); - $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformAttributeProvider(), 'gemini'); + $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), 'gemini'); try { $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); @@ -167,6 +168,6 @@ private function buildPlatform(string $system, TokenUsageExtractorInterface|null $inner = $this->createMock(PlatformInterface::class); $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult())); - return new TracingPlatform($inner, $this->tracerProvider, new PlatformAttributeProvider(), $system); + return new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), $system); } } diff --git a/tests/Tracing/AI/Toolbox/TracingToolboxTest.php b/tests/Tracing/AI/Toolbox/TracingToolboxTest.php index fbe4bd0..9293c11 100644 --- a/tests/Tracing/AI/Toolbox/TracingToolboxTest.php +++ b/tests/Tracing/AI/Toolbox/TracingToolboxTest.php @@ -10,6 +10,7 @@ namespace Tests\Instrumentation\Tracing\AI\Toolbox; use Instrumentation\Semantics\Attribute\ToolAttributeProvider; +use Instrumentation\Semantics\OperationName\ToolOperationNameResolver; use Instrumentation\Tracing\AI\Toolbox\TracingToolbox; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -75,7 +76,7 @@ public function testItRecordsExceptionsAndEndsTheSpan(): void $inner = $this->createMock(ToolboxInterface::class); $inner->method('execute')->willThrowException($exception); - $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolAttributeProvider()); + $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider()); try { $toolbox->execute($toolCall); @@ -94,7 +95,7 @@ public function testItDelegatesGetTools(): void $inner = $this->createMock(ToolboxInterface::class); $inner->method('getTools')->willReturn([]); - $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolAttributeProvider()); + $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider()); $this->assertSame([], $toolbox->getTools()); } @@ -104,6 +105,6 @@ private function buildToolbox(ToolCall $expected, ToolResult $result): TracingTo $inner = $this->createMock(ToolboxInterface::class); $inner->method('execute')->with($expected)->willReturn($result); - return new TracingToolbox($inner, $this->tracerProvider, new ToolAttributeProvider()); + return new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider()); } } From 08be806331db4f1ac8d49faae2f1dbe298608972 Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Fri, 19 Jun 2026 13:59:31 +0200 Subject: [PATCH 09/12] feat(Metrics): record gen_ai.client.operation.duration histogram for AI platform calls Adds the second OTel gen_ai client metric (semconv): operation duration in seconds, measured from platform invoke() to result conversion, with error.type set when the call fails. Token usage and duration recording are centralised in a shared AiMetricRecorder used across the metering decorator chain. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/Metrics/AI/Platform/AiMetricRecorder.php | 73 +++++++++++++++++++ src/Metrics/AI/Platform/MeteringPlatform.php | 23 ++++-- .../AI/Platform/MeteringResultConverter.php | 34 +++++++-- .../Platform/MeteringTokenUsageExtractor.php | 32 +------- .../AI/Platform/MeteringPlatformTest.php | 34 +++++++++ 5 files changed, 157 insertions(+), 39 deletions(-) create mode 100644 src/Metrics/AI/Platform/AiMetricRecorder.php diff --git a/src/Metrics/AI/Platform/AiMetricRecorder.php b/src/Metrics/AI/Platform/AiMetricRecorder.php new file mode 100644 index 0000000..2d8e76d --- /dev/null +++ b/src/Metrics/AI/Platform/AiMetricRecorder.php @@ -0,0 +1,73 @@ + + */ + +namespace Instrumentation\Metrics\AI\Platform; + +use OpenTelemetry\API\Metrics\MeterProviderInterface; +use OpenTelemetry\SemConv\TraceAttributes; +use OpenTelemetry\SemConv\TraceAttributeValues; +use Symfony\AI\Platform\TokenUsage\TokenUsageInterface; + +/** + * Records the OTel gen_ai client metrics (semconv) for a platform call: + * the `gen_ai.client.token.usage` and `gen_ai.client.operation.duration` + * histograms, sharing the common gen_ai.* attribute set. + */ +final class AiMetricRecorder +{ + public function __construct( + private readonly MeterProviderInterface $meterProvider, + private readonly string $system, + ) { + } + + public function recordTokenUsage(string $model, TokenUsageInterface $tokenUsage): void + { + $histogram = $this->meterProvider + ->getMeter('instrumentation') + ->createHistogram('gen_ai.client.token.usage', '{token}', 'Number of tokens used by the model'); + + if (null !== $tokenUsage->getPromptTokens()) { + $histogram->record($tokenUsage->getPromptTokens(), $this->attributes($model) + [ + TraceAttributes::GEN_AI_TOKEN_TYPE => TraceAttributeValues::GEN_AI_TOKEN_TYPE_INPUT, + ]); + } + if (null !== $tokenUsage->getCompletionTokens()) { + $histogram->record($tokenUsage->getCompletionTokens(), $this->attributes($model) + [ + TraceAttributes::GEN_AI_TOKEN_TYPE => TraceAttributeValues::GEN_AI_TOKEN_TYPE_OUTPUT, + ]); + } + } + + public function recordDuration(string $model, float $seconds, \Throwable|null $error = null): void + { + $attributes = $this->attributes($model); + + if (null !== $error) { + $attributes[TraceAttributes::ERROR_TYPE] = $error::class; + } + + $this->meterProvider + ->getMeter('instrumentation') + ->createHistogram('gen_ai.client.operation.duration', 's', 'GenAI operation duration') + ->record($seconds, $attributes); + } + + /** + * @return array + */ + private function attributes(string $model): array + { + return [ + TraceAttributes::GEN_AI_OPERATION_NAME => TraceAttributeValues::GEN_AI_OPERATION_NAME_CHAT, + TraceAttributes::GEN_AI_SYSTEM => $this->system, + TraceAttributes::GEN_AI_REQUEST_MODEL => $model, + ]; + } +} diff --git a/src/Metrics/AI/Platform/MeteringPlatform.php b/src/Metrics/AI/Platform/MeteringPlatform.php index a263f50..4aca24a 100644 --- a/src/Metrics/AI/Platform/MeteringPlatform.php +++ b/src/Metrics/AI/Platform/MeteringPlatform.php @@ -15,8 +15,8 @@ use Symfony\AI\Platform\Result\DeferredResult; /** - * Decorates a platform to record the `gen_ai.client.token.usage` histogram - * (OTel gen_ai metric semconv) from the token usage of each model call. + * Decorates a platform to record the OTel gen_ai client metrics (semconv): + * the token usage and operation duration histograms. * * Independent from {@see \Instrumentation\Tracing\AI\Platform\TracingPlatform}: * metrics and tracing are separate, separately-toggled concerns, so each wraps @@ -24,19 +24,30 @@ */ final class MeteringPlatform implements PlatformInterface { + private readonly AiMetricRecorder $recorder; + public function __construct( private readonly PlatformInterface $platform, - private readonly MeterProviderInterface $meterProvider, - private readonly string $system, + MeterProviderInterface $meterProvider, + string $system, ) { + $this->recorder = new AiMetricRecorder($meterProvider, $system); } public function invoke(string $model, array|string|object $input, array $options = []): DeferredResult { - $deferredResult = $this->platform->invoke($model, $input, $options); + $start = hrtime(true); + + try { + $deferredResult = $this->platform->invoke($model, $input, $options); + } catch (\Throwable $e) { + $this->recorder->recordDuration($model, (hrtime(true) - $start) / 1e9, $e); + + throw $e; + } return new DeferredResult( - new MeteringResultConverter($deferredResult->getResultConverter(), $this->meterProvider, $this->system, $model), + new MeteringResultConverter($deferredResult->getResultConverter(), $this->recorder, $model, $start), $deferredResult->getRawResult(), $options, ); diff --git a/src/Metrics/AI/Platform/MeteringResultConverter.php b/src/Metrics/AI/Platform/MeteringResultConverter.php index 802fa64..a0c801f 100644 --- a/src/Metrics/AI/Platform/MeteringResultConverter.php +++ b/src/Metrics/AI/Platform/MeteringResultConverter.php @@ -9,7 +9,6 @@ namespace Instrumentation\Metrics\AI\Platform; -use OpenTelemetry\API\Metrics\MeterProviderInterface; use Symfony\AI\Platform\Model; use Symfony\AI\Platform\Result\RawResultInterface; use Symfony\AI\Platform\Result\ResultInterface; @@ -18,11 +17,16 @@ final class MeteringResultConverter implements ResultConverterInterface { + private bool $durationRecorded = false; + + /** + * @param int $start the hrtime(true) nanosecond timestamp captured when the platform was invoked + */ public function __construct( private readonly ResultConverterInterface $inner, - private readonly MeterProviderInterface $meterProvider, - private readonly string $system, + private readonly AiMetricRecorder $recorder, private readonly string $model, + private readonly int $start, ) { } @@ -33,7 +37,17 @@ public function supports(Model $model): bool public function convert(RawResultInterface $result, array $options = []): ResultInterface { - return $this->inner->convert($result, $options); + try { + $converted = $this->inner->convert($result, $options); + } catch (\Throwable $e) { + $this->recordDuration($e); + + throw $e; + } + + $this->recordDuration(null); + + return $converted; } public function getTokenUsageExtractor(): TokenUsageExtractorInterface|null @@ -44,6 +58,16 @@ public function getTokenUsageExtractor(): TokenUsageExtractorInterface|null return null; } - return new MeteringTokenUsageExtractor($inner, $this->meterProvider, $this->system, $this->model); + return new MeteringTokenUsageExtractor($inner, $this->recorder, $this->model); + } + + private function recordDuration(\Throwable|null $error): void + { + if ($this->durationRecorded) { + return; + } + + $this->durationRecorded = true; + $this->recorder->recordDuration($this->model, (hrtime(true) - $this->start) / 1e9, $error); } } diff --git a/src/Metrics/AI/Platform/MeteringTokenUsageExtractor.php b/src/Metrics/AI/Platform/MeteringTokenUsageExtractor.php index c5dd80f..1d8f170 100644 --- a/src/Metrics/AI/Platform/MeteringTokenUsageExtractor.php +++ b/src/Metrics/AI/Platform/MeteringTokenUsageExtractor.php @@ -9,24 +9,19 @@ namespace Instrumentation\Metrics\AI\Platform; -use OpenTelemetry\API\Metrics\MeterProviderInterface; -use OpenTelemetry\SemConv\TraceAttributes; -use OpenTelemetry\SemConv\TraceAttributeValues; use Symfony\AI\Platform\Result\RawResultInterface; use Symfony\AI\Platform\TokenUsage\TokenUsageExtractorInterface; use Symfony\AI\Platform\TokenUsage\TokenUsageInterface; /** - * Records the OTel gen_ai `gen_ai.client.token.usage` histogram, split by - * `gen_ai.token.type` (input/output), while delegating to the platform's own - * token usage extractor so the result chain is untouched. + * Records the OTel gen_ai `gen_ai.client.token.usage` histogram while delegating + * to the platform's own token usage extractor so the result chain is untouched. */ final class MeteringTokenUsageExtractor implements TokenUsageExtractorInterface { public function __construct( private readonly TokenUsageExtractorInterface $inner, - private readonly MeterProviderInterface $meterProvider, - private readonly string $system, + private readonly AiMetricRecorder $recorder, private readonly string $model, ) { } @@ -39,26 +34,7 @@ public function extract(RawResultInterface $rawResult, array $options = []): Tok return null; } - $histogram = $this->meterProvider - ->getMeter('instrumentation') - ->createHistogram('gen_ai.client.token.usage', '{token}', 'Number of tokens used by the model'); - - $attributes = [ - TraceAttributes::GEN_AI_OPERATION_NAME => TraceAttributeValues::GEN_AI_OPERATION_NAME_CHAT, - TraceAttributes::GEN_AI_SYSTEM => $this->system, - TraceAttributes::GEN_AI_REQUEST_MODEL => $this->model, - ]; - - if (null !== $tokenUsage->getPromptTokens()) { - $histogram->record($tokenUsage->getPromptTokens(), $attributes + [ - TraceAttributes::GEN_AI_TOKEN_TYPE => TraceAttributeValues::GEN_AI_TOKEN_TYPE_INPUT, - ]); - } - if (null !== $tokenUsage->getCompletionTokens()) { - $histogram->record($tokenUsage->getCompletionTokens(), $attributes + [ - TraceAttributes::GEN_AI_TOKEN_TYPE => TraceAttributeValues::GEN_AI_TOKEN_TYPE_OUTPUT, - ]); - } + $this->recorder->recordTokenUsage($this->model, $tokenUsage); return $tokenUsage; } diff --git a/tests/Metrics/AI/Platform/MeteringPlatformTest.php b/tests/Metrics/AI/Platform/MeteringPlatformTest.php index 8dbe159..e2daf28 100644 --- a/tests/Metrics/AI/Platform/MeteringPlatformTest.php +++ b/tests/Metrics/AI/Platform/MeteringPlatformTest.php @@ -80,6 +80,40 @@ public function testItRecordsNothingWhenExtractorReturnsNull(): void $this->assertSame([], $this->collectHistogramDataPoints('gen_ai.client.token.usage')); } + public function testItRecordsTheOperationDurationHistogram(): void + { + $platform = $this->buildPlatform('openai', null); + $platform->invoke('gpt-4o', 'Hello')->asText(); + + $dataPoints = $this->collectHistogramDataPoints('gen_ai.client.operation.duration'); + $this->assertCount(1, $dataPoints); + + $attributes = $dataPoints[0]->attributes->toArray(); + $this->assertSame('chat', $attributes['gen_ai.operation.name']); + $this->assertSame('openai', $attributes['gen_ai.system']); + $this->assertSame('gpt-4o', $attributes['gen_ai.request.model']); + $this->assertArrayNotHasKey('error.type', $attributes); + $this->assertGreaterThanOrEqual(0, $dataPoints[0]->sum); + } + + public function testItRecordsDurationWithErrorTypeWhenInvokeThrows(): void + { + $inner = $this->createMock(PlatformInterface::class); + $inner->method('invoke')->willThrowException(new \RuntimeException('boom')); + + $platform = new MeteringPlatform($inner, $this->meterProvider, 'openai'); + + try { + $platform->invoke('gpt-4o', 'Hello'); + $this->fail('Exception should have been rethrown'); + } catch (\RuntimeException) { + } + + $dataPoints = $this->collectHistogramDataPoints('gen_ai.client.operation.duration'); + $this->assertCount(1, $dataPoints); + $this->assertSame(\RuntimeException::class, $dataPoints[0]->attributes->toArray()['error.type']); + } + /** * @return list<\OpenTelemetry\SDK\Metrics\Data\HistogramDataPoint> */ From b21f07d36a8e08afeb7b6a9db03b4347b4b0c91d Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Fri, 19 Jun 2026 14:25:47 +0200 Subject: [PATCH 10/12] feat(Tracing): add per-feature blacklist for AI tracing Each AI tracing decorator now gates span creation on an OperationNameVoter (regex blacklist matched against the model, agent name, or tool name). When an operation matches, the decorator delegates to the inner service without creating a span. Configurable via: tracing.ai.platform.blacklist (regex matched against the model) tracing.ai.agent.blacklist (regex matched against the agent name) tracing.ai.tool.blacklist (regex matched against the tool name) AI tracing is decorator-based, so it can't use the event/sampler Voter machinery; OperationNameVoter reuses the same regex-blacklist semantics as AbstractVoter but gates the decorator directly. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../AIAgentTracingCompilerPass.php | 8 ++++ .../AIPlatformTracingCompilerPass.php | 5 +++ src/DependencyInjection/Configuration.php | 30 ++++++++++++++ src/DependencyInjection/Extension.php | 4 ++ src/Tracing/AI/Agent/TracingAgent.php | 6 +++ src/Tracing/AI/Platform/TracingPlatform.php | 6 +++ .../AI/Sampling/OperationNameVoter.php | 40 +++++++++++++++++++ src/Tracing/AI/Toolbox/TracingToolbox.php | 6 +++ tests/Tracing/AI/Agent/TracingAgentTest.php | 18 ++++++++- .../AI/Platform/TracingPlatformTest.php | 23 +++++++++-- .../Tracing/AI/Toolbox/TracingToolboxTest.php | 20 ++++++++-- 11 files changed, 158 insertions(+), 8 deletions(-) create mode 100644 src/Tracing/AI/Sampling/OperationNameVoter.php diff --git a/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php b/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php index 8a78506..1c2c860 100644 --- a/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php +++ b/src/DependencyInjection/CompilerPass/AIAgentTracingCompilerPass.php @@ -14,6 +14,7 @@ use Instrumentation\Semantics\OperationName\AgentOperationNameResolverInterface; use Instrumentation\Semantics\OperationName\ToolOperationNameResolverInterface; use Instrumentation\Tracing\AI\Agent\TracingAgent; +use Instrumentation\Tracing\AI\Sampling\OperationNameVoter; use Instrumentation\Tracing\AI\Toolbox\TracingToolbox; use OpenTelemetry\API\Trace\TracerProviderInterface; use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; @@ -25,6 +26,11 @@ final class AIAgentTracingCompilerPass implements CompilerPassInterface { public function process(ContainerBuilder $container): void { + /** @var array $agentBlacklist */ + $agentBlacklist = $container->hasParameter('tracing.ai.agent.blacklist') ? $container->getParameter('tracing.ai.agent.blacklist') : []; + /** @var array $toolBlacklist */ + $toolBlacklist = $container->hasParameter('tracing.ai.tool.blacklist') ? $container->getParameter('tracing.ai.tool.blacklist') : []; + // Low priority ensures the tracing decorator is the outermost one, so the // span covers the full call including any inner decorators (e.g. the // profiler's traceable decorators registered at -1024). @@ -38,6 +44,7 @@ public function process(ContainerBuilder $container): void new Reference(TracerProviderInterface::class), new Reference(AgentOperationNameResolverInterface::class), new Reference(AgentAttributeProviderInterface::class), + new Definition(OperationNameVoter::class, [$agentBlacklist]), ]); $container->setDefinition('instrumentation.tracing.ai.agent.'.$name, $definition); @@ -53,6 +60,7 @@ public function process(ContainerBuilder $container): void new Reference(TracerProviderInterface::class), new Reference(ToolOperationNameResolverInterface::class), new Reference(ToolAttributeProviderInterface::class), + new Definition(OperationNameVoter::class, [$toolBlacklist]), ]); $container->setDefinition('instrumentation.tracing.ai.toolbox.'.$name, $definition); diff --git a/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php index 1039d8a..b8b8dbd 100644 --- a/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php +++ b/src/DependencyInjection/CompilerPass/AIPlatformTracingCompilerPass.php @@ -12,6 +12,7 @@ use Instrumentation\Semantics\Attribute\PlatformAttributeProviderInterface; use Instrumentation\Semantics\OperationName\PlatformOperationNameResolverInterface; use Instrumentation\Tracing\AI\Platform\TracingPlatform; +use Instrumentation\Tracing\AI\Sampling\OperationNameVoter; use OpenTelemetry\API\Trace\TracerProviderInterface; use Symfony\Component\DependencyInjection\Compiler\CompilerPassInterface; use Symfony\Component\DependencyInjection\ContainerBuilder; @@ -22,6 +23,9 @@ final class AIPlatformTracingCompilerPass implements CompilerPassInterface { public function process(ContainerBuilder $container): void { + /** @var array $blacklist */ + $blacklist = $container->hasParameter('tracing.ai.platform.blacklist') ? $container->getParameter('tracing.ai.platform.blacklist') : []; + foreach ($container->findTaggedServiceIds('ai.platform') as $platformId => $tags) { $system = $tags[0]['name'] ?? $platformId; @@ -35,6 +39,7 @@ public function process(ContainerBuilder $container): void new Reference(PlatformOperationNameResolverInterface::class), new Reference(PlatformAttributeProviderInterface::class), $system, + new Definition(OperationNameVoter::class, [$blacklist]), ]); $container->setDefinition('instrumentation.tracing.ai.platform.'.$system, $definition); diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index f129aad..0507c0e 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -160,6 +160,36 @@ public function getConfigTreeBuilder(): TreeBuilder ->addDefaultsIfNotSet() ->children() ->booleanNode('enabled')->defaultFalse()->end() + ->arrayNode('platform') + ->addDefaultsIfNotSet() + ->children() + ->arrayNode('blacklist') + ->info('Models (regex) to exclude from tracing') + ->defaultValue([]) + ->scalarPrototype()->end() + ->end() + ->end() + ->end() + ->arrayNode('agent') + ->addDefaultsIfNotSet() + ->children() + ->arrayNode('blacklist') + ->info('Agent names (regex) to exclude from tracing') + ->defaultValue([]) + ->scalarPrototype()->end() + ->end() + ->end() + ->end() + ->arrayNode('tool') + ->addDefaultsIfNotSet() + ->children() + ->arrayNode('blacklist') + ->info('Tool names (regex) to exclude from tracing') + ->defaultValue([]) + ->scalarPrototype()->end() + ->end() + ->end() + ->end() ->end() ->end() ->arrayNode('doctrine') diff --git a/src/DependencyInjection/Extension.php b/src/DependencyInjection/Extension.php index 432be89..dcecf70 100644 --- a/src/DependencyInjection/Extension.php +++ b/src/DependencyInjection/Extension.php @@ -158,6 +158,10 @@ protected function loadTracing(array $config, ContainerBuilder $container): void } } + $container->setParameter('tracing.ai.platform.blacklist', $config['ai']['platform']['blacklist']); + $container->setParameter('tracing.ai.agent.blacklist', $config['ai']['agent']['blacklist']); + $container->setParameter('tracing.ai.tool.blacklist', $config['ai']['tool']['blacklist']); + $container->setParameter('tracing.doctrine.connections', $config['doctrine']['connections']); $container->setParameter('tracing.doctrine.log_queries', $config['doctrine']['log_queries']); $container->setParameter('tracing.doctrine.propagation', $config['doctrine']['propagation']); diff --git a/src/Tracing/AI/Agent/TracingAgent.php b/src/Tracing/AI/Agent/TracingAgent.php index 0815cf4..fb844c5 100644 --- a/src/Tracing/AI/Agent/TracingAgent.php +++ b/src/Tracing/AI/Agent/TracingAgent.php @@ -11,6 +11,7 @@ use Instrumentation\Semantics\Attribute\AgentAttributeProviderInterface; use Instrumentation\Semantics\OperationName\AgentOperationNameResolverInterface; +use Instrumentation\Tracing\AI\Sampling\OperationNameVoter; use Instrumentation\Tracing\TracerAwareTrait; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -36,12 +37,17 @@ public function __construct( TracerProviderInterface $tracerProvider, private readonly AgentOperationNameResolverInterface $operationNameResolver, private readonly AgentAttributeProviderInterface $attributeProvider, + private readonly OperationNameVoter $voter, ) { $this->tracerProvider = $tracerProvider; } public function call(MessageBag $messages, array $options = []): ResultInterface { + if (!$this->voter->shouldTrace($this->agent->getName())) { + return $this->agent->call($messages, $options); + } + $span = $this->getTracer() ->spanBuilder($this->operationNameResolver->getOperationName($this->agent)) ->setSpanKind(SpanKind::KIND_INTERNAL) diff --git a/src/Tracing/AI/Platform/TracingPlatform.php b/src/Tracing/AI/Platform/TracingPlatform.php index 298c0da..86ad05e 100644 --- a/src/Tracing/AI/Platform/TracingPlatform.php +++ b/src/Tracing/AI/Platform/TracingPlatform.php @@ -11,6 +11,7 @@ use Instrumentation\Semantics\Attribute\PlatformAttributeProviderInterface; use Instrumentation\Semantics\OperationName\PlatformOperationNameResolverInterface; +use Instrumentation\Tracing\AI\Sampling\OperationNameVoter; use Instrumentation\Tracing\TracerAwareTrait; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -29,12 +30,17 @@ public function __construct( private readonly PlatformOperationNameResolverInterface $operationNameResolver, private readonly PlatformAttributeProviderInterface $attributeProvider, private readonly string $system, + private readonly OperationNameVoter $voter, ) { $this->tracerProvider = $tracerProvider; } public function invoke(string $model, array|string|object $input, array $options = []): DeferredResult { + if (!$this->voter->shouldTrace($model)) { + return $this->platform->invoke($model, $input, $options); + } + $operationName = $this->operationNameResolver->getOperationName($model, $options); $span = $this->getTracer() diff --git a/src/Tracing/AI/Sampling/OperationNameVoter.php b/src/Tracing/AI/Sampling/OperationNameVoter.php new file mode 100644 index 0000000..77b4630 --- /dev/null +++ b/src/Tracing/AI/Sampling/OperationNameVoter.php @@ -0,0 +1,40 @@ + + */ + +namespace Instrumentation\Tracing\AI\Sampling; + +/** + * Decides whether an AI operation should be traced, based on a regex blacklist + * matched against an identifier (the model, agent name or tool name). + * + * AI tracing is decorator-based rather than event-based, so it cannot use the + * event/sampler Voter machinery in Tracing\Bridge\Sampling; the decorator gates + * span creation on this voter directly. The blacklist matching mirrors + * {@see \Instrumentation\Tracing\Bridge\Sampling\Voter\AbstractVoter}. + */ +final class OperationNameVoter +{ + /** + * @param array $blacklist + */ + public function __construct(private readonly array $blacklist) + { + } + + public function shouldTrace(string $name): bool + { + foreach ($this->blacklist as $pattern) { + if (1 === preg_match("|$pattern|", $name)) { + return false; + } + } + + return true; + } +} diff --git a/src/Tracing/AI/Toolbox/TracingToolbox.php b/src/Tracing/AI/Toolbox/TracingToolbox.php index 14d0f22..d639895 100644 --- a/src/Tracing/AI/Toolbox/TracingToolbox.php +++ b/src/Tracing/AI/Toolbox/TracingToolbox.php @@ -11,6 +11,7 @@ use Instrumentation\Semantics\Attribute\ToolAttributeProviderInterface; use Instrumentation\Semantics\OperationName\ToolOperationNameResolverInterface; +use Instrumentation\Tracing\AI\Sampling\OperationNameVoter; use Instrumentation\Tracing\TracerAwareTrait; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -35,6 +36,7 @@ public function __construct( TracerProviderInterface $tracerProvider, private readonly ToolOperationNameResolverInterface $operationNameResolver, private readonly ToolAttributeProviderInterface $attributeProvider, + private readonly OperationNameVoter $voter, ) { $this->tracerProvider = $tracerProvider; } @@ -49,6 +51,10 @@ public function getTools(): array public function execute(ToolCall $toolCall): ToolResult { + if (!$this->voter->shouldTrace($toolCall->getName())) { + return $this->toolbox->execute($toolCall); + } + $span = $this->getTracer() ->spanBuilder($this->operationNameResolver->getOperationName($toolCall)) ->setSpanKind(SpanKind::KIND_INTERNAL) diff --git a/tests/Tracing/AI/Agent/TracingAgentTest.php b/tests/Tracing/AI/Agent/TracingAgentTest.php index d125660..dd2edfe 100644 --- a/tests/Tracing/AI/Agent/TracingAgentTest.php +++ b/tests/Tracing/AI/Agent/TracingAgentTest.php @@ -12,6 +12,7 @@ use Instrumentation\Semantics\Attribute\AgentAttributeProvider; use Instrumentation\Semantics\OperationName\AgentOperationNameResolver; use Instrumentation\Tracing\AI\Agent\TracingAgent; +use Instrumentation\Tracing\AI\Sampling\OperationNameVoter; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; @@ -72,7 +73,7 @@ public function testItRecordsExceptionsAndEndsTheSpan(): void $inner->method('getName')->willReturn('my_agent'); $inner->method('call')->willThrowException($exception); - $agent = new TracingAgent($inner, $this->tracerProvider, new AgentOperationNameResolver(), new AgentAttributeProvider()); + $agent = new TracingAgent($inner, $this->tracerProvider, new AgentOperationNameResolver(), new AgentAttributeProvider(), new OperationNameVoter([])); try { $agent->call(new MessageBag()); @@ -86,6 +87,19 @@ public function testItRecordsExceptionsAndEndsTheSpan(): void $this->assertSame('exception', $this->spans[0]->getEvents()[0]->getName()); } + public function testItDoesNotTraceBlacklistedAgents(): void + { + $inner = $this->createMock(AgentInterface::class); + $inner->method('getName')->willReturn('internal_agent'); + $inner->method('call')->willReturn(new TextResult('Hello')); + + $agent = new TracingAgent($inner, $this->tracerProvider, new AgentOperationNameResolver(), new AgentAttributeProvider(), new OperationNameVoter(['^internal_'])); + + $agent->call(new MessageBag()); + + $this->assertCount(0, $this->spans); + } + public function testItDelegatesGetName(): void { $agent = $this->buildAgent('my_agent', new TextResult('Hello')); @@ -99,6 +113,6 @@ private function buildAgent(string $name, ResultInterface $result): TracingAgent $inner->method('getName')->willReturn($name); $inner->method('call')->willReturn($result); - return new TracingAgent($inner, $this->tracerProvider, new AgentOperationNameResolver(), new AgentAttributeProvider()); + return new TracingAgent($inner, $this->tracerProvider, new AgentOperationNameResolver(), new AgentAttributeProvider(), new OperationNameVoter([])); } } diff --git a/tests/Tracing/AI/Platform/TracingPlatformTest.php b/tests/Tracing/AI/Platform/TracingPlatformTest.php index 0f30e00..e452915 100644 --- a/tests/Tracing/AI/Platform/TracingPlatformTest.php +++ b/tests/Tracing/AI/Platform/TracingPlatformTest.php @@ -12,6 +12,7 @@ use Instrumentation\Semantics\Attribute\PlatformAttributeProvider; use Instrumentation\Semantics\OperationName\PlatformOperationNameResolver; use Instrumentation\Tracing\AI\Platform\TracingPlatform; +use Instrumentation\Tracing\AI\Sampling\OperationNameVoter; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; use OpenTelemetry\SDK\Trace\SpanExporter\InMemoryExporter; @@ -126,7 +127,7 @@ public function testItEndsSpanWithErrorStatusWhenPlatformThrows(): void $inner = $this->createMock(PlatformInterface::class); $inner->method('invoke')->willThrowException(new \RuntimeException('API unreachable')); - $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), 'gemini'); + $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), 'gemini', new OperationNameVoter([])); try { $platform->invoke('gemini-2.5-pro', 'Hello'); @@ -147,7 +148,7 @@ public function testItEndsSpanWithErrorStatusWhenConverterThrows(): void $inner = $this->createMock(PlatformInterface::class); $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult())); - $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), 'gemini'); + $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), 'gemini', new OperationNameVoter([])); try { $platform->invoke('gemini-2.5-pro', 'Hello')->asText(); @@ -159,6 +160,22 @@ public function testItEndsSpanWithErrorStatusWhenConverterThrows(): void $this->assertSame(StatusCode::STATUS_ERROR, $this->spans[0]->getStatus()->getCode()); } + public function testItDoesNotTraceBlacklistedModels(): void + { + $converter = $this->createMock(ResultConverterInterface::class); + $converter->method('convert')->willReturn(new TextResult('response')); + $converter->method('getTokenUsageExtractor')->willReturn(null); + + $inner = $this->createMock(PlatformInterface::class); + $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult())); + + $platform = new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), 'openai', new OperationNameVoter(['-mini$'])); + + $platform->invoke('gpt-4o-mini', 'Hello')->asText(); + + $this->assertCount(0, $this->spans); + } + private function buildPlatform(string $system, TokenUsageExtractorInterface|null $extractor = null): TracingPlatform { $converter = $this->createMock(ResultConverterInterface::class); @@ -168,6 +185,6 @@ private function buildPlatform(string $system, TokenUsageExtractorInterface|null $inner = $this->createMock(PlatformInterface::class); $inner->method('invoke')->willReturn(new DeferredResult($converter, new InMemoryRawResult())); - return new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), $system); + return new TracingPlatform($inner, $this->tracerProvider, new PlatformOperationNameResolver(), new PlatformAttributeProvider(), $system, new OperationNameVoter([])); } } diff --git a/tests/Tracing/AI/Toolbox/TracingToolboxTest.php b/tests/Tracing/AI/Toolbox/TracingToolboxTest.php index 9293c11..7abc1ed 100644 --- a/tests/Tracing/AI/Toolbox/TracingToolboxTest.php +++ b/tests/Tracing/AI/Toolbox/TracingToolboxTest.php @@ -11,6 +11,7 @@ use Instrumentation\Semantics\Attribute\ToolAttributeProvider; use Instrumentation\Semantics\OperationName\ToolOperationNameResolver; +use Instrumentation\Tracing\AI\Sampling\OperationNameVoter; use Instrumentation\Tracing\AI\Toolbox\TracingToolbox; use OpenTelemetry\API\Trace\SpanKind; use OpenTelemetry\API\Trace\StatusCode; @@ -76,7 +77,7 @@ public function testItRecordsExceptionsAndEndsTheSpan(): void $inner = $this->createMock(ToolboxInterface::class); $inner->method('execute')->willThrowException($exception); - $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider()); + $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider(), new OperationNameVoter([])); try { $toolbox->execute($toolCall); @@ -90,12 +91,25 @@ public function testItRecordsExceptionsAndEndsTheSpan(): void $this->assertSame('exception', $this->spans[0]->getEvents()[0]->getName()); } + public function testItDoesNotTraceBlacklistedTools(): void + { + $toolCall = new ToolCall('call_1', 'debug_dump'); + $inner = $this->createMock(ToolboxInterface::class); + $inner->method('execute')->willReturn(new ToolResult($toolCall, 'ok')); + + $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider(), new OperationNameVoter(['^debug_'])); + + $toolbox->execute($toolCall); + + $this->assertCount(0, $this->spans); + } + public function testItDelegatesGetTools(): void { $inner = $this->createMock(ToolboxInterface::class); $inner->method('getTools')->willReturn([]); - $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider()); + $toolbox = new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider(), new OperationNameVoter([])); $this->assertSame([], $toolbox->getTools()); } @@ -105,6 +119,6 @@ private function buildToolbox(ToolCall $expected, ToolResult $result): TracingTo $inner = $this->createMock(ToolboxInterface::class); $inner->method('execute')->with($expected)->willReturn($result); - return new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider()); + return new TracingToolbox($inner, $this->tracerProvider, new ToolOperationNameResolver(), new ToolAttributeProvider(), new OperationNameVoter([])); } } From 40894c336d5e47a30b5f50393ab0d6e45350bfa0 Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Fri, 19 Jun 2026 14:29:29 +0200 Subject: [PATCH 11/12] test: add unit tests for AI providers, resolvers and the span-leak backstop Adds the per-class unit tests the bundle keeps for every AttributeProvider and OperationNameResolver (Platform/Agent/Tool), covering the resolver's operation_name-from-options and non-string fallback branches, and a test that the platform span is ended by the destructor when a deferred result is never consumed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Attribute/AgentAttributeProviderTest.php | 35 +++++++++++++++ .../PlatformAttributeProviderTest.php | 32 ++++++++++++++ .../Attribute/ToolAttributeProviderTest.php | 33 ++++++++++++++ .../AgentOperationNameResolverTest.php | 33 ++++++++++++++ .../PlatformOperationNameResolverTest.php | 44 +++++++++++++++++++ .../ToolOperationNameResolverTest.php | 30 +++++++++++++ .../AI/Platform/TracingPlatformTest.php | 15 +++++++ 7 files changed, 222 insertions(+) create mode 100644 tests/Semantics/Attribute/AgentAttributeProviderTest.php create mode 100644 tests/Semantics/Attribute/PlatformAttributeProviderTest.php create mode 100644 tests/Semantics/Attribute/ToolAttributeProviderTest.php create mode 100644 tests/Semantics/OperationName/AgentOperationNameResolverTest.php create mode 100644 tests/Semantics/OperationName/PlatformOperationNameResolverTest.php create mode 100644 tests/Semantics/OperationName/ToolOperationNameResolverTest.php diff --git a/tests/Semantics/Attribute/AgentAttributeProviderTest.php b/tests/Semantics/Attribute/AgentAttributeProviderTest.php new file mode 100644 index 0000000..b9bddfe --- /dev/null +++ b/tests/Semantics/Attribute/AgentAttributeProviderTest.php @@ -0,0 +1,35 @@ + + */ + +namespace Tests\Instrumentation\Semantics\Attribute; + +use Instrumentation\Semantics\Attribute\AgentAttributeProvider; +use Instrumentation\Semantics\Attribute\AgentAttributeProviderInterface; +use PHPUnit\Framework\TestCase; +use Symfony\AI\Agent\AgentInterface; + +class AgentAttributeProviderTest extends TestCase +{ + public function testItImplementsAgentAttributeProviderInterface(): void + { + $this->assertTrue(is_a(AgentAttributeProvider::class, AgentAttributeProviderInterface::class, true)); + } + + public function testItSetsGenAiAgentAttributes(): void + { + $agent = $this->createMock(AgentInterface::class); + $agent->method('getName')->willReturn('my_agent'); + + $provider = new AgentAttributeProvider(); + $attributes = $provider->getAttributes($agent); + + $this->assertSame('invoke_agent', $attributes['gen_ai.operation.name']); + $this->assertSame('my_agent', $attributes['gen_ai.agent.name']); + } +} diff --git a/tests/Semantics/Attribute/PlatformAttributeProviderTest.php b/tests/Semantics/Attribute/PlatformAttributeProviderTest.php new file mode 100644 index 0000000..699953e --- /dev/null +++ b/tests/Semantics/Attribute/PlatformAttributeProviderTest.php @@ -0,0 +1,32 @@ + + */ + +namespace Tests\Instrumentation\Semantics\Attribute; + +use Instrumentation\Semantics\Attribute\PlatformAttributeProvider; +use Instrumentation\Semantics\Attribute\PlatformAttributeProviderInterface; +use PHPUnit\Framework\TestCase; + +class PlatformAttributeProviderTest extends TestCase +{ + public function testItImplementsPlatformAttributeProviderInterface(): void + { + $this->assertTrue(is_a(PlatformAttributeProvider::class, PlatformAttributeProviderInterface::class, true)); + } + + public function testItSetsGenAiAttributes(): void + { + $provider = new PlatformAttributeProvider(); + $attributes = $provider->getAttributes('openai', 'gpt-4o', 'chat'); + + $this->assertSame('chat', $attributes['gen_ai.operation.name']); + $this->assertSame('openai', $attributes['gen_ai.system']); + $this->assertSame('gpt-4o', $attributes['gen_ai.request.model']); + } +} diff --git a/tests/Semantics/Attribute/ToolAttributeProviderTest.php b/tests/Semantics/Attribute/ToolAttributeProviderTest.php new file mode 100644 index 0000000..e3794d8 --- /dev/null +++ b/tests/Semantics/Attribute/ToolAttributeProviderTest.php @@ -0,0 +1,33 @@ + + */ + +namespace Tests\Instrumentation\Semantics\Attribute; + +use Instrumentation\Semantics\Attribute\ToolAttributeProvider; +use Instrumentation\Semantics\Attribute\ToolAttributeProviderInterface; +use PHPUnit\Framework\TestCase; +use Symfony\AI\Platform\Result\ToolCall; + +class ToolAttributeProviderTest extends TestCase +{ + public function testItImplementsToolAttributeProviderInterface(): void + { + $this->assertTrue(is_a(ToolAttributeProvider::class, ToolAttributeProviderInterface::class, true)); + } + + public function testItSetsGenAiToolAttributes(): void + { + $provider = new ToolAttributeProvider(); + $attributes = $provider->getAttributes(new ToolCall('call_123', 'get_weather', ['city' => 'Paris'])); + + $this->assertSame('execute_tool', $attributes['gen_ai.operation.name']); + $this->assertSame('get_weather', $attributes['gen_ai.tool.name']); + $this->assertSame('call_123', $attributes['gen_ai.tool.call.id']); + } +} diff --git a/tests/Semantics/OperationName/AgentOperationNameResolverTest.php b/tests/Semantics/OperationName/AgentOperationNameResolverTest.php new file mode 100644 index 0000000..c566584 --- /dev/null +++ b/tests/Semantics/OperationName/AgentOperationNameResolverTest.php @@ -0,0 +1,33 @@ + + */ + +namespace Tests\Instrumentation\Semantics\OperationName; + +use Instrumentation\Semantics\OperationName\AgentOperationNameResolver; +use Instrumentation\Semantics\OperationName\AgentOperationNameResolverInterface; +use PHPUnit\Framework\TestCase; +use Symfony\AI\Agent\AgentInterface; + +class AgentOperationNameResolverTest extends TestCase +{ + public function testItImplementsAgentOperationNameResolverInterface(): void + { + $this->assertTrue(is_a(AgentOperationNameResolver::class, AgentOperationNameResolverInterface::class, true)); + } + + public function testItResolvesInvokeAgentOperationName(): void + { + $agent = $this->createMock(AgentInterface::class); + $agent->method('getName')->willReturn('my_agent'); + + $resolver = new AgentOperationNameResolver(); + + $this->assertSame('invoke_agent my_agent', $resolver->getOperationName($agent)); + } +} diff --git a/tests/Semantics/OperationName/PlatformOperationNameResolverTest.php b/tests/Semantics/OperationName/PlatformOperationNameResolverTest.php new file mode 100644 index 0000000..ecd2328 --- /dev/null +++ b/tests/Semantics/OperationName/PlatformOperationNameResolverTest.php @@ -0,0 +1,44 @@ + + */ + +namespace Tests\Instrumentation\Semantics\OperationName; + +use Instrumentation\Semantics\OperationName\PlatformOperationNameResolver; +use Instrumentation\Semantics\OperationName\PlatformOperationNameResolverInterface; +use PHPUnit\Framework\TestCase; + +class PlatformOperationNameResolverTest extends TestCase +{ + public function testItImplementsPlatformOperationNameResolverInterface(): void + { + $this->assertTrue(is_a(PlatformOperationNameResolver::class, PlatformOperationNameResolverInterface::class, true)); + } + + public function testItDefaultsWhenNoOperationNameIsGiven(): void + { + $resolver = new PlatformOperationNameResolver(); + + $this->assertSame('symfony_ai', $resolver->getOperationName('gpt-4o', [])); + } + + public function testItUsesTheOperationNameFromOptions(): void + { + $resolver = new PlatformOperationNameResolver(); + + $this->assertSame('chat', $resolver->getOperationName('gpt-4o', ['extra' => ['operation_name' => 'chat']])); + } + + public function testItFallsBackWhenOperationNameIsNotANonEmptyString(): void + { + $resolver = new PlatformOperationNameResolver(); + + $this->assertSame('symfony_ai', $resolver->getOperationName('gpt-4o', ['extra' => ['operation_name' => 123]])); + $this->assertSame('symfony_ai', $resolver->getOperationName('gpt-4o', ['extra' => ['operation_name' => '']])); + } +} diff --git a/tests/Semantics/OperationName/ToolOperationNameResolverTest.php b/tests/Semantics/OperationName/ToolOperationNameResolverTest.php new file mode 100644 index 0000000..116bc0e --- /dev/null +++ b/tests/Semantics/OperationName/ToolOperationNameResolverTest.php @@ -0,0 +1,30 @@ + + */ + +namespace Tests\Instrumentation\Semantics\OperationName; + +use Instrumentation\Semantics\OperationName\ToolOperationNameResolver; +use Instrumentation\Semantics\OperationName\ToolOperationNameResolverInterface; +use PHPUnit\Framework\TestCase; +use Symfony\AI\Platform\Result\ToolCall; + +class ToolOperationNameResolverTest extends TestCase +{ + public function testItImplementsToolOperationNameResolverInterface(): void + { + $this->assertTrue(is_a(ToolOperationNameResolver::class, ToolOperationNameResolverInterface::class, true)); + } + + public function testItResolvesExecuteToolOperationName(): void + { + $resolver = new ToolOperationNameResolver(); + + $this->assertSame('execute_tool get_weather', $resolver->getOperationName(new ToolCall('call_123', 'get_weather'))); + } +} diff --git a/tests/Tracing/AI/Platform/TracingPlatformTest.php b/tests/Tracing/AI/Platform/TracingPlatformTest.php index e452915..2697867 100644 --- a/tests/Tracing/AI/Platform/TracingPlatformTest.php +++ b/tests/Tracing/AI/Platform/TracingPlatformTest.php @@ -83,6 +83,21 @@ public function testItSetsStatusOkOnSuccess(): void $this->assertSame(StatusCode::STATUS_OK, $this->spans[0]->getStatus()->getCode()); } + public function testItEndsSpanWhenResultIsNeverConsumed(): void + { + $platform = $this->buildPlatform('gemini'); + + $result = $platform->invoke('gemini-2.5-pro', 'Hello'); + $this->assertCount(0, $this->spans, 'Span must not end before the result is consumed'); + + // Drop the result without consuming it: the converter's destructor must + // end the span so it is exported rather than leaked. + unset($result); + gc_collect_cycles(); + + $this->assertCount(1, $this->spans); + } + public function testItAddsTokenUsageAttributesFromExtractor(): void { $extractor = $this->createMock(TokenUsageExtractorInterface::class); From fd47af3c9fe287d3ff6b2ee25d487155e7909f9e Mon Sep 17 00:00:00 2001 From: Christian Daguerre Date: Fri, 19 Jun 2026 14:33:47 +0200 Subject: [PATCH 12/12] test: add standalone unit test for OperationNameVoter Covers the empty-blacklist (trace everything), single-pattern match, and multi-pattern match cases of the AI tracing blacklist gate. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../AI/Sampling/OperationNameVoterTest.php | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/Tracing/AI/Sampling/OperationNameVoterTest.php diff --git a/tests/Tracing/AI/Sampling/OperationNameVoterTest.php b/tests/Tracing/AI/Sampling/OperationNameVoterTest.php new file mode 100644 index 0000000..60e155a --- /dev/null +++ b/tests/Tracing/AI/Sampling/OperationNameVoterTest.php @@ -0,0 +1,41 @@ + + */ + +namespace Tests\Instrumentation\Tracing\AI\Sampling; + +use Instrumentation\Tracing\AI\Sampling\OperationNameVoter; +use PHPUnit\Framework\TestCase; + +class OperationNameVoterTest extends TestCase +{ + public function testItTracesEverythingWhenBlacklistIsEmpty(): void + { + $voter = new OperationNameVoter([]); + + $this->assertTrue($voter->shouldTrace('gpt-4o')); + $this->assertTrue($voter->shouldTrace('anything')); + } + + public function testItDoesNotTraceNamesMatchingABlacklistPattern(): void + { + $voter = new OperationNameVoter(['-mini$']); + + $this->assertFalse($voter->shouldTrace('gpt-4o-mini')); + $this->assertTrue($voter->shouldTrace('gpt-4o')); + } + + public function testItMatchesAgainstAnyPatternInTheBlacklist(): void + { + $voter = new OperationNameVoter(['^internal_', '^debug_']); + + $this->assertFalse($voter->shouldTrace('internal_agent')); + $this->assertFalse($voter->shouldTrace('debug_dump')); + $this->assertTrue($voter->shouldTrace('get_weather')); + } +}