[SVLS-9300] add instrumentation configuration flags for ecs-fargate instrument

packages/base/src/commands/ecs-fargate/instrument.ts

@@ -5,7 +5,7 @@ import {toBoolean} from '../../helpers/env'
 import {enableFips} from '../../helpers/fips'
 import {executePluginCommand} from '../../helpers/plugin'
 import {dryRunTag} from '../../helpers/renderer'
-import {AGENT_IMAGE} from '../../helpers/serverless/constants'
+import {AGENT_IMAGE, ENV_VAR_REGEX, EXTRA_TAGS_REG_EXP} from '../../helpers/serverless/constants'
 import {DEFAULT_CONFIG_PATHS, removeUndefinedValues, resolveConfigFromFile} from '../../helpers/utils'
 
 import {BaseCommand} from '../..'
@@ -24,6 +24,21 @@ export type EcsFargateConfigOptions = Partial<{
   apiKeySecretArn: string
   agentImage: string
   agentSocket: boolean
+  logCollection: boolean
+  service: string
+  environment: string
+  version: string
+  extraTags: string
+  envVars: string[]
+  // no-dd-sa:typescript-best-practices/boolean-prop-naming
+  sourceCodeIntegration: boolean
+  // no-dd-sa:typescript-best-practices/boolean-prop-naming
+  uploadGitMetadata: boolean
+  tracing: string
+  logLevel: string
+  // no-dd-sa:typescript-best-practices/boolean-prop-naming
+  appsec: boolean
+  llmobs: string
 }>
 
 /**
@@ -132,6 +147,48 @@ export class EcsFargateInstrumentCommand extends BaseCommand {
     description:
       'Have the tracers reach the Agent over the task loopback address instead of the Unix socket they use by default.',
   })
+  private logCollection = Option.Boolean('--log-collection,--logCollection', {
+    description: `Send the task's logs to Datadog. Replaces each container's existing log configuration.`,
+  })
+  private service = Option.String('--service', {
+    description:
+      'The value for the service tag. Use this to group related tasks belonging to similar workloads. For example, `my-service`. If not provided, the task definition family is used.',
+  })
+  private environment = Option.String('--env,--environment', {
+    description:
+      'The value for the env tag. Use this to separate your staging, development, and production environments. For example, `prod`.',
+  })
+  private version = Option.String('--version', {
+    description:
+      'The value for the version tag. Use this to correlate spikes in latency, load, or errors to new versions. For example, `1.0.0`.',
+  })
+  private extraTags = Option.String('--extra-tags,--extraTags', {
+    description: 'Additional tags to add to the task in the format "key1:value1,key2:value2".',
+  })
+  private envVars = Option.Array('-e,--env-vars', {
+    description:
+      'Additional environment variables to set on the application containers and the Datadog Agent. Can specify multiple variables in the format `--env-vars VAR1=VALUE1 --env-vars VAR2=VALUE2`.',
+  })
+  private sourceCodeIntegration = Option.Boolean('--source-code-integration,--sourceCodeIntegration', {
+    description: `Whether to enable the Datadog Source Code integration. This tags your service(s) with the Git repository and the latest commit hash of the local directory. Specify \`--no-source-code-integration\` to disable. Defaults to 'true'`,
+  })
+  private uploadGitMetadata = Option.Boolean('--upload-git-metadata,--uploadGitMetadata', {
+    description: `Whether to enable Git metadata uploading, as a part of the source code integration. Git metadata uploading is only required if you don't have the Datadog GitHub integration installed. Specify \`--no-upload-git-metadata\` to disable. Defaults to 'true'`,
+  })
+  private tracing = Option.String('--tracing', {
+    description:
+      'Enables tracing of your application if the tracer is installed. Disable tracing by setting `--tracing false`.',
+  })
+  private logLevel = Option.String('--log-level,--logLevel', {
+    description: 'Specify your Datadog log level.',
+  })
+  private appsec = Option.Boolean('--appsec', {
+    description: `Enable Application Security Monitoring for the instrumented task. Defaults to 'false'`,
+  })
+  private llmobs = Option.String('--llmobs', {
+    description:
+      'If specified, enables LLM Observability for the instrumented task with the provided ML application name.',
+  })
   private configPath = Option.String('--config', {
     description: 'Path to the configuration file.',
   })
@@ -165,6 +222,18 @@ export class EcsFargateInstrumentCommand extends BaseCommand {
       apiKeySecretArn: this.apiKeySecretArn,
       agentImage: this.agentImage,
       agentSocket: this.noAgentSocket === undefined ? undefined : !this.noAgentSocket,
+      logCollection: this.logCollection,
+      service: this.service,
+      environment: this.environment,
+      version: this.version,
+      extraTags: this.extraTags,
+      envVars: this.envVars,
+      sourceCodeIntegration: this.sourceCodeIntegration,
+      uploadGitMetadata: this.uploadGitMetadata,
+      tracing: this.tracing,
+      logLevel: this.logLevel,
+      appsec: this.appsec,
+      llmobs: this.llmobs,
     }
 
     let fileConfig: EcsFargateConfigOptions
@@ -195,6 +264,15 @@ export class EcsFargateInstrumentCommand extends BaseCommand {
       )
     }
 
+    if (config.envVars?.some((envVar) => !ENV_VAR_REGEX.test(envVar))) {
+      errors.push('All env vars must be in the format `KEY=VALUE`')
+    }
+    if (config.extraTags && !config.extraTags.match(EXTRA_TAGS_REG_EXP)) {
+      errors.push('Extra tags do not comply with the <key>:<value> array.')
+    }
+    if (config.tracing !== undefined && toBoolean(config.tracing) === undefined) {
+      errors.push('--tracing must be either `true` or `false`.')
+    }
     if (config.cluster && !config.ecsServices?.length) {
       errors.push('--cluster names the cluster of the services to update, so it only applies with --ecs-service.')
     }

packages/base/src/helpers/serverless/constants.ts

@@ -8,6 +8,7 @@ export const LOGS_PATH_ENV_VAR = 'DD_SERVERLESS_LOG_PATH'
 export const HEALTH_PORT_ENV_VAR = 'DD_HEALTH_PORT'
 export const DD_LOG_LEVEL_ENV_VAR = 'DD_LOG_LEVEL'
 export const DD_TRACE_ENABLED_ENV_VAR = 'DD_TRACE_ENABLED'
+export const DD_APPSEC_ENABLED_ENV_VAR = 'DD_APPSEC_ENABLED'
 export const DD_LLMOBS_ENABLED_ENV_VAR = 'DD_LLMOBS_ENABLED'
 export const DD_LLMOBS_ML_APP_ENV_VAR = 'DD_LLMOBS_ML_APP'
 export const DD_LLMOBS_AGENTLESS_ENABLED_ENV_VAR = 'DD_LLMOBS_AGENTLESS_ENABLED'

packages/plugin-ecs-fargate/README.md

@@ -24,11 +24,23 @@ datadog-ci ecs-fargate instrument --task-definition my-app --task-definition my-
 datadog-ci ecs-fargate instrument --task-definition my-app -r us-east-1 --api-key-secret-arn <secret-arn> \
   --ecs-service my-app-service --cluster my-cluster
 
+# Instrument with unified service tagging
+datadog-ci ecs-fargate instrument --task-definition my-app -r us-east-1 --api-key-secret-arn <secret-arn> \
+  --service my-service --env prod --version 1.0.0
+
+# Send container logs to Datadog
+datadog-ci ecs-fargate instrument --task-definition my-app -r us-east-1 --api-key-secret-arn <secret-arn> \
+  --log-collection
+
 # Preview the changes without registering a revision
 datadog-ci ecs-fargate instrument --task-definition my-app -r us-east-1 --api-key-secret-arn <secret-arn> --dry-run
 ```
 
-Application containers are given `DD_TRACE_ENABLED` and `DD_LOGS_INJECTION`, so the tracers already installed in them send traces and tie your logs to those traces. Both are only filled in when the container does not set them itself, so a task definition that has already made a choice keeps it.
+Application containers are given `DD_SERVICE`, `DD_ENV`, `DD_VERSION`, and `DD_TAGS` from the arguments above, so the traces, logs, and metrics your tracers send are tagged consistently. `DD_SERVICE` is `--service`, or the task definition family when `--service` is omitted, and is written to the application containers, the Agent, the Docker labels, and the revision tags so they cannot disagree. `DD_TRACE_ENABLED` and `DD_LOGS_INJECTION` are only filled in when the container does not set them itself, so a task definition that has already made a choice keeps it. Everything else the command is asked for wins over what the task definition had.
+
+Product settings the command owns are applied when those flags are on and removed when they are not: `--no-appsec` drops `DD_APPSEC_ENABLED`, omitting `--llmobs` drops the LLM Observability variables, and `--no-source-code-integration` drops the Git tags from `DD_TAGS`.
+
+The same three unified service tag values are also written to the application containers as the `com.datadoghq.tags.service`, `com.datadoghq.tags.env`, and `com.datadoghq.tags.version` Docker labels. The environment variables tag what a tracer running inside a container sends; these labels are what the Agent reads to tag the metrics it collects about the container from the outside, so the two line up in Datadog. The Agent container is deliberately left unlabelled, so that it reports its own resource usage under its own name rather than your service's. Labels that are not Datadog's are left alone.
 
 #### Reaching the Agent
 
@@ -38,7 +50,12 @@ Unlike the switches above, the command owns these: the two ways of reaching the
 
 The Agent sidecar accepts custom metrics over DogStatsD: `DD_DOGSTATSD_ORIGIN_DETECTION` and `DD_DOGSTATSD_ORIGIN_DETECTION_CLIENT` are turned on and `DD_DOGSTATSD_TAG_CARDINALITY` is set to `orchestrator`, so your metrics are tagged with the task that submitted them. These are filled in the same way, so a task definition that already sets them keeps its own values.
 
-Running the command twice is safe: the Agent container is matched by name, so an already instrumented task definition is reported as such and no revision is registered. Each revision the command registers is tagged `dd_sls_ci` with the version of `datadog-ci` that created it. Upgrading the CLI does not on its own produce a new revision, since that tag is not part of the comparison.
+Running the command twice is safe: the sidecars are matched by name, so an already instrumented task definition is reported as such and no revision is registered. Each revision the command registers is tagged `dd_sls_ci` with the version of `datadog-ci` that created it. Upgrading the CLI does not on its own produce a new revision, since that tag is not part of the comparison.
+
+#### Collecting logs
+
+Pass `--log-collection` to send the task's logs to Datadog. A `datadog-log-router` sidecar running [AWS for Fluent Bit](https://github.com/aws/aws-for-fluent-bit) is added and the other containers, including the Agent, are routed through it.
+Existing log configurations are replaced. Omitting `--log-collection` on a later run removes the router and the Datadog FireLens configuration it wrote, but does not restore the log configuration it replaced.
 
 #### Deploying the new revision
 
@@ -52,7 +69,7 @@ A task definition and the services running it are instrumented and deployed on t
 
 #### AWS credentials
 
-You must have valid [AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html#envvars-list) configured with access to the ECS actions `ecs:DescribeTaskDefinition`, `ecs:RegisterTaskDefinition`, and `ecs:TagResource`. The last one is required because the new revision is registered with tags: the ones the task definition already had, plus `dd_sls_ci`. Deploying with `--ecs-service` also needs `ecs:DescribeServices` and `ecs:UpdateService`.
+You must have valid [AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html#envvars-list) configured with access to the ECS actions `ecs:DescribeTaskDefinition`, `ecs:RegisterTaskDefinition`, and `ecs:TagResource`. The last one is required because the new revision is registered with tags: the ones the task definition already had, plus `service`, `env`, `version`, and `dd_sls_ci`. Deploying with `--ecs-service` also needs `ecs:DescribeServices` and `ecs:UpdateService`.
 
 `--profile` uses a named profile from your AWS configuration instead. A profile with an `mfa_serial` is supported: the command asks for the code when it loads the profile.
 
@@ -89,6 +106,18 @@ You can pass the following arguments to `instrument` to specify its behavior. `-
 | `--api-key-secret-arn` or `--apiKeySecretArn` |  | The ARN of the AWS Secrets Manager secret holding your Datadog API key. Preferred over DD_API_KEY, which is written to the task definition in plain text |  |
 | `--agent-image` or `--sidecar-image` |  | Override to pin a specific version tag or to use a mirrored image from a custom registry (for example, ECR) to avoid pull rate limits. | `public.ecr.aws/datadog/agent:latest` |
 | `--no-agent-socket` |  | Have the tracers reach the Agent over the task loopback address instead of the Unix socket they use by default. |  |
+| `--log-collection` or `--logCollection` |  | Send the task's logs to Datadog. Replaces each container's existing log configuration. |  |
+| `--service` |  | The value for the service tag. Use this to group related tasks belonging to similar workloads. For example, `my-service`. If not provided, the task definition family is used. |  |
+| `--env` or `--environment` |  | The value for the env tag. Use this to separate your staging, development, and production environments. For example, `prod`. |  |
+| `--version` |  | The value for the version tag. Use this to correlate spikes in latency, load, or errors to new versions. For example, `1.0.0`. |  |
+| `--extra-tags` or `--extraTags` |  | Additional tags to add to the task in the format "key1:value1,key2:value2". |  |
+| `--env-vars` | `-e` | Additional environment variables to set on the application containers and the Datadog Agent. Can specify multiple variables in the format `--env-vars VAR1=VALUE1 --env-vars VAR2=VALUE2`. |  |
+| `--source-code-integration` or `--sourceCodeIntegration` |  | Whether to enable the Datadog Source Code integration. This tags your service(s) with the Git repository and the latest commit hash of the local directory. Specify `--no-source-code-integration` to disable. | `true` |
+| `--upload-git-metadata` or `--uploadGitMetadata` |  | Whether to enable Git metadata uploading, as a part of the source code integration. Git metadata uploading is only required if you don't have the Datadog GitHub integration installed. Specify `--no-upload-git-metadata` to disable. | `true` |
+| `--tracing` |  | Enables tracing of your application if the tracer is installed. Disable tracing by setting `--tracing false`. |  |
+| `--log-level` or `--logLevel` |  | Specify your Datadog log level. |  |
+| `--appsec` |  | Enable Application Security Monitoring for the instrumented task. | `false` |
+| `--llmobs` |  | If specified, enables LLM Observability for the instrumented task with the provided ML application name. |  |
 | `--config` |  | Path to the configuration file. |  |
 <!-- END_USAGE:instrument -->
 
@@ -103,7 +132,13 @@ Instead of supplying arguments, you can create a configuration file in your proj
     "region": "us-east-1",
     "ecsServices": ["my-app-service", "my-worker-service"],
     "cluster": "my-cluster",
-    "apiKeySecretArn": "arn:aws:secretsmanager:us-east-1:123456789012:secret:dd-api-key"
+    "apiKeySecretArn": "arn:aws:secretsmanager:us-east-1:123456789012:secret:dd-api-key",
+    "service": "my-service",
+    "environment": "prod",
+    "version": "1.0.0",
+    "extraTags": "team:backend,project:api",
+    "envVars": ["CUSTOM_VAR1=value1", "CUSTOM_VAR2=value2"],
+    "logCollection": true
   }
 }
 ```

packages/plugin-ecs-fargate/src/__tests__/__snapshots__/instrument.test.ts.snap

@@ -2,8 +2,13 @@
 
 exports[`ecs-fargate instrument prints the diff and registers nothing on a dry run 1`] = `
 "[Dry Run] Instrumenting my-app:
-@@ -3,10 +3,26 @@
+@@ -1,12 +1,35 @@
+  {
+    "containerDefinitions": [
       {
++       "dockerLabels": {
++         "com.datadoghq.tags.service": "my-app"
++       },
         "environment": [
           {
             "name": "PORT",
@@ -18,6 +23,10 @@ exports[`ecs-fargate instrument prints the diff and registers nothing on a dry r
 +           "value": "true"
 +         },
 +         {
++           "name": "DD_SERVICE",
++           "value": "my-app"
++         },
++         {
 +           "name": "DD_TRACE_AGENT_URL",
 +           "value": "unix:///var/run/datadog/apm.socket"
 +         },
@@ -29,7 +38,7 @@ exports[`ecs-fargate instrument prints the diff and registers nothing on a dry r
         "essential": true,
         "image": "my-app:latest",
         "logConfiguration": {
-@@ -15,20 +31,106 @@
+@@ -15,20 +38,114 @@
             "awslogs-group": "/ecs/my-app",
             "awslogs-region": "us-east-1",
             "awslogs-stream-prefix": "ecs"
@@ -77,6 +86,10 @@ exports[`ecs-fargate instrument prints the diff and registers nothing on a dry r
 +         {
 +           "name": "DD_ECS_TASK_COLLECTION_ENABLED",
 +           "value": "true"
++         },
++         {
++           "name": "DD_SERVICE",
++           "value": "my-app"
 +         }
 +       ],
 +       "essential": false,
@@ -126,6 +139,10 @@ exports[`ecs-fargate instrument prints the diff and registers nothing on a dry r
 -   "tags": [],
 +   "tags": [
 +     {
++       "key": "service",
++       "value": "my-app"
++     },
++     {
 +       "key": "dd_sls_ci",
 +       "value": "vXXXX"
 +     }

packages/plugin-ecs-fargate/src/__tests__/config/datadog-ci.json

@@ -1,7 +1,11 @@
 {
   "ecsFargate": {
     "taskDefinitions": ["my-app"],
+    "service": "from-config",
+    "environment": "staging",
+    "sourceCodeIntegration": false,
     "agentImage": "public.ecr.aws/datadog/agent:7-from-config",
-    "agentSocket": false
+    "agentSocket": false,
+    "logCollection": true
   }
 }

packages/plugin-ecs-fargate/src/__tests__/fixtures.ts

@@ -1,6 +1,7 @@
 import type {InstrumentSettings} from '../task-definition'
 import type {
   ContainerDefinition,
+  LogConfiguration,
   MountPoint,
   RegisterTaskDefinitionCommandInput,
   Service,
@@ -24,15 +25,38 @@ export const serviceArn = (service = MOCK_SERVICE, cluster = MOCK_CLUSTER): stri
   `arn:aws:ecs:${MOCK_REGION}:123456789012:service/${cluster}/${service}`
 
 export const CLI_VERSION_TAG: Tag = {key: 'dd_sls_ci', value: 'vXXXX'}
+/** The service tag a run that was not given a `--service` writes, from the task definition family. */
+export const SERVICE_TAG: Tag = {key: 'service', value: MOCK_FAMILY}
 /** The tags every instrumented revision carries. */
-export const INSTRUMENTATION_TAGS: Tag[] = [CLI_VERSION_TAG]
+export const INSTRUMENTATION_TAGS: Tag[] = [SERVICE_TAG, CLI_VERSION_TAG]
 
 /** No `agentImage`, so the transform picks the default build. */
 export const MOCK_SETTINGS: InstrumentSettings = {
   site: 'datadoghq.com',
   apiKeySecretArn: MOCK_API_KEY_SECRET_ARN,
 }
 
+/** The same, with the log router collecting the containers' logs. */
+export const MOCK_LOG_COLLECTION_SETTINGS: InstrumentSettings = {...MOCK_SETTINGS, logCollection: true}
+
+/** The log driver every container the router collects is given. */
+export const firelensLogConfiguration = (
+  apiKey: {secretArn: string} | {plaintext: string} = {secretArn: MOCK_API_KEY_SECRET_ARN}
+): LogConfiguration => ({
+  logDriver: 'awsfirelens',
+  options: {
+    Name: 'datadog',
+    Host: 'http-intake.logs.datadoghq.com',
+    TLS: 'on',
+    provider: 'ecs',
+    retry_limit: '2',
+    ...('plaintext' in apiKey ? {apikey: apiKey.plaintext} : {}),
+  },
+  ...('secretArn' in apiKey ? {secretOptions: [{name: 'apikey', valueFrom: apiKey.secretArn}]} : {}),
+})
+
+export const FIRELENS_LOG_CONFIGURATION = firelensLogConfiguration()
+
 /** What carries the Agent's APM and DogStatsD sockets between the containers sharing them. */
 export const SOCKET_VOLUME: Volume = {name: 'dd-sockets'}
 export const SOCKET_MOUNT: MountPoint = {
@@ -56,6 +80,29 @@ export const APP_CONTAINER: ContainerDefinition = {
   },
 }
 
+/**
+ * The log router sidecar as the command writes it, keeping the `awslogs` configuration it borrows
+ * from the application container so that its own output is not routed through itself.
+ */
+export const LOG_ROUTER_CONTAINER: ContainerDefinition = {
+  name: 'datadog-log-router',
+  image: 'public.ecr.aws/aws-observability/aws-for-fluent-bit:stable',
+  essential: false,
+  user: '0',
+  firelensConfiguration: {
+    type: 'fluentbit',
+    options: {'enable-ecs-log-metadata': 'true'},
+  },
+  healthCheck: {
+    command: ['CMD-SHELL', 'exit 0'],
+    interval: 5,
+    timeout: 5,
+    retries: 3,
+    startPeriod: 15,
+  },
+  logConfiguration: APP_CONTAINER.logConfiguration,
+}
+
 /**
  * A task definition as `DescribeTaskDefinition` returns it, read-only fields included.
  */