.gitlab-ci.yml

include:
  - local: ".gitlab/one-pipeline.locked.yml"
  - local: ".gitlab/benchmarks.yml"
  - local: ".gitlab/exploration-tests.yml"
  - local: ".gitlab/ci-visibility-tests.yml"
  - project: 'DataDog/apm-reliability/apm-sdks-benchmarks'
    file: '.gitlab/ci-java-spring-petclinic-parallel.yml'
    ref: &apm_sdks_benchmarks_sha 'cbb693ea82c5a07634865e6edcc79cbaf947ee67' # pinned by .github/workflows/update-apm-sdks-benchmarks-reference.yaml
  - project: 'DataDog/apm-reliability/apm-sdks-benchmarks'
    file: '.gitlab/ci-java-load-parallel.yml'
    ref: *apm_sdks_benchmarks_sha
  - project: 'DataDog/apm-reliability/apm-sdks-benchmarks'
    file: '.gitlab/ci-java-startup-parallel.yml'
    ref: *apm_sdks_benchmarks_sha
  - project: 'DataDog/apm-reliability/apm-sdks-benchmarks'
    file: '.gitlab/ci-java-dacapo-parallel.yml'
    ref: *apm_sdks_benchmarks_sha
  - project: 'DataDog/apm-reliability/apm-sdks-benchmarks'
    file: '.gitlab/ci-java-post-pr-comment.yml'
    ref: *apm_sdks_benchmarks_sha
  - local: ".gitlab/java-benchmark-configs.yml"

stages:
  - build
  - publish
  - java-spring-petclinic-parallel
  - java-spring-petclinic-parallel-slo
  - java-startup-parallel
  - java-startup-parallel-slo
  - java-load-parallel
  - java-load-parallel-slo
  - java-dacapo-parallel
  - java-dacapo-parallel-slo
  - java-post-pr-comment
  - shared-pipeline-build
  - shared-pipeline-test
  - publish-release-artifacts
  - shared-pipeline-publish
  - benchmarks
  - tests
  - tests-arm64
  - test-summary
  - exploration-tests
  - ci-visibility-tests
  - generate-signing-key

variables:
  APM_SDKS_BENCHMARKS_SHA: *apm_sdks_benchmarks_sha
  # Test the OpenTelemetry Operator-compatible Java image jobs from one-pipeline.
  CI_OTEL_OPERATOR_IMAGES_ENABLED: "true"
  CI_OTEL_OPERATOR_LANGUAGE: java
  # Gitlab runner features; see https://docs.gitlab.com/runner/configuration/feature-flags.html
  # Fold and time all script sections
  FF_SCRIPT_SECTIONS: 1
  REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
  BUILD_JOB_NAME: "build"
  DEPENDENCY_CACHE_POLICY: pull
  BUILD_CACHE_POLICY: pull
  GRADLE_VERSION: "9.8.2" # must match gradle-wrapper.properties
  MASS_READ_URL: "https://mass-read.us1.ddbuild.io"
  MAVEN_REPOSITORY_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/"
  GRADLE_PLUGIN_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/"
  BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" # images are pinned in images/mirror.lock.yaml in the ddoghq/images repo
  BUILDER_IMAGE_VERSION_PREFIX: "ci-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "ci-" or "123_merge-")
  TEST_COUNTS_S3_BUCKET: "dd-trace-java-ci-test-reports"
  REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
  DEFAULT_TEST_JVMS: /^(8|11|17|21|25|tip)$/ # the latest "tip" version is 27
  PROFILE_TESTS:
    description: "Enable profiling of tests"
    value: "false"
  NON_DEFAULT_JVMS:
    description: "Enable tests on JVMs that are not the default"
    value: "false"
  RUN_FLAKY_TESTS:
    description: "Enable flaky tests"
    value: "false"
  SKIP_BUILD_CACHE:
    description: "Run Gradle tasks without build-cache or up-to-date reuse"
    value: "false"

  # One pipeline injection package size ratchet
  OCI_PACKAGE_MAX_SIZE_BYTES: 40_000_000
  LIB_INJECTION_IMAGE_MAX_SIZE_BYTES: 40_000_000

# trigger new commit cancel
workflow:
  auto_cancel:
    on_new_commit: interruptible
  rules:
    # skip gitlab pipeline for github merge queue - we are currently using the datadog merge queue
    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue\//'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master"'
      variables:
        SYSTEM_TESTS_RUN_ALL_VMS: "true"
      auto_cancel:
        on_new_commit: none
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      auto_cancel:
        on_new_commit: none
    - when: always

.test_matrix: &test_matrix
  - testJvm: &test_jvms
      - "8"
      - "11"
      - "17"
      - "21"
      - "25"
      - "oracle8"
      - "zulu8"
      - "semeru8"
      - "ibm8"
      - "zulu11"
      - "semeru11"
      - "semeru17"
      - "tip"
    CI_SPLIT: ["1/1"]

# Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
# These blocks emulate "parallel" by including it in the matrix
.test_matrix_2: &test_matrix_2
  - testJvm: *test_jvms
    CI_SPLIT: ["1/2", "2/2"]

.test_matrix_4: &test_matrix_4
  - testJvm: *test_jvms
    CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]

.test_matrix_6: &test_matrix_6
  - testJvm: *test_jvms
    CI_SPLIT: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]

.test_matrix_8: &test_matrix_8
  - testJvm: *test_jvms
    CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"]

.test_matrix_12: &test_matrix_12
  - testJvm: *test_jvms
    CI_SPLIT: [ "1/12", "2/12", "3/12", "4/12", "5/12", "6/12", "7/12", "8/12", "9/12", "10/12", "11/12", "12/12" ]

.master_only: &master_only
  - if: $CI_COMMIT_BRANCH == "master"
    when: on_success

default:
  tags: [ "arch:amd64" ]
  interruptible: true

.set_datadog_api_keys: &set_datadog_api_keys
  - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text)

# CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs
.normalize_node_index: &normalize_node_index
  - if [ "$CI_NO_SPLIT" == "true" ] ; then CI_NODE_INDEX=1; CI_NODE_TOTAL=1; fi # A job uses parallel but doesn't intend to split by index
  - if [ -n "$CI_SPLIT" ]; then CI_NODE_INDEX="${CI_SPLIT%%/*}"; CI_NODE_TOTAL="${CI_SPLIT##*/}"; fi
  - echo "CI_NODE_INDEX=$CI_NODE_INDEX, CI_NODE_TOTAL=${CI_NODE_TOTAL}"

.cgroup_info: &cgroup_info
  - source .gitlab/gitlab-utils.sh
  - gitlab_section_start "cgroup-info" "cgroup info"
  - .gitlab/cgroup-info.sh
  - gitlab_section_end "cgroup-info"

.container_info: &container_info
  - |
    # Containers and processes are limited to a 1-hour window, and expire after 36 hours
    if [ -n "$CI_JOB_STARTED_AT" ]; then
      FROM_EPOCH_MS=$(date -d "$CI_JOB_STARTED_AT" +%s)000
      TO_EPOCH_MS=$((FROM_EPOCH_MS + 3600000))
      TIME_PARAMS="from_ts=${FROM_EPOCH_MS}&to_ts=${TO_EPOCH_MS}&"
    else
      TIME_PARAMS=""
    fi

    echo -e "${TEXT_BOLD}${TEXT_YELLOW}Runner dashboard, these are live (limited to a 1-hour window, and expire after 36 hours)${TEXT_CLEAR}"
    echo -e "${TEXT_BOLD}${TEXT_YELLOW}   Containers:${TEXT_CLEAR} https://app.datadoghq.com/containers?${TIME_PARAMS}query=image_name%3A%2A%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false"
    echo -e "${TEXT_BOLD}${TEXT_YELLOW}   Processes:${TEXT_CLEAR} https://app.datadoghq.com/process?${TIME_PARAMS}query=image_name%3A%2A%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false"

.tier_m:
  variables: &tier_m_variables
    GRADLE_WORKERS: 4
    GRADLE_MEMORY_MIN: 1G
    GRADLE_MEMORY_MAX: 5G
    KUBERNETES_CPU_REQUEST: 6
    KUBERNETES_MEMORY_REQUEST: 16Gi
    KUBERNETES_MEMORY_LIMIT: 16Gi

.tier_l:
  variables: &tier_l_variables
    GRADLE_WORKERS: 6
    GRADLE_MEMORY_MIN: 1G
    GRADLE_MEMORY_MAX: 6G
    KUBERNETES_CPU_REQUEST: 10
    KUBERNETES_MEMORY_REQUEST: 20Gi
    KUBERNETES_MEMORY_LIMIT: 20Gi

.tier_xl:
  variables: &tier_xl_variables
    <<: *tier_l_variables
    KUBERNETES_MEMORY_REQUEST: 32Gi
    KUBERNETES_MEMORY_LIMIT: 32Gi

.gitlab_base_ref_params: &gitlab_base_ref_params
  - |
    export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh)
    if [[ -n "$GIT_BASE_REF" ]]; then
      export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF"
    else
      echo "Failed to find base ref for PR" >&2
    fi

.gradle_build: &gradle_build
  image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base
  stage: build
  variables:
    <<: *tier_m_variables
    MAVEN_OPTS: "-Xms256M -Xmx1024M"
    CACHE_TYPE: "lib" #default
    FF_USE_FASTZIP: "true"
    CACHE_COMPRESSION_LEVEL: "slowest"

    RUNTIME_AVAILABLE_PROCESSORS_OVERRIDE: 4 # Runtime.getRuntime().availableProcessors() returns incorrect or very high values in Kubernetes
    GIT_SUBMODULE_STRATEGY: normal
    GIT_SUBMODULE_DEPTH: 1
  cache:
    - &dependency_cache
      key: dependency-$CACHE_TYPE # Dependencies cache
      paths:
        # Cached dependencies and wrappers for Gradle and Maven:
        - .gradle/wrapper
        - .gradle/caches
        - .gradle/notifications
        - .mvn/caches
      policy: $DEPENDENCY_CACHE_POLICY
      unprotect: true
      fallback_keys: # Use fallback keys because all cache types are not populated. See note under: populate_dep_cache
        - 'dependency-base'
        - 'dependency-lib'
    - &build_cache
      key: $CI_PIPELINE_ID-$CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type
      paths:
        - .gradle/caches/$GRADLE_VERSION
        - .gradle/$GRADLE_VERSION/executionHistory
        - workspace
      policy: $BUILD_CACHE_POLICY
      unprotect: true
  before_script:
    - source .gitlab/gitlab-utils.sh
    - *container_info
    # Akka token added to SSM from https://account.akka.io/token
    - export ORG_GRADLE_PROJECT_akkaRepositoryToken=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.akka_repo_token --with-decryption --query "Parameter.Value" --out text)
    - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY
    - export ORG_GRADLE_PROJECT_gradlePluginProxy=$GRADLE_PLUGIN_PROXY
    - |
      JAVA_HOMES=$(env | grep -E '^JAVA_[A-Z0-9_]+_HOME=' | sed 's/=.*//' | paste -sd,)
      cat >> gradle.properties <<EOF
      org.gradle.java.installations.auto-detect=false
      org.gradle.java.installations.auto-download=false
      org.gradle.java.installations.fromEnv=$JAVA_HOMES
      org.gradle.console=colored
      EOF
    - mkdir -p .gradle
    - export GRADLE_USER_HOME=$(pwd)/.gradle
    # Apache Maven Wrapper supports MVNW_REPOURL for repository-manager downloads:
    # https://maven.apache.org/tools/wrapper/#Using_a_Maven_Repository_Manager
    - export MVNW_REPOURL=${MAVEN_REPOSITORY_PROXY%/}
    # Route Gradle distribution download through MASS pull-through cache
    - |
      mass_read_host="${MASS_READ_URL#https://}"
      mass_read_host="${mass_read_host%/}"
      sed -i "/^distributionUrl=/ s|services.gradle.org|${mass_read_host}/internal/artifact/services.gradle.org|" gradle/wrapper/gradle-wrapper.properties
    - mkdir -p .mvn/caches
    # Redirect Spotless's Equo/Solstice P2 cache into the project tree so it is captured by the GitLab cache.
    # Solstice (https://github.com/equodev/equo-ide) defaults to ~/.m2/repository/dev/equo/p2-data, which is outside $CI_PROJECT_DIR.
    - |
      mkdir -p .mvn/caches/equo "$HOME/.m2/repository/dev"
      if [ ! -L "$HOME/.m2/repository/dev/equo" ]; then
        rm -rf "$HOME/.m2/repository/dev/equo"
        ln -s "$(pwd)/.mvn/caches/equo" "$HOME/.m2/repository/dev/equo"
      fi
    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
    - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
    - if [ "$SKIP_BUILD_CACHE" = "true" ]; then export GRADLE_ARGS="$GRADLE_ARGS --no-build-cache --rerun-tasks"; fi
    - *normalize_node_index
    # GitLab's cache helper restores .gradle as root, but we run as non-root-user (uid 1001),
    # and Gradle does `chmod 700 .gradle` on startup which requires user ownership.
    - sudo chown -R 1001:1001 .gradle
  after_script:
    - *cgroup_info
    - *container_info

# Check and fail early if maven central credentials are incorrect. When a new token is generated
# on the central publisher portal, it invalidates the old one. This check prevents going further.
# See https://datadoghq.atlassian.net/wiki/x/Oog5OgE
maven-central-pre-release-check:
  image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base
  stage: .pre
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
      allow_failure: false
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
      allow_failure: false
  script:
    - |
      MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
      MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
      # See https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/
      # Use the staging API search endpoint to validate the tokens without relying on a specific deployment
      AUTHORIZATION_HEADER="Authorization: Bearer $(printf '%s:%s' "$MAVEN_CENTRAL_USERNAME" "$MAVEN_CENTRAL_PASSWORD" | base64)"
      if ! curl --silent --show-error --fail --retry 5 --retry-delay 2 --retry-connrefused \
        "https://ossrh-staging-api.central.sonatype.com/manual/search/repositories?ip=any" \
        --header "$AUTHORIZATION_HEADER" \
        > /dev/null; then
        echo "Failed to authenticate tokens against maven central staging API. Check credentials and see https://datadoghq.atlassian.net/wiki/x/Oog5OgE"
        exit 1
      fi

dd-octo-sts-pre-release-check:
  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
  stage: .pre
  tags: [ "arch:amd64" ]
  id_tokens:
    DDOCTOSTS_ID_TOKEN:
      aud: dd-octo-sts
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
      allow_failure: false
  before_script:
    - dd-octo-sts version
    - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release
    - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > test-github-token.txt
  script:
    - gh auth login --with-token < test-github-token.txt
    - gh auth status
  after_script:
    - dd-octo-sts revoke -t $(cat test-github-token.txt)
  retry:
    max: 2
    when: always

build:
  needs:
    - job: maven-central-pre-release-check
      optional: true
    - job: dd-octo-sts-pre-release-check
      optional: true
  extends: .gradle_build
  variables:
    BUILD_CACHE_POLICY: push
    CACHE_TYPE: "lib"
    DEPENDENCY_CACHE_POLICY: pull
  script:
    - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi
    - ./gradlew --version
    - ./gradlew clean :dd-java-agent:shadowJar :dd-java-agent:check :dd-trace-api:jar :dd-trace-ot:shadowJar :dd-openfeature:jar -PskipTests -x spotlessCheck $GRADLE_ARGS
    - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env
    - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env
    - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env
  artifacts:
    when: always
    paths:
      - 'workspace/dd-java-agent/build/libs/*.jar'
      - 'workspace/dd-trace-api/build/libs/*.jar'
      - 'workspace/dd-trace-ot/build/libs/*.jar'
      - 'workspace/dd-openfeature/build/libs/*.jar'
      - 'upstream.env'
      - '.gradle/daemon/*/*.out.log'
    reports:
      dotenv: build.env

build_tests:
  extends: .gradle_build
  variables:
    <<: *tier_xl_variables
    BUILD_CACHE_POLICY: push
    DEPENDENCY_CACHE_POLICY: pull
  parallel:
    matrix:
      - GRADLE_TARGET: ":baseTest"
        CACHE_TYPE: "base"
      - GRADLE_TARGET: ":profilingTest"
        CACHE_TYPE: "profiling"
      - GRADLE_TARGET: ":instrumentationTest"
        CACHE_TYPE: "inst"
      - GRADLE_TARGET: ":instrumentationLatestDepTest"
        CACHE_TYPE: "latestdep"
      - GRADLE_TARGET: ":smokeTest"
        CACHE_TYPE: "smoke"
        MAVEN_OPTS: "-Xms256M -Xmx1024M"
  script:
    - *gitlab_base_ref_params
    - ./gradlew --version
    - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS

populate_plugin_cache:
  extends: .gradle_build
  cache:
    - &plugin_cache
      key: dependency-plugins-v1
      paths:
        - .gradle-plugin-cache/caches/modules-2
      policy: pull-push
      when: always
      unprotect: true
  rules:
    - if: '$POPULATE_CACHE'
      when: on_success
  script:
    - mkdir -p .gradle-plugin-cache
    - sudo chown -R 1001:1001 .gradle-plugin-cache
    - export GRADLE_USER_HOME=$(pwd)/.gradle-plugin-cache
    - |
      gradle_status=0
      ./gradlew help -PskipTests $GRADLE_ARGS || gradle_status=$?
      if ! find .gradle-plugin-cache/caches/modules-2/files-2.1 -type f -print -quit 2>/dev/null | grep -q .; then
        echo "WARNING: No plugin artifacts were cached; skipping the cache upload."
        rm -rf .gradle-plugin-cache/caches/modules-2
      fi
      exit "$gradle_status"
  # This cache warmer is an optimization; population jobs can continue with a cold plugin cache.
  allow_failure: true

populate_dep_cache:
  extends: build_tests
  # Wait for build-cache producers so their caches are uploaded before this job restores them.
  needs:
    - job: populate_plugin_cache
    - job: build
      artifacts: false
    - job: build_tests
      artifacts: false
  variables:
    BUILD_CACHE_POLICY: pull
    DEPENDENCY_CACHE_POLICY: push
  cache:
    - <<: *plugin_cache
      policy: pull
    - *dependency_cache
    - *build_cache
  rules:
    - if: '$POPULATE_CACHE'
      when: on_success
  before_script:
    - !reference [.gradle_build, before_script]
    # Seed the otherwise cold writable cache, keeping the cache uploaded below self-contained.
    - mkdir -p .gradle/caches
    - |
      if [ -d .gradle-plugin-cache/caches/modules-2 ]; then
        sudo chown -R 1001:1001 .gradle-plugin-cache
        cp -a .gradle-plugin-cache/caches/modules-2 .gradle/caches/
      else
        echo "Plugin cache is unavailable; continuing with a cold dependency cache."
      fi
  parallel:
    matrix:
      - GRADLE_TARGET: ":dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar :dd-openfeature:jar"
        CACHE_TYPE: "lib"
      - GRADLE_TARGET: ":baseTest"
        CACHE_TYPE: "base"
      - GRADLE_TARGET: ":profilingTest"
        CACHE_TYPE: "profiling"
      - GRADLE_TARGET: ":instrumentationTest"
        CACHE_TYPE: "inst"
      - GRADLE_TARGET: ":instrumentationLatestDepTest"
        CACHE_TYPE: "latestdep"
      - GRADLE_TARGET: ":smokeTest"
        CACHE_TYPE: "smoke"
      - GRADLE_TARGET: "spotlessCheck"
        CACHE_TYPE: "spotless"
        GRADLE_MEMORY_MAX: "6G"

publish-artifacts-to-s3:
  image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29
  stage: publish
  needs: [ build ]
  script:
    - source upstream.env
    - export VERSION="${UPSTREAM_TRACER_VERSION%~*}" # remove ~githash from the end of version
    - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-java-agent.jar
    - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-api.jar
    - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-ot.jar
    - aws s3 cp workspace/dd-openfeature/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-openfeature.jar
    - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar
    - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-api.jar
    - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-ot.jar
    - aws s3 cp workspace/dd-openfeature/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar
    - |
      # Print the agent version and public jar links so they are easy to copy from the DDCI UI
      echo "Version of dd-java-agent: ${UPSTREAM_TRACER_VERSION}"
      echo "Public Link to dd-java-agent.jar: https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar"
      echo "Public Link to dd-openfeature.jar: https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar"
    - |
      cat << EOF > links.json
        {
          "S3 Links": [
            {
              "external_link": {
                "label": "Public Link to dd-java-agent.jar",
                "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar"
              }
            },
            {
              "external_link": {
                "label": "Public Link to dd-openfeature.jar",
                "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar"
              }
            }
          ]
        }
      EOF
  artifacts:
    reports:
      annotations:
        - links.json


spotless:
  extends: .gradle_build
  stage: tests
  needs: []
  variables:
    GRADLE_MEMORY_MAX: 6G
    CACHE_TYPE: "spotless"
  script:
    - ./gradlew --version
    # test-published-dependencies's build file needs main version file
    - ./gradlew spotlessCheck writeMainVersionFile $GRADLE_ARGS
    - cd test-published-dependencies && ./gradlew spotlessCheck $GRADLE_ARGS

check-instrumentation-naming:
  extends: .gradle_build
  stage: tests
  needs: [ ]
  script:
    - ./gradlew --version
    - ./gradlew checkInstrumentationNaming $GRADLE_ARGS

config-inversion-linter:
  extends: .gradle_build
  stage: tests
  needs: []
  script:
    - ./gradlew --version
    - ./gradlew checkConfigurations $GRADLE_ARGS

test_published_artifacts:
  extends: .gradle_build
  image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests
  stage: tests
  needs: [ build ]
  variables:
    CACHE_TYPE: "lib"
  script:
    - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
    - rm -rf "${mvn_local_repo}/com/datadoghq"
    - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
    - ./gradlew publishToMavenLocal $GRADLE_ARGS
    - unset GPG_PRIVATE_KEY GPG_PASSWORD
    - cd test-published-dependencies
    - printf '\norg.gradle.console=colored\n' >> gradle.properties
    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms1G -Xmx1G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
    - ./gradlew --version
    - ./gradlew check --info $GRADLE_ARGS
  after_script:
    - *cgroup_info
    - source .gitlab/gitlab-utils.sh
    - gitlab_section_start "collect-reports" "Collecting reports"
    - .gitlab/collect_reports.sh
    - gitlab_section_end "collect-reports"
  artifacts:
    when: always
    paths:
      - ./check_reports

validate_build:
  extends: .gradle_build
  stage: tests
  needs: [ build ]
  variables:
    CACHE_TYPE: "lib"
  script:
    # Preserve the `build` job artifacts before Gradle rebuilds and overwrites them under
    # workspace/**/build/libs, so jardiff can compare the rebuilt jars against them.
    - mkdir -p reference-artifacts
    - cp workspace/dd-java-agent/build/libs/*.jar reference-artifacts/
    - cp workspace/dd-trace-api/build/libs/*.jar reference-artifacts/
    - cp workspace/dd-trace-ot/build/libs/*.jar reference-artifacts/
    - cp workspace/dd-openfeature/build/libs/*.jar reference-artifacts/
    # Scheduled builds refresh dependency locks before creating the reference artifacts.
    # Refresh them here as well so the candidate uses the same dependency resolution.
    - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi
    - ./gradlew --version
    # This will run the shadowJar task and exercise the build cache, allowing to identify build-cache issues
    # Keep both JAR sets for direct inspection when the comparison fails.
    - |
      if ! ./gradlew compareToReferenceJar -PjardiffReferenceDir="$CI_PROJECT_DIR/reference-artifacts" -PskipTests $GRADLE_ARGS; then
        mkdir -p check_reports/jar-comparison-artifacts/reference check_reports/jar-comparison-artifacts/candidate
        cp reference-artifacts/*.jar check_reports/jar-comparison-artifacts/reference/
        cp workspace/dd-java-agent/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/
        cp workspace/dd-trace-api/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/
        cp workspace/dd-trace-ot/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/
        cp workspace/dd-openfeature/build/libs/*.jar check_reports/jar-comparison-artifacts/candidate/
        exit 1
      fi
  after_script:
    - source .gitlab/gitlab-utils.sh
    - gitlab_section_start "collect-reports" "Collecting reports"
    - .gitlab/collect_reports.sh --destination ./check_reports
    - gitlab_section_end "collect-reports"
  artifacts:
    when: always
    paths:
      - ./check_reports
      - '.gradle/daemon/*/*.out.log'

.check_job:
  extends: .gradle_build
  needs: [ build ]
  stage: tests
  variables:
    CACHE_TYPE: "lib"
  script:
    - *gitlab_base_ref_params
    - ./gradlew --version
    - ./gradlew $GRADLE_TARGET -x spotlessCheck $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS
  after_script:
    - *container_info
    - *cgroup_info
    - source .gitlab/gitlab-utils.sh
    - gitlab_section_start "collect-reports" "Collecting reports"
    - .gitlab/collect_reports.sh --destination ./check_reports --move
    - .gitlab/collect_results.sh
    - gitlab_section_end "collect-reports"
  artifacts:
    when: always
    paths:
      - ./check_reports
      - ./results
      - '.gradle/daemon/*/*.out.log'
    reports:
      junit: results/*.xml
  retry:
    max: 2
    when:
      - unknown_failure
      - stuck_or_timeout_failure
      - runner_system_failure
      - unmet_prerequisites
      - scheduler_failure
      - data_integrity_failure

check_build_src:
  extends: .check_job
  needs: []
  variables:
    GRADLE_TARGET: ":buildSrc:build :build-logic:check"

check_base:
  extends: .check_job
  variables:
    GRADLE_TARGET: ":baseCheck"

check_inst:
  extends: .check_job
  parallel: 4
  variables:
    GRADLE_TARGET: ":instrumentationCheck"
    CACHE_TYPE: "inst"

check_smoke:
  extends: .check_job
  parallel: 4
  variables:
    GRADLE_TARGET: ":smokeCheck"
    CACHE_TYPE: "smoke"

check_profiling:
  extends: .check_job
  variables:
    GRADLE_TARGET: ":profilingCheck"

check_debugger:
  extends: .check_job
  variables:
    GRADLE_TARGET: ":debuggerCheck"

muzzle:
  extends: .gradle_build
  # needs:parallel:matrix limits this job to a specific build_tests combination.
  # Keep matrix vars exact and in build_tests declaration order:
  # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
  needs: &needs_build_tests_inst
    - job: build_tests
      parallel:
        matrix:
          - GRADLE_TARGET: ":instrumentationTest"
            CACHE_TYPE: "inst"
  stage: tests
  rules:
    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
      when: never
    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
      when: never
    - when: on_success
  parallel:
    matrix:
      - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"]
  variables:
    CACHE_TYPE: "inst"
  script:
    - export SKIP_BUILDSCAN="true"
    - ./gradlew --version
    - ./gradlew :runMuzzle -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS
  after_script:
    - *container_info
    - *cgroup_info
    - *set_datadog_api_keys
    - source .gitlab/gitlab-utils.sh
    - gitlab_section_start "collect-reports" "Collecting reports"
    - .gitlab/collect_reports.sh
    - .gitlab/collect_results.sh
    - .gitlab/upload_ciapp.sh $CACHE_TYPE
    - gitlab_section_end "collect-reports"
  artifacts:
    when: always
    paths:
      - ./reports
      - ./results
      - '.gradle/daemon/*/*.out.log'
    reports:
      junit: results/*.xml

muzzle-dep-report:
  extends: .gradle_build
  needs: *needs_build_tests_inst
  stage: tests
  rules:
    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
      when: never
    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
      when: never
    - when: on_success
  variables:
    CACHE_TYPE: "inst"
  script:
    - export SKIP_BUILDSCAN="true"
    - ./gradlew --version
    - ./gradlew :dd-java-agent:instrumentation:aggregateMuzzleReports $GRADLE_ARGS
  after_script:
    - *container_info
    - *cgroup_info
    - .gitlab/collect_muzzle_deps.sh
  artifacts:
    when: always
    paths:
      - ./reports
      - '.gradle/daemon/*/*.out.log'

# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry
# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards
.prepare_test_env: &prepare_test_env
  - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER")
  - '[ ! -e pretest.env ] || rm pretest.env'
  - |
    for VARIABLE in "${gitlabVariables[@]}"
    do
      echo "export $VARIABLE=${!VARIABLE}" >> pretest.env
      unset "$VARIABLE"
    done

.restore_pretest_env: &restore_pretest_env
  - source pretest.env

.test_job_common:
  extends: .gradle_build
  image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm
  variables:
    <<: *tier_m_variables
    GRADLE_PARAMS: "-PskipFlakyTests"
    CONTINUE_ON_FAILURE: "false"
    TEST_JVM_ARGS: ""
    TESTCONTAINERS_CHECKS_DISABLE: "true"
    TESTCONTAINERS_RYUK_DISABLED: "true"
    TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/"
    # CI pull policy: allow slow image fetching without overriding local file-based settings.
    TESTCONTAINERS_PULL_TIMEOUT: "240"
    TESTCONTAINERS_PULL_PAUSE_TIMEOUT: "60"
    JETTY_AVAILABLE_PROCESSORS: 4 # Jetty incorrectly calculates processor count in containers
  script:
    - *gitlab_base_ref_params
    - >
      if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
      then
      export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
      fi
    - *prepare_test_env
    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='$TEST_JVM_ARGS -Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMinHeapSize=128M -Ddatadog.forkedMaxHeapSize=1024M"
    - ./gradlew --version
    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
  after_script:
    - *restore_pretest_env
    - *set_datadog_api_keys
    - *container_info
    - *cgroup_info
    - source .gitlab/gitlab-utils.sh
    - gitlab_section_start "collect-reports" "Collecting reports"
    - .gitlab/collect_reports.sh
    - if [ "$PROFILE_TESTS" == "true" ]; then .gitlab/collect_profiles.sh; fi
    - .gitlab/collect_results.sh
    - .gitlab/upload_ciapp.sh $CACHE_TYPE $testJvm
    - gitlab_section_end "collect-reports"
    - .gitlab/count_tests.sh "$GRADLE_TARGET" "$testJvm" "./results" "./test_counts_${CI_JOB_ID}.json"
    - export TEST_COUNTS_S3_PREFIX="test-counts/${CI_PIPELINE_ID}"
    - export TEST_COUNTS_FILE="./test_counts_${CI_JOB_ID}.json"
    # Use logical job identity in S3 so retried jobs overwrite stale attempts.
    - export TEST_COUNTS_S3_FILE="test_counts_${CI_JOB_NAME_SLUG}_${CI_NODE_INDEX}-${CI_NODE_TOTAL}.json"
    - export TEST_COUNTS_S3_URI="s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/${TEST_COUNTS_S3_FILE}"
    - echo "Uploading ${TEST_COUNTS_FILE} to ${TEST_COUNTS_S3_URI}"
    - aws s3 cp "$TEST_COUNTS_FILE" "$TEST_COUNTS_S3_URI" --only-show-errors
    - URL_ENCODED_JOB_NAME=$(jq -rn --arg x "$CI_JOB_NAME" '$x|@uri')
    - echo -e "${TEXT_BOLD}${TEXT_YELLOW}See test results in Datadog:${TEXT_CLEAR} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}%20%40ci.job.name%3A%22${URL_ENCODED_JOB_NAME}%22"
  artifacts:
    when: always
    paths:
      - ./reports.tar
      - ./profiles.tar
      - ./results
      - './test_counts_*.json'
      - '.gradle/daemon/*/*.out.log'
    reports:
      junit: results/*.xml
  retry:
    max: 2
    when:
      - unknown_failure
      - stuck_or_timeout_failure
      - runner_system_failure
      - unmet_prerequisites
      - scheduler_failure
      - data_integrity_failure

.test_job_amd64:
  extends: .test_job_common
  tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers
  needs: [ build_tests ]
  stage: tests
  rules:
    # Protected branches (master/mq/gh-readonly): all JVMs run unconditionally
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
      when: on_success
    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
      when: on_success
    # Enable for default test JVMs or for NON_DEFAULT_JVMS
    - if: '$NON_DEFAULT_JVMS == "true"'
      when: on_success
    # [ci: NON_DEFAULT_JVMS] with any other parameters in the commit message.
    - if: '$CI_COMMIT_MESSAGE =~ /\[ci:[^\]]*\bNON_DEFAULT_JVMS\b[^\]]*\]/'
      when: on_success
    - if: '$testJvm =~ $DEFAULT_TEST_JVMS'
      when: on_success

.test_job_arm64:
  extends: .test_job_common
  tags: [ "docker-in-docker:arm64" ]
  stage: tests-arm64
  # Use the amd64 build only as a compilation gate. Do not download its platform-specific artifacts.
  needs:
    - job: build
      artifacts: false
  variables:
    DEFAULT_TEST_JVMS: /^(11|17|21|25|tip)$/
    TEST_JVM_ARGS: "-Xshare:off"
  rules:
    # Java 8 is excluded from the arm64 test matrix for all distributions.
    # Java 8 JVMs are very slow on AARCH64.
    - if: '$testJvm =~ /^(8|oracle8|zulu8|semeru8|ibm8)$/'
      when: never
    # Java 11 is opt-in on arm64; HotSpot has very slow class retransformation.
    # Vendor JVMs still follow the non-default JVM selection rules below.
    - if: '$testJvm == "11" && $CI_COMMIT_MESSAGE =~ /\[ci:[^\]]*\bJAVA11\b[^\]]*\]/'
      when: on_success
      allow_failure: true
    - if: '$testJvm =~ /11$/ && $CI_COMMIT_MESSAGE !~ /\[ci:[^\]]*\bJAVA11\b[^\]]*\]/'
      when: never
    # arm64 tests are newly introduced to the merge queue and master. Keep them
    # non-blocking (allow_failure) for now so we can collect stability stats and
    # fix flaky/failing jobs without blocking the whole team. Remove allow_failure
    # once the arm64 suite is proven stable.
    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
      when: on_success
      allow_failure: true
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
      allow_failure: true
    # Enable non-default JVMs on demand.
    - if: '$NON_DEFAULT_JVMS == "true"'
      when: on_success
      allow_failure: true
    # On feature branches/PRs, [ci: ARM64] in the commit message runs the default JVM subset,
    # and [ci: ARM64, NON_DEFAULT_JVMS] runs all eligible JVMs. [ci: NON_DEFAULT_JVMS] alone only affects amd64.
    - if: '$CI_COMMIT_MESSAGE =~ /\[ci:[^\]]*\bARM64\b[^\]]*\]/ && $CI_COMMIT_MESSAGE =~ /\[ci:[^\]]*\bNON_DEFAULT_JVMS\b[^\]]*\]/'
      when: on_success
      allow_failure: true
    - if: '$CI_COMMIT_MESSAGE =~ /\[ci:[^\]]*\bARM64\b[^\]]*\]/ && $testJvm =~ $DEFAULT_TEST_JVMS'
      when: on_success
      allow_failure: true
    # Otherwise keep the default JVM subset available for manual runs.
    - if: '$testJvm =~ $DEFAULT_TEST_JVMS'
      when: manual
      allow_failure: true
  cache:
    - key: dependency-$CACHE_TYPE
      paths:
        - .gradle/wrapper
        - .gradle/caches
        - .gradle/notifications
        - .mvn/caches
      policy: pull
      fallback_keys:
        - dependency-base
        - dependency-lib
      unprotect: true
  before_script:
    - git config --global --add safe.directory "$CI_PROJECT_DIR"
    - !reference [.gradle_build, before_script]

.test_job_with_test_agent_common:
  variables:
    CI_USE_TEST_AGENT: "true"
    CI_AGENT_HOST: local-agent
  services:
    - name: registry.ddbuild.io/images/mirror/dd-apm-test-agent/ddapm-test-agent:v1.64.1
      alias: local-agent
      variables:
        LOG_LEVEL: "DEBUG"
        TRACE_LANGUAGE: "java"
        DD_SUPPRESS_TRACE_PARSE_ERRORS: "true"
        DD_POOL_TRACE_CHECK_FAILURES: "true"
        DD_DISABLE_ERROR_RESPONSES: "true"
        ENABLED_CHECKS: "trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service"
  script:
    - !reference [.test_job_common, script]
    - .gitlab/check_test_agent_results.sh

.test_job_amd64_with_test_agent:
  extends:
    - .test_job_amd64
    - .test_job_with_test_agent_common

.test_job_arm64_with_test_agent:
  extends:
    - .test_job_arm64
    - .test_job_with_test_agent_common

agent_integration_tests:
  extends: .test_job_amd64
  tags: [ "arch:amd64" ]
  variables:
    testJvm: "8"
    CI_AGENT_HOST: local-agent
    GRADLE_TARGET: "traceAgentTest"
    CACHE_TYPE: "base"
  services:
    - name: registry.ddbuild.io/images/mirror/datadog/agent:7.40.1
      alias: local-agent
      variables:
        DD_APM_ENABLED: "true"
        DD_BIND_HOST: "0.0.0.0"
        DD_HOSTNAME: "local-agent"
        DD_API_KEY: "invalid_key_but_this_is_fine"

test_base:
  extends: .test_job_amd64
  # needs:parallel:matrix limits this job to a specific build_tests combination.
  # Keep matrix vars exact and in build_tests declaration order:
  # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
  needs:
    - job: build_tests
      parallel:
        matrix:
          - GRADLE_TARGET: ":baseTest"
            CACHE_TYPE: "base"
  variables:
    GRADLE_TARGET: ":baseTest"
    CACHE_TYPE: "base"
  parallel:
    matrix: *test_matrix_4
  script:
    - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi
    - !reference [.test_job_common, script]

test_base_arm64:
  extends: .test_job_arm64
  variables:
    GRADLE_TARGET: ":baseTest"
    CACHE_TYPE: "base"
  parallel:
    matrix: *test_matrix_4

test_inst:
  extends: .test_job_amd64_with_test_agent
  needs: *needs_build_tests_inst
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: ":instrumentationTest"
    CACHE_TYPE: "inst"
  parallel:
    matrix: *test_matrix_8

test_inst_arm64:
  extends: .test_job_arm64_with_test_agent
  # Instrumentation tests are slower on arm64 runners than the 1h default allows.
  timeout: 75 minutes
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: ":instrumentationTest"
    CACHE_TYPE: "inst"
  parallel:
    matrix: *test_matrix_8

test_inst_latest:
  extends: .test_job_amd64_with_test_agent
  # needs:parallel:matrix limits this job to a specific build_tests combination.
  # Keep matrix vars exact and in build_tests declaration order:
  # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
  needs:
    - job: build_tests
      parallel:
        matrix:
          - GRADLE_TARGET: ":instrumentationLatestDepTest"
            CACHE_TYPE: "latestdep"
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: ":instrumentationLatestDepTest"
    CACHE_TYPE: "latestdep"
  parallel:
    matrix:
      - testJvm: ["8", "17", "21", "25", "tip"]
        # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
        # This emulates "parallel" by including it in the matrix
        CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]

test_inst_latest_arm64:
  extends: .test_job_arm64_with_test_agent
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: ":instrumentationLatestDepTest"
    CACHE_TYPE: "latestdep"
  parallel:
    matrix:
      - testJvm: ["17", "21", "25", "tip"]
        # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
        # This emulates "parallel" by including it in the matrix
        CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]

test_flaky:
  extends: .test_job_amd64_with_test_agent
  variables:
    GRADLE_PARAMS: "-PrunFlakyTests"
    CACHE_TYPE: "smoke"
    testJvm: "8"
    CONTINUE_ON_FAILURE: "true"
  rules:
    - *master_only
    - if: $RUN_FLAKY_TESTS == "true"
      when: on_success
  parallel:
    matrix:
      - GRADLE_TARGET: [":baseTest", ":smokeTest", ":debuggerTest"]
        # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
        # This emulates "parallel" by including it in the matrix
        CI_SPLIT: [ "1/4", "2/4", "3/4", "4/4" ]

test_flaky_inst:
  extends: .test_job_amd64
  needs: *needs_build_tests_inst
  variables:
    GRADLE_TARGET: ":instrumentationTest"
    GRADLE_PARAMS: "-PrunFlakyTests"
    CACHE_TYPE: "inst"
    testJvm: "8"
    CONTINUE_ON_FAILURE: "true"
  rules:
    - *master_only
    - if: $RUN_FLAKY_TESTS == "true"
      when: on_success
  parallel: 6

test_profiling:
  extends: .test_job_amd64
  # needs:parallel:matrix limits this job to a specific build_tests combination.
  # Keep matrix vars exact and in build_tests declaration order:
  # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
  needs:
    - job: build_tests
      parallel:
        matrix:
          - GRADLE_TARGET: ":profilingTest"
            CACHE_TYPE: "profiling"
  variables:
    GRADLE_TARGET: ":profilingTest"
    CACHE_TYPE: "profiling"
  parallel:
    matrix: *test_matrix

test_profiling_arm64:
  extends: .test_job_arm64
  variables:
    GRADLE_TARGET: ":profilingTest"
    CACHE_TYPE: "profiling"
  parallel:
    matrix: *test_matrix

# specific jvms list for debugger project because J9-based JVMs have issues with local vars
# so need to test at least against one J9-based JVM
test_debugger:
  extends: .test_job_amd64
  variables:
    GRADLE_TARGET: ":debuggerTest"
    CACHE_TYPE: "base"
    DEFAULT_TEST_JVMS: /^(8|11|17|21|25|tip|semeru8)$/
  parallel:
    matrix: *test_matrix
  # avoid running coverage for semeru8, semeru11, semeru17 and ibm8 as some tests are disabled and therefore cannot reach the
  # exepected coverage
  script:
    - if [[ "$testJvm" != "semeru8" && "$testJvm" != "semeru11" && "$testJvm" != "semeru17" && "$testJvm" != "ibm8" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi
    - !reference [.test_job_common, script]

test_debugger_arm64:
  extends: .test_job_arm64
  variables:
    GRADLE_TARGET: ":debuggerTest"
    CACHE_TYPE: "base"
  parallel:
    matrix: *test_matrix
  # avoid running coverage for semeru11 and semeru17 as some tests are disabled and therefore cannot reach the
  # expected coverage
  script:
    - if [[ "$testJvm" != "semeru11" && "$testJvm" != "semeru17" ]]; then export GRADLE_PARAMS="$GRADLE_PARAMS -PcheckCoverage"; fi
    - !reference [.test_job_common, script]

test_smoke:
  extends: .test_job_amd64_with_test_agent
  # needs:parallel:matrix limits this job to a specific build_tests combination.
  # Keep matrix vars exact and in build_tests declaration order:
  # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
  needs: &needs_build_tests_smoke
    - job: build_tests
      parallel:
        matrix:
          - GRADLE_TARGET: ":smokeTest"
            CACHE_TYPE: "smoke"
            MAVEN_OPTS: "-Xms256M -Xmx1024M"
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: "stageMainDist :smokeTest"
    GRADLE_PARAMS: "-PskipFlakyTests"
    CACHE_TYPE: "smoke"
  parallel:
    matrix: *test_matrix_8

test_smoke_arm64:
  extends: .test_job_arm64_with_test_agent
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: "stageMainDist :smokeTest"
    GRADLE_PARAMS: "-PskipFlakyTests"
    CACHE_TYPE: "smoke"
  parallel:
    matrix: *test_matrix_8

test_ssi_smoke:
  extends: .test_job_amd64
  needs: *needs_build_tests_smoke
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
      when: on_success
    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
      when: on_success
    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
      when: on_success
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: "stageMainDist :smokeTest"
    CACHE_TYPE: "smoke"
    DD_INJECT_FORCE: "true"
    DD_INJECTION_ENABLED: "tracer"
  parallel:
    matrix: *test_matrix_8

test_ssi_smoke_arm64:
  extends: .test_job_arm64
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: "stageMainDist :smokeTest"
    CACHE_TYPE: "smoke"
    DD_INJECT_FORCE: "true"
    DD_INJECTION_ENABLED: "tracer"
  parallel:
    matrix: *test_matrix_8

test_smoke_graalvm:
  extends: .test_job_amd64
  needs: *needs_build_tests_smoke
  tags: [ "arch:amd64" ]
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test"
    CACHE_TYPE: "smoke"
    CI_NO_SPLIT: "true"
    NON_DEFAULT_JVMS: "true"
  parallel:
    matrix:
      - testJvm: ["graalvm17", "graalvm21", "graalvm25"]

test_smoke_graalvm_arm64:
  extends: .test_job_arm64
  tags: [ "arch:arm64" ]
  variables:
    <<: *tier_l_variables
    GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test :dd-smoke-test:quarkus-native:test"
    CACHE_TYPE: "smoke"
    CI_NO_SPLIT: "true"
    # Treat the GraalVM JVMs as this job's defaults, so it follows the arm64 rules like the other arm64 jobs:
    # automatic on master and the merge queue, manual on PRs unless the commit message contains [ci: ARM64].
    DEFAULT_TEST_JVMS: /^graalvm(17|21|25)$/
  parallel:
    matrix:
      - testJvm: ["graalvm17", "graalvm21", "graalvm25"]

test_smoke_semeru8_debugger:
  extends: .test_job_amd64
  needs: *needs_build_tests_smoke
  tags: [ "arch:amd64" ]
  variables:
    GRADLE_TARGET: "stageMainDist dd-smoke-tests:debugger-integration-tests:test"
    CACHE_TYPE: "smoke"
    NON_DEFAULT_JVMS: "true"
    testJvm: "semeru8"

aggregate_test_counts:
  image: ${BUILDER_IMAGE_REPO}:${BUILDER_IMAGE_VERSION_PREFIX}base
  stage: test-summary
  # Keep stage ordering, but prevent GitLab from downloading all previous-stage artifacts.
  dependencies: []
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
      when: on_success
    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
      when: on_success
    - if: '$CI_COMMIT_BRANCH'
      when: on_success
  script:
    - *set_datadog_api_keys
    - export TEST_COUNTS_S3_PREFIX="test-counts/${CI_PIPELINE_ID}"
    - mkdir -p ./test_counts_aggregate
    - echo "Downloading test count files from s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/"
    - aws s3 cp "s3://${TEST_COUNTS_S3_BUCKET}/${TEST_COUNTS_S3_PREFIX}/" ./test_counts_aggregate/ --recursive --exclude "*" --include "test_counts_*.json" --only-show-errors
    - find ./test_counts_aggregate -name 'test_counts_*.json' -type f -print | sort
    - .gitlab/aggregate_test_counts.sh ./test_counts_aggregate
  artifacts:
    when: always
    paths:
      - test_counts_aggregate/test_counts_*.json
      - test_counts_summary.json
      - test_counts_report.md

deploy_to_profiling_backend:
  stage: publish
  needs: [ build ]
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      when: on_success
    - if: '$CI_COMMIT_TAG =~ /^v.*/'
      when: on_success
    - when: manual
      allow_failure: true
  trigger:
    project: DataDog/profiling-backend
    branch: dogfooding
  variables:
    UPSTREAM_PACKAGE_JOB: $BUILD_JOB_NAME
    UPSTREAM_PACKAGE_JOB_ID: $BUILD_JOB_ID
    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
    UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
    UPSTREAM_BRANCH: $CI_COMMIT_BRANCH
    UPSTREAM_TAG: $CI_COMMIT_TAG

deploy_to_di_backend:manual:
  stage: publish
  needs: [ build ]
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - when: manual
      allow_failure: true
  trigger:
    project: DataDog/debugger-demos
    branch: main
  variables:
    UPSTREAM_PACKAGE_JOB: build
    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
    UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
    UPSTREAM_BRANCH: $CI_COMMIT_BRANCH
    UPSTREAM_TAG: $CI_COMMIT_TAG
    UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
    UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA

deploy_to_reliability_env:
  needs: [ build ]

# If the deploy_to_maven_central job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match.
deploy_to_maven_central:
  extends: .gradle_build
  stage: publish-release-artifacts
  needs:
    - job: build
    - job: system_tests
      optional: true
  variables:
    CACHE_TYPE: "lib"
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    # Do not deploy release candidate versions
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
      allow_failure: true
    - when: manual
      allow_failure: true
  script:
    - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
    - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
    - ./gradlew publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS
  artifacts:
    paths:
      - 'workspace/dd-java-agent/build/libs/*.jar'
      - 'workspace/dd-trace-api/build/libs/*.jar'
      - 'workspace/dd-trace-ot/build/libs/*.jar'
      - 'workspace/dd-openfeature/build/libs/*.jar'

deploy_snapshot_with_ddprof_snapshot:
  extends: .gradle_build
  stage: publish
  needs: [ build ]
  variables:
    CACHE_TYPE: "lib"
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    # Manual trigger only - for testing with ddprof snapshot versions
    - when: manual
      allow_failure: true
  script:
    - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
    - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
    - echo "Publishing dd-trace-java snapshot with ddprof snapshot dependency"
    - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID -PddprofUseSnapshot publishToSonatype -PskipTests $GRADLE_ARGS
  artifacts:
    paths:
      - 'workspace/dd-java-agent/build/libs/*.jar'
      - 'workspace/dd-trace-api/build/libs/*.jar'
      - 'workspace/dd-trace-ot/build/libs/*.jar'
      - 'workspace/dd-openfeature/build/libs/*.jar'

deploy_artifacts_to_github:
  stage: publish-release-artifacts
  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
  tags: [ "arch:amd64" ]
  id_tokens:
    DDOCTOSTS_ID_TOKEN:
      aud: dd-octo-sts
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
  # Requires the deploy_to_maven_central job to have run first (the UP-TO-DATE gradle check across jobs is broken)
  # This will deploy the artifacts built from the publishToSonatype task to the GitHub release
  needs:
    - job: deploy_to_maven_central
      # The deploy_to_maven_central job is not run for release candidate versions
      optional: true
  before_script:
    - dd-octo-sts version
    - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release
    - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > github-token.txt
  script:
    - gh auth login --with-token < github-token.txt
    - gh auth status
    - export VERSION=${CI_COMMIT_TAG##v} # remove "v" from front of tag to get version
    - cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar workspace/dd-java-agent/build/libs/dd-java-agent.jar # upload two filenames
    - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent.jar
    - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar
    - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar
    - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar
    - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-openfeature/build/libs/dd-openfeature-${VERSION}.jar
  after_script:
    - dd-octo-sts revoke -t $(cat github-token.txt)
  retry:
    max: 2
    when: always

requirements_json_test:
  rules:
    - when: on_success
  variables:
    REQUIREMENTS_BLOCK_JSON_PATH: "metadata/requirements-block.json"
    REQUIREMENTS_ALLOW_JSON_PATH: "metadata/requirements-allow.json"

package-oci:
  needs: [ build ]

override_verify_maven_central:
  image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release
  stage: publish
  needs: [ ]
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - when: manual
      allow_failure: true
  script:
    - touch OVERRIDE_MAVEN_VERIFY
  cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs
    - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL
      paths:
        - OVERRIDE_MAVEN_VERIFY
      policy: push
      unprotect: true

# Verify Maven Central deployment is publicly available before publishing OCI images
verify_maven_central_deployment:
  image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release
  stage: publish-release-artifacts
  needs: [ deploy_to_maven_central ]
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
  cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs
    - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL
      paths:
        - OVERRIDE_MAVEN_VERIFY
      policy: pull
      unprotect: true
  script:
    - if [ -f OVERRIDE_MAVEN_VERIFY ]; then echo "SKIPPING MAVEN VERIFICATION"; exit 0; fi
    - |
      export VERSION=${CI_COMMIT_TAG##v}
      ARTIFACT_URLS=(
        "https://repo1.maven.org/maven2/com/datadoghq/dd-java-agent/${VERSION}/dd-java-agent-${VERSION}.jar"
        "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-api/${VERSION}/dd-trace-api-${VERSION}.jar"
        "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-ot/${VERSION}/dd-trace-ot-${VERSION}.jar"
        "https://repo1.maven.org/maven2/com/datadoghq/dd-openfeature/${VERSION}/dd-openfeature-${VERSION}.jar"
      )
      # Try once immediately (fast path on job retry), then every 5 mins for 45 mins.
      TRY=0
      MAX_TRIES=10
      RETRY_DELAY=300
      while [ $TRY -lt $MAX_TRIES ]; do
        ARTIFACTS_AVAILABLE=true
        for URL in "${ARTIFACT_URLS[@]}"; do
          if ! curl --location --fail --silent --show-error -I "$URL"; then
            ARTIFACTS_AVAILABLE=false
            break
          fi
        done
        if [ "$ARTIFACTS_AVAILABLE" = true ]; then
          break
        fi
        TRY=$((TRY + 1))
        if [ $TRY -eq $MAX_TRIES ]; then
          echo "The release was not available after 45 mins. Manually re-run the job to try again."
          exit 1
        fi
        sleep $RETRY_DELAY
      done

publishing-gate:
  stage: publish-release-artifacts
  needs:
    - job: verify_maven_central_deployment
      optional: true # Required for releases only

configure_system_tests:
  variables:
    SYSTEM_TESTS_REF: "main" # system tests are pinned on release branches only
    SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection"

create_key:
  stage: generate-signing-key
  when: manual
  needs: [ ]
  variables:
    PROJECT_NAME: "dd-trace-java"
    EXPORT_TO_KEYSERVER: "true"
  image: $REGISTRY/ci/agent-key-management-tools/gpg:1
  script:
    - /create.sh
  artifacts:
    expire_in: 13 mos
    paths:
      - pubkeys

validate_supported_configurations_v2_local_file:
  extends: .validate_supported_configurations_v2_local_file
  variables:
    LOCAL_JSON_PATH: "metadata/supported-configurations.json"
    BACKFILLED: "false"

update_central_configurations_version_range_v2:
  extends: .update_central_configurations_version_range_v2
  variables:
    LOCAL_REPO_NAME: "dd-trace-java"
    LOCAL_JSON_PATH: "metadata/supported-configurations.json"
    LANGUAGE_NAME: "java"
    MULTIPLE_RELEASE_LINES: "false"
    SEND_ALIASES: "true"