Reliable and automated base image build process

.dockerignore

@@ -10,6 +10,10 @@ __pycache__
 scenario_groups.yml
 /binaries/**/.git/config
 
+# Materialized build context for utils/base_images/build_base_images.py
+/.base_image_build/
+/.base_images_build/
+
 # dotnet
 **/dotnet/**/bin/
 **/dotnet/**/obj/

.github/workflows/ci.yml

@@ -86,7 +86,6 @@ jobs:
       _build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }}
       _build_lambda_proxy_image: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.rebuild_lambda_proxy) }}
       _build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }}
-      _build_weblog_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') && matrix.library == 'python' || contains(github.event.pull_request.labels.*.name, 'build-php-base-images') && matrix.library == 'php' || contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') && matrix.library == 'nodejs' }}
       _enable_replay_scenarios: true
       _system_tests_dev_mode: ${{ matrix.version == 'dev' }}
       _system_tests_library_target_branch_map: ${{ needs.compute_libraries_and_scenarios.outputs.target-branch-map }}

.github/workflows/compute-workflow-parameters.yml

@@ -52,11 +52,6 @@ on:
         default: ''
         required: false
         type: string
-      _build_weblog_base_images:
-        description: "Shall we build weblog base images"
-        default: false
-        required: false
-        type: boolean
 
     # Map the workflow outputs to job outputs
     outputs:
@@ -142,7 +137,6 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
-          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --output $GITHUB_OUTPUT
     - name: log
       run: |
@@ -156,7 +150,6 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
-          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --format json | jq
 
     - name: Extract library target branch

.github/workflows/run-end-to-end.yml

@@ -62,11 +62,6 @@ on:
         default: false
         required: false
         type: boolean
-      _build_weblog_base_image:
-        description: "Shall we build the weblog base image (only valid if weblog build is local)"
-        default: false
-        required: false
-        type: boolean
       skip_empty_scenarios:
         description: "Skip scenarios that contains only xfail or irrelevant tests"
         default: false
@@ -147,11 +142,9 @@ jobs:
       if: inputs._build_lambda_proxy_image
       run: ./build.sh -i lambda-proxy
 
-    - name: Build weblog base image
-      if: inputs._build_weblog_base_image
-      run: |
-        source venv/bin/activate
-        ./utils/scripts/build-base-image.py ${{ inputs.library }} ${{ inputs.weblog }}
+    - name: Wait for weblog base image
+      if: github.repository == 'DataDog/system-tests'
+      run: python3 ./utils/scripts/wait-for-base-images.py ${{ inputs.library }} ${{ inputs.weblog }} --timeout 900 --poll-interval 30
 
     - name: Pull images
       uses: ./.github/actions/pull_images

.github/workflows/system-tests.yml

@@ -44,11 +44,6 @@ on:
         default: 'custom'
         required: false
         type: string
-      _build_weblog_base_images:
-        description: "Shall we rebuild weblog base images"
-        default: false
-        required: false
-        type: boolean
       _build_buddies_images:
         description: "Shall we build buddies images"
         default: false
@@ -154,7 +149,6 @@ jobs:
       binaries_artifact: ${{ inputs.binaries_artifact }}
       _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }}
       _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }}
-      _build_weblog_base_images: ${{ inputs._build_weblog_base_images }}
 
   parametric:
     needs:
@@ -215,14 +209,9 @@ jobs:
         run: ls -la binaries/
       - name: Export github token to a file
         run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt"
-      - name: Install runner
-        if: matrix.weblog.build_base_images
-        uses: ./.github/actions/install_runner
-      - name: Build weblog base images
-        if: matrix.weblog.build_base_images
-        run: |
-          source venv/bin/activate
-          ./utils/scripts/build-base-image.py ${{ inputs.library }} ${{ matrix.weblog.name }}
+      - name: Wait for weblog base image
+        if: github.repository == 'DataDog/system-tests'
+        run: python3 ./utils/scripts/wait-for-base-images.py ${{ inputs.library }} ${{ matrix.weblog.name }} --timeout 900 --poll-interval 30
       - name: Build weblog
         id: build
         run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog.name }} -s --github-token-file "$RUNNER_TEMP/github_token.txt"
@@ -277,6 +266,5 @@ jobs:
       _build_buddies_images: ${{ inputs._build_buddies_images }}
       _build_proxy_image: ${{ inputs._build_proxy_image }}
       _build_lambda_proxy_image: ${{ inputs._build_lambda_proxy_image }}
-      _build_weblog_base_image: ${{ matrix.job.build_weblog_base_image }}
       _enable_replay_scenarios: ${{ inputs._enable_replay_scenarios }}
       _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }}

.gitignore

@@ -74,6 +74,10 @@ version-check-output/
 
 tmp/
 
+# Materialized build context for utils/base_images/build_base_images.py
+/.base_image_build/
+/.base_images_build/
+
 # Go
 go.work*
 

.gitlab-ci.yml

@@ -390,8 +390,8 @@ build_ci_image:
 #   * signs every image copied during that run with ddsign (`--sign`), so
 #     already-mirrored images are not needlessly re-signed on every push.
 #
-# It never updates the lock file: mirror_images.lock.yaml is the committed
-# contract, regenerated by developers via update_mirror_images.py.
+# It regenerates the mirror files in the CI worktree to detect drift, but never
+# commits them. mirror_images.lock.yaml remains a committed contract.
 #
 # crane copies registry-to-registry over HTTPS (no Docker daemon needed), so this
 # runs on the framework runner image used for the scenario enumeration.
@@ -411,6 +411,9 @@ mirror_images:
     - job: build_ci_image
       artifacts: false
       optional: true
+    # The mirror lock can only resolve tags after this job has published them.
+    - job: build_base_images
+      artifacts: false
   before_script:
     # crane handles multi-arch manifest lists cleanly; mirror_images.py picks it up on PATH.
     - CRANE_VERSION=v0.20.2
@@ -426,7 +429,7 @@ mirror_images:
     - ln -sf /system-tests/venv venv
     - source venv/bin/activate
     - python -m pip install --quiet uv
-    # Fail if the committed image list is stale vs the scenarios (does not touch the lock).
+    # Fail if any committed mirror artifact is stale vs the scenarios and base-image lock.
     - python utils/scripts/update_mirror_images.py --skip-lock
     - |
       if ! git diff --exit-code -- mirror_images.yaml; then
@@ -445,3 +448,36 @@ mirror_images:
     - uv run --no-config --no-project --script "$MIRROR_IMAGES_URL" mirror --sign
   rules:
     - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null'
+
+# ──────────────────────────────────────────────
+# Rebuild weblog base images whose content changed.
+#
+# For every changed library with a docker-bake.hcl, computes a deterministic content-hash
+# tag per target and pushes it if missing (see utils/base_images/build_base_images.py).
+# Runs on every push, on every branch: it is idempotent (only pushes new tags, never
+# overwrites existing ones), so it is safe to build ahead of merge. Since new tags never
+# replace old ones. A stale committed base-images.lock.json makes the job fail after
+# publishing the missing images, with instructions for the follow-up lock update.
+# ──────────────────────────────────────────────
+
+build_base_images:
+  image: $CI_IMAGE
+  tags:
+    - docker-in-docker:amd64
+  needs:
+    - job: build_ci_image
+      artifacts: false
+      optional: true
+  stage: e2e
+  before_script:
+    - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text)
+    - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text)
+    - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin
+    - ln -sf /system-tests/venv venv
+    - source venv/bin/activate
+    # Needed by --changed-only for the merge-base (CI clones are shallow); fails open if missing.
+    - git fetch origin main
+  script:
+    - python utils/base_images/build_base_images.py --changed-only
+  rules:
+    - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null'

docs/CI/github-actions.md

@@ -100,5 +100,4 @@ Those parameters are used only by system-tests own CI
 | `_build_buddies_images`               | Shall we build buddies images                                        | boolean | false    | false      |
 | `_build_proxy_image`                  | Shall we build proxy image                                           | boolean | false    | false      |
 | `_build_lambda_proxy_image`           | Shall we build the lambda-proxy image                                | boolean | false    | false      |
-| `_build_weblog_base_images`           | Shall we build weblog base images                                    | boolean | false    | false      |
 | `_enable_replay_scenarios`            | Enable replay scenarios, should only be used in system-tests CI      | boolean | false    | false      |

docs/edit/update-docker-images.md

@@ -1,11 +1,32 @@
 Some of images used in system-tests are prebuild and used threw [hub.docker.com/datadog/system-tests](https://hub.docker.com/repository/docker/datadog/system-tests/).
 
-If you need to update them, you will need to follow those
-
-1. update the version in the tag for the image you've just modified (there should be 3 or 4 occurences in the code)
-2. create your PR, and add the relevant label to rebuild the image in the CI
-  * `build-python-base-images` for python weblogs
-  * `build-php-base-images` for PHP weblogs
-  * `build-nodejs-base-images` for Node.js weblogs
-  * `build-proxy-image` for proxy image
-3. just before merging your PR, ping somebody from Reliability & Performance team to push your image to hub.docker.com (`#apm-shared-testing` on slack)
+For weblog base images (nodejs, python, php), publishing and consumer selection use a
+two-step lock-file workflow:
+
+1. GitLab CI's `build_base_images` job (`utils/base_images/build_base_images.py`) runs on every push, on every
+   branch. For each target in a library's `docker-bake.hcl`, it derives the target's dependencies from the
+   `COPY` instructions in its `<name>.base.Dockerfile` (see `docs/understand/weblogs/weblog-metadata.md`
+   for the Dockerfile rules this relies on), computes a content hash of those dependencies, and, if a base
+   image tagged with that hash doesn't already exist on Docker Hub, builds and pushes it as
+   `<base tag>-<hash>`. If `utils/build/docker/base-images.lock.json` is stale, the job fails
+   only after it has published all missing images.
+2. After that publish job finishes, regenerate the lock and mirror artifacts:
+
+   ```sh
+   python utils/scripts/update-base-images-lock.py
+   python utils/scripts/update_mirror_images.py
+   ```
+
+   Commit `base-images.lock.json`, `mirror_images.yaml`, `mirror_images.lock.yaml`, and
+   `utils/build/docker/buildkitd.toml`. The lock maps stable aliases such as
+   `system_tests_base_nodejs_express4` to immutable content tags. Consumer Dockerfiles keep
+   the stable alias in `FROM`; they do not change when base content changes.
+3. GitHub Actions never builds these images itself. It resolves the consumer alias through
+   the committed lock and polls Docker Hub for that real image before building the weblog.
+
+The supported consumer build entrypoint is `./build.sh`, which supplies each locked image as
+a BuildKit named context. A raw `docker build` command does not resolve these aliases.
+
+For other prebuilt images (e.g. the proxy image), add the `build-proxy-image` label to your PR to force a
+rebuild in GitHub CI; then, just before merging, ping somebody from Reliability & Performance team to push
+your image to hub.docker.com (`#apm-shared-testing` on slack).

docs/understand/weblogs/weblog-metadata.md

@@ -122,3 +122,49 @@ produces `openai-js@6.0.0` and `openai-js@7.0.0`.
 `WeblogMetaData.load(library)` in `utils/_context/weblog_metadata.py` merges:
 1. Weblogs discovered from `*.Dockerfile` files in the library folder (default metadata).
 2. Explicit overrides from `weblog_metadata.yml`.
+
+## Base image dependencies
+
+Base images (built by the `build_base_images` CI job, `utils/base_images/build_base_images.py`) are
+declared in each library's `utils/build/docker/<library>/docker-bake.hcl`, one target per base
+image. There is no separate dependency list to maintain: for each target, the job parses the
+target's own `<name>.base.Dockerfile` and treats every `COPY` source as a dependency. This works
+because base Dockerfiles are required to follow a few rules that make that derivation
+unambiguous:
+
+- No `ADD` — use `COPY` for everything (no remote URLs).
+- Every `COPY` has exactly one source: `COPY [flags] <source> <dest>`.
+- The bake target's `context` is always the Dockerfile's own directory, so every `COPY` source
+  is a plain path relative to that directory.
+- A source may be a file, a glob, or a directory; glob sources are expanded, and directory
+  sources include every file beneath the directory.
+- No `RUN --mount` — a bind/cache/secret mount reads from a path the script can't see, so it
+  would silently escape the derived dependency list.
+
+(`COPY --from=<stage-or-image>` is unaffected: it isn't a local repository path, so it's skipped.)
+
+The job computes a content hash from normalized build arguments, the target's Dockerfile, and
+every git-tracked file under each derived dependency path, then pushes the base image to Docker
+Hub tagged `<base-tag>-<hash12>` if that tag doesn't already exist. Other Bake target fields are
+rejected until their hash semantics are explicitly defined.
+
+Consumer Dockerfiles use stable BuildKit context aliases in `FROM`, for example
+`FROM system_tests_base_nodejs_express4`. The versioned, sorted
+`utils/build/docker/base-images.lock.json` maps each alias to its immutable content tag.
+`./build.sh` reads the lock and supplies `--build-context
+<alias>=docker-image://<locked-reference>` to Buildx. Consequently, raw `docker build` is not a
+supported entrypoint for these consumers.
+
+As a safety net, before building, every derived dependency is hardlinked (or copied, if
+hardlinking isn't possible) into an isolated build context under `.base_images_build/`, and the
+image is built from that directory instead of the real one. This way, if the Dockerfile
+references a file the parser failed to recognize as a dependency, the build fails loudly
+("file not found") instead of silently succeeding against the full checkout — which would leave
+the tag's content hash stale without anyone noticing.
+
+GitHub Actions never builds these base images itself: `utils/scripts/wait-for-base-images.py`
+resolves the alias through the same lock and polls Docker Hub for the locked tag (with a timeout)
+before building the weblog, since GitLab CI is the only pipeline that builds and pushes them.
+After changing a base input, first let GitLab publish the prospective content tags. Then run
+`python utils/scripts/update-base-images-lock.py` and
+`python utils/scripts/update_mirror_images.py`, and commit the lock and mirror artifacts.