release: publish-gated append-only provenance ledger for src tarballs

.github/workflows/release-src.yml

@@ -70,6 +70,13 @@ jobs:
     permissions:
       contents: write
     steps:
+      # The provenance ledger tool lives in the repo — the publish job needs
+      # the checkout (the release legs run their own; publish previously had
+      # none).
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
+        with:
+          ruby-version: "3.2"
       # Diagnosability first: needs.<job>.result is evaluated server-side,
       # so echo exactly what the gate saw before doing anything else -- the
       # next publish-gate flake must be self-explaining in the job log.
@@ -88,6 +95,23 @@ jobs:
         run: |
           cat *.sha256 > SHA256SUMS
           rm -f *.sha256
+      # The provenance ledger (README § "Provenance ledger"): the
+      # cumulative, append-only record of every src-tarball digest this
+      # factory has published — what makes a runtime's recorded
+      # built_from.sources[].sha256 verifiable after the fact, even after
+      # the tarball re-rolls under the same version pin. Chained from the
+      # newest existing release's provenance.yaml (bootstrapped from every
+      # published SHA256SUMS while none carries one). Publish-gated: the
+      # ledger must extend the previous one as an exact prefix and cover
+      # every asset of this release, or the publish fails named.
+      - name: build + check the provenance ledger
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          tools/provenance --build \
+            --tag "${{ github.event_name == 'push' && github.ref_name || inputs.release_tag }}" \
+            --sums dist/SHA256SUMS \
+            --out dist/provenance.yaml
       # Record the published tag so the workflow_run follow-up
       # (runtime-pin-bump.yml) can read it back deterministically for both
       # trigger paths (tag push and manual dispatch).
@@ -104,3 +128,4 @@ jobs:
           files: |
             dist/*.tar.gz
             dist/SHA256SUMS
+            dist/provenance.yaml

README.md

@@ -118,6 +118,46 @@ identifiers to anything `tools/apply` writes into the tree. The local
 harness scripts tolerate bsdtar (macOS) without the clamp flags: their
 mirror is single-machine scratch consumed in the same run.
 
+## Provenance ledger
+
+The authoritative provenance of a runtime build input is the tarball's
+**content digest** (tebako-runtime-ruby records it as
+`built_from.sources[].sha256`). A release *name* is only fetch metadata:
+the rolling releases re-roll a version's tarball whenever its line's
+patches move, and a deleted release leaves a recorded name dangling — so
+the digest is what must be verifiable after the fact.
+
+Every release therefore carries **`provenance.yaml`**: the cumulative,
+append-only ledger of every `(asset, sha256)` pair this factory has ever
+published, each with the release that first carried those bytes
+(`Tfs::Provenance`, built by `tools/provenance --build` in release-src's
+publish job). Two publish-gate assertions make the invariant loud:
+
+- **append-only** — the previous release's ledger must be an exact prefix
+  of the new one: a re-rolled tarball appends an entry; a published digest
+  is never dropped, reordered, or re-attributed;
+- **completeness** — every asset of the release being published is
+  recorded.
+
+Either failure fails the publish with a named error. The ledger chains
+through the newest existing release (bootstrapped from every published
+SHA256SUMS while none carries one): **never delete factory releases** —
+deleting the newest one breaks the chain and blocks the next publish.
+
+Audit any recorded digest against the ledger (a runtime manifest's
+`built_from.sources[].sha256` resolves against the `provenance.yaml` of
+its own `built_from.release`, or of the newest release — the ledger only
+ever grows):
+
+```
+tools/provenance --verify <sha256>                      # newest release's ledger
+tools/provenance --verify <sha256> --ledger <file>      # offline / pinned ledger
+```
+
+An attested digest prints its asset and first-publishing release; an
+unattested one is a named error, exit 1 — loud, never a silent dangle.
+
+
 ## Placeholders
 
 The gem computes the tebako static library list (`MAINLIBS`) dynamically per
@@ -235,6 +275,14 @@ wires children with `autoload`):
   windows half of the patch set uncompiled); a missing cross compiler
   is a named error, not a silent host-tree smoke. Fails named:
   `FAIL <version> (<platform>): <objects> did not compile`.
+- `tools/provenance --build | --verify` — the src-tarball provenance
+  ledger (see "Provenance ledger"). `--build` (release-src's publish job;
+  `Tfs::ProvenanceFeed` over the releases API) chains the release's
+  `provenance.yaml` from the newest existing release's ledger — or
+  bootstraps it from every published SHA256SUMS when none carries one —
+  asserting append-only + completeness before the release is created.
+  `--verify <sha256>` is the audit lookup: resolves a digest to its asset
+  and first-publishing release, or fails named when unattested.
 
 CI: `.github/workflows/lint-patches.yml` lints every version on a matrix
 generated from versions.yml, then runs the same compile gate at change
@@ -251,7 +299,8 @@ forward from the previous release as sha256-verified copies
 (`tools/copy_asset`), so the release's asset set stays complete
 (`linux-gnu` stays the unsuffixed back-compat asset; msys ships
 `-msys-pass1`/`-msys-pass2`). Built artifacts are verified against the
-apply output, and the tarballs plus a `SHA256SUMS` are published to the
+apply output, and the tarballs plus a `SHA256SUMS` plus the publish-gated
+`provenance.yaml` ledger (see "Provenance ledger") are published to the
 release. Publish is gated on the compile-smoke matrix (roadmap 17.0; the
 v0.2.8 lesson — a patch release shipped apply-clean but uncompilable and
 broke every linux runtime leg): one representative leg per changed patch

spec/tfs/provenance_feed_spec.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "json"
+
+RSpec.describe Tfs::ProvenanceFeed do
+  let(:api) { "https://api.test/repos/x/y/releases" }
+  let(:download) { "https://api.test/dl" }
+
+  def release_json(tag, created_at, assets, draft: false)
+    { "tag_name" => tag, "created_at" => created_at, "draft" => draft,
+      "assets" => assets.map { |name| { "name" => name } } }
+  end
+
+  # A fetcher stub keyed by exact url, capturing the headers each call got
+  # and raising like the real Tfs::HttpGet on unknown urls.
+  def stub_fetcher(bodies, captured)
+    lambda do |url, headers:|
+      captured << [url, headers]
+      bodies.fetch(url) { raise Tfs::HttpGet::Error, "HTTP 404 from #{url}" }
+    end
+  end
+
+  it "lists published releases oldest-first with their asset names" do
+    captured = []
+    bodies = {
+      "#{api}?per_page=100&page=1" => JSON.generate([
+        release_json("v0.2.2", "2026-08-02T00:00:00Z", %w[SHA256SUMS provenance.yaml]),
+        release_json("v0.2.1", "2026-08-01T00:00:00Z", %w[SHA256SUMS]),
+        release_json("v0.2.3-draft", "2026-08-03T00:00:00Z", %w[SHA256SUMS], draft: true)
+      ])
+    }
+    feed = described_class.new(api_url: api, download_url: download, fetcher: stub_fetcher(bodies, captured))
+    expect(feed.releases).to eq([
+                                  { "tag" => "v0.2.1", "created_at" => "2026-08-01T00:00:00Z", "assets" => %w[SHA256SUMS] },
+                                  { "tag" => "v0.2.2", "created_at" => "2026-08-02T00:00:00Z", "assets" => %w[SHA256SUMS provenance.yaml] }
+                                ])
+    expect(captured.first[1]).to include("User-Agent" => Tfs::ProvenanceFeed::USER_AGENT)
+    expect(captured.first[1]).not_to have_key("Authorization")
+  end
+
+  it "paginates until a short page and sends the token when given" do
+    captured = []
+    full_page = Array.new(100) { |i| release_json("v0.0.#{i}", "2026-08-01T00:00:#{i.to_s.rjust(2, '0')}Z", %w[SHA256SUMS]) }
+    bodies = {
+      "#{api}?per_page=100&page=1" => JSON.generate(full_page),
+      "#{api}?per_page=100&page=2" => JSON.generate([release_json("v0.1.0", "2026-09-01T00:00:00Z", %w[SHA256SUMS])])
+    }
+    feed = described_class.new(api_url: api, download_url: download, token: "sekrit", fetcher: stub_fetcher(bodies, captured))
+    expect(feed.releases.size).to eq(101)
+    expect(feed.releases.last["tag"]).to eq("v0.1.0")
+    expect(captured.map { |(_url, headers)| headers["Authorization"] }.uniq).to eq(["Bearer sekrit"])
+  end
+
+  it "fails named on a malformed releases response" do
+    captured = []
+    feed = described_class.new(api_url: api, download_url: download,
+                               fetcher: stub_fetcher({ "#{api}?per_page=100&page=1" => "not json" }, captured))
+    expect { feed.releases }.to raise_error(Tfs::ProvenanceFeed::Error, /cannot parse the releases API response/)
+
+    feed = described_class.new(api_url: api, download_url: download,
+                               fetcher: stub_fetcher({ "#{api}?per_page=100&page=1" => "{}" }, captured))
+    expect { feed.releases }.to raise_error(Tfs::ProvenanceFeed::Error, /want an array/)
+  end
+
+  it "fetches an asset body and wraps failures naming the release and asset" do
+    captured = []
+    bodies = { "#{download}/v0.2.38/SHA256SUMS" => "abc  tfs-ruby-3.3.12-src.tar.gz\n" }
+    feed = described_class.new(api_url: api, download_url: download, fetcher: stub_fetcher(bodies, captured))
+    expect(feed.asset_body("v0.2.38", "SHA256SUMS")).to eq("abc  tfs-ruby-3.3.12-src.tar.gz\n")
+
+    expect { feed.asset_body("v0.2.38", "provenance.yaml") }
+      .to raise_error(Tfs::ProvenanceFeed::Error, /cannot fetch provenance\.yaml of release v0\.2\.38: HTTP 404/)
+  end
+end

spec/tfs/provenance_spec.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+RSpec.describe Tfs::Provenance do
+  let(:sha_a) { "a" * 64 }
+  let(:sha_b) { "b" * 64 }
+  let(:sha_c) { "c" * 64 }
+  let(:entry_a) { { "asset" => "tfs-ruby-3.3.12-src.tar.gz", "sha256" => sha_a, "first_release" => "v0.2.21" } }
+  let(:entry_b) { { "asset" => "tfs-ruby-3.3.12-src.tar.gz", "sha256" => sha_b, "first_release" => "v0.2.27" } }
+  let(:entry_c) { { "asset" => "tfs-ruby-3.4.10-src.tar.gz", "sha256" => sha_c, "first_release" => "v0.2.21" } }
+
+  describe ".parse + .serialize" do
+    it "round-trips entries through the published document form" do
+      entries = described_class.parse(described_class.serialize([entry_a, entry_b, entry_c]))
+      expect(entries).to eq([entry_a, entry_b, entry_c])
+    end
+
+    it "emits the header comment and the version/entries mapping" do
+      text = described_class.serialize([entry_a])
+      expect(text).to start_with("# tfs-ruby src-tarball provenance ledger")
+      expect(text).to include("version: 1\n")
+    end
+
+    it "rejects a document that is not exactly version/entries" do
+      expect { described_class.parse("entries: []\n") }
+        .to raise_error(Tfs::Provenance::Error, /must be a mapping of exactly version\/entries/)
+      expect { described_class.parse("version: 1\nentries: []\nextra: true\n") }
+        .to raise_error(Tfs::Provenance::Error, /must be a mapping of exactly version\/entries/)
+    end
+
+    it "rejects a ledger of an unknown version" do
+      expect { described_class.parse("version: 2\nentries: []\n") }
+        .to raise_error(Tfs::Provenance::Error, /version 2.*understands 1/)
+    end
+
+    it "rejects a non-array entries value" do
+      expect { described_class.parse("version: 1\nentries: {}\n") }
+        .to raise_error(Tfs::Provenance::Error, /entries must be an array/)
+    end
+
+    it "rejects an entry with a malformed asset, sha256, or first_release" do
+      bad_asset = entry_a.merge("asset" => "ruby-3.3.12.tar.gz")
+      expect { described_class.serialize([bad_asset]) }
+        .to raise_error(Tfs::Provenance::Error, /entry #1: malformed asset/)
+
+      bad_sha = entry_a.merge("sha256" => "abc")
+      expect { described_class.serialize([bad_sha]) }
+        .to raise_error(Tfs::Provenance::Error, /entry #1 \(tfs-ruby-3\.3\.12-src\.tar\.gz\): malformed sha256/)
+
+      bad_tag = entry_a.merge("first_release" => "3.3.12")
+      expect { described_class.serialize([bad_tag]) }
+        .to raise_error(Tfs::Provenance::Error, /malformed first_release/)
+
+      extra_key = entry_a.merge("note" => "hand-edited")
+      expect { described_class.serialize([extra_key]) }
+        .to raise_error(Tfs::Provenance::Error, /must be a mapping of exactly asset\/sha256\/first_release/)
+    end
+
+    it "rejects a duplicate (asset, sha256) pair, naming both positions" do
+      expect { described_class.serialize([entry_a, entry_b, entry_a]) }
+        .to raise_error(Tfs::Provenance::Error, /duplicate provenance entry.*tfs-ruby-3\.3\.12-src\.tar\.gz/)
+
+      text = "version: 1\nentries:\n" \
+             "- {asset: #{entry_a['asset']}, sha256: #{sha_a}, first_release: v0.2.21}\n" \
+             "- {asset: tfs-ruby-3.4.10-src.tar.gz, sha256: #{sha_c}, first_release: v0.2.21}\n" \
+             "- {asset: #{entry_a['asset']}, sha256: #{sha_a}, first_release: v0.2.22}\n"
+      expect { described_class.parse(text) }
+        .to raise_error(Tfs::Provenance::Error, /duplicate provenance entry #1 and #3/)
+    end
+  end
+
+  describe ".entries_from_sums" do
+    it "parses sha256sum lines, one entry per line pinned to the tag" do
+      entries = described_class.entries_from_sums("v0.2.38", "#{sha_a}  tfs-ruby-3.3.12-src.tar.gz\n#{sha_c}  tfs-ruby-3.4.10-src.tar.gz\n")
+      expect(entries).to eq([
+                              { "asset" => "tfs-ruby-3.3.12-src.tar.gz", "sha256" => sha_a, "first_release" => "v0.2.38" },
+                              { "asset" => "tfs-ruby-3.4.10-src.tar.gz", "sha256" => sha_c, "first_release" => "v0.2.38" }
+                            ])
+    end
+
+    it "accepts the binary-mode marker and an uppercase sum" do
+      entries = described_class.entries_from_sums("v0.2.38", "#{sha_a.upcase} *tfs-ruby-3.3.12-src.tar.gz\n")
+      expect(entries).to eq([entry_a.merge("first_release" => "v0.2.38")])
+    end
+
+    it "rejects a malformed line, naming the release" do
+      expect { described_class.entries_from_sums("v0.2.38", "#{sha_a}  not-a-tarball.txt\n") }
+        .to raise_error(Tfs::Provenance::Error, /malformed SHA256SUMS line in release v0\.2\.38/)
+    end
+
+    it "rejects a malformed tag" do
+      expect { described_class.entries_from_sums("latest", "#{sha_a}  tfs-ruby-3.3.12-src.tar.gz\n") }
+        .to raise_error(Tfs::Provenance::Error, /malformed release tag "latest"/)
+    end
+  end
+
+  describe ".chain" do
+    it "appends new digests and keeps recorded pairs' first_release" do
+      merged = described_class.chain([entry_a], [entry_b])
+      expect(merged).to eq([entry_a, entry_b])
+    end
+
+    it "does not duplicate a carried-forward pair" do
+      carry = entry_a.merge("first_release" => "v0.2.22")
+      merged = described_class.chain([entry_a], [carry])
+      expect(merged).to eq([entry_a])
+    end
+
+    it "is a prefix extension of the previous ledger" do
+      previous = [entry_a, entry_c]
+      merged = described_class.chain(previous, [entry_b])
+      expect(merged.first(previous.size)).to eq(previous)
+    end
+  end
+
+  describe ".assert_prefix" do
+    it "passes when the previous ledger is an exact prefix" do
+      expect(described_class.assert_prefix([entry_a], [entry_a, entry_b])).to be(true)
+      expect(described_class.assert_prefix([], [entry_a])).to be(true)
+    end
+
+    it "raises naming the dropped entry" do
+      expect { described_class.assert_prefix([entry_a, entry_b], [entry_a]) }
+        .to raise_error(Tfs::Provenance::Error, /not append-only: entry #2 .*was dropped/)
+    end
+
+    it "raises on an altered entry, showing what it became" do
+      replaced = entry_b.merge("first_release" => "v0.9.9")
+      expect { described_class.assert_prefix([entry_a, entry_b], [entry_a, replaced]) }
+        .to raise_error(Tfs::Provenance::Error, /not append-only: entry #2 .*became/)
+    end
+
+    it "raises on a reordered ledger" do
+      expect { described_class.assert_prefix([entry_a, entry_b], [entry_b, entry_a]) }
+        .to raise_error(Tfs::Provenance::Error, /not append-only: entry #1/)
+    end
+  end
+
+  describe ".assert_complete" do
+    it "passes when every published asset is recorded" do
+      expect(described_class.assert_complete([entry_a, entry_b], [entry_b])).to be(true)
+    end
+
+    it "raises naming the first unrecorded asset" do
+      expect { described_class.assert_complete([entry_a], [entry_b]) }
+        .to raise_error(Tfs::Provenance::Error, /incomplete: tfs-ruby-3\.3\.12-src\.tar\.gz.*\(release v0\.2\.27\) is published but unrecorded/)
+    end
+  end
+
+  describe ".verify" do
+    it "returns every entry attesting the digest" do
+      expect(described_class.verify([entry_a, entry_b], sha_a)).to eq([entry_a])
+    end
+
+    it "returns multiple entries when one digest ships under two assets" do
+      twin = entry_a.merge("asset" => "tfs-ruby-3.3.12-src-linux-musl.tar.gz")
+      expect(described_class.verify([entry_a, twin], sha_a)).to eq([entry_a, twin])
+    end
+
+    it "raises a named error on an unattested digest — loud, never a silent dangle" do
+      expect { described_class.verify([entry_a], sha_b, source: "release v0.2.38") }
+        .to raise_error(Tfs::Provenance::Error, /sha256 #{sha_b} is unattested in release v0\.2\.38.*provenance is unresolvable/)
+    end
+
+    it "rejects a malformed digest" do
+      expect { described_class.verify([entry_a], "ABC") }
+        .to raise_error(Tfs::Provenance::Error, /malformed sha256 "ABC"/)
+    end
+  end
+end

tools/lib/tfs.rb

@@ -18,4 +18,6 @@ module Tfs
   autoload :BuildPlan, "tfs/build_plan"
   autoload :SmokePlan, "tfs/smoke_plan"
   autoload :ReleaseCopier, "tfs/release_copier"
+  autoload :Provenance, "tfs/provenance"
+  autoload :ProvenanceFeed, "tfs/provenance_feed"
 end

tools/lib/tfs/http_get.rb

@@ -11,11 +11,13 @@ class Error < StandardError; end
 
     MAX_REDIRECTS = 5
 
-    def self.body(url)
+    def self.body(url, headers: {})
       uri = URI.parse(url)
       MAX_REDIRECTS.times do
         Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
-          response = http.request(Net::HTTP::Get.new(uri))
+          request = Net::HTTP::Get.new(uri)
+          headers.each { |name, value| request[name] = value }
+          response = http.request(request)
           case response
           when Net::HTTPSuccess
             return response.body