From f1c7a01a21a39849b37751caafb76c86ff07367b Mon Sep 17 00:00:00 2001 From: tebako-ci Date: Thu, 8 Oct 2026 23:44:26 +0800 Subject: [PATCH] release: publish-gated append-only provenance ledger for src tarballs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rolling releases re-roll a version's src tarball whenever its line's patches move, and a deleted release leaves a recorded release NAME dangling — so the durable provenance of a runtime build input is the tarball's content digest (already recorded downstream as built_from.sources[].sha256; the deterministic-roll rule makes it content-addressed). What was missing is a permanent, verifiable record that the factory published a digest. Every release now carries provenance.yaml: the cumulative, append-only ledger of every (asset, sha256) the factory has ever published, with the release that first carried those bytes. The publish job builds it with tools/provenance --build (chained from the newest existing release's ledger, bootstrapped from every published SHA256SUMS while none carries one) and gates the release on two assertions: - append-only: the previous ledger is an exact prefix of the new one — a re-rolled tarball appends; a published digest is never dropped, reordered, or re-attributed; - completeness: every asset of the release is recorded. Audit path: tools/provenance --verify resolves a recorded digest to its asset and first-publishing release (default: the newest release's ledger; --ledger for offline/pinned use); an unattested digest is a named error, exit 1 — loud, never a silent dangle. Tfs::HttpGet gains an optional headers kwarg (the releases API requires a User-Agent; GH_TOKEN/GITHUB_TOKEN authorizes when present). --- .github/workflows/release-src.yml | 25 ++++ README.md | 51 +++++++- spec/tfs/provenance_feed_spec.rb | 74 ++++++++++++ spec/tfs/provenance_spec.rb | 169 ++++++++++++++++++++++++++ tools/lib/tfs.rb | 2 + tools/lib/tfs/http_get.rb | 6 +- tools/lib/tfs/provenance.rb | 195 ++++++++++++++++++++++++++++++ tools/lib/tfs/provenance_feed.rb | 67 ++++++++++ tools/provenance | 124 +++++++++++++++++++ 9 files changed, 710 insertions(+), 3 deletions(-) create mode 100644 spec/tfs/provenance_feed_spec.rb create mode 100644 spec/tfs/provenance_spec.rb create mode 100644 tools/lib/tfs/provenance.rb create mode 100644 tools/lib/tfs/provenance_feed.rb create mode 100755 tools/provenance diff --git a/.github/workflows/release-src.yml b/.github/workflows/release-src.yml index 39e7136..91fde6a 100644 --- a/.github/workflows/release-src.yml +++ b/.github/workflows/release-src.yml @@ -70,6 +70,13 @@ jobs: permissions: contents: write steps: + # The provenance ledger tool lives in the repo — the publish job needs + # the checkout (the release legs run their own; publish previously had + # none). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 + with: + ruby-version: "3.2" # Diagnosability first: needs..result is evaluated server-side, # so echo exactly what the gate saw before doing anything else -- the # next publish-gate flake must be self-explaining in the job log. @@ -88,6 +95,23 @@ jobs: run: | cat *.sha256 > SHA256SUMS rm -f *.sha256 + # The provenance ledger (README § "Provenance ledger"): the + # cumulative, append-only record of every src-tarball digest this + # factory has published — what makes a runtime's recorded + # built_from.sources[].sha256 verifiable after the fact, even after + # the tarball re-rolls under the same version pin. Chained from the + # newest existing release's provenance.yaml (bootstrapped from every + # published SHA256SUMS while none carries one). Publish-gated: the + # ledger must extend the previous one as an exact prefix and cover + # every asset of this release, or the publish fails named. + - name: build + check the provenance ledger + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tools/provenance --build \ + --tag "${{ github.event_name == 'push' && github.ref_name || inputs.release_tag }}" \ + --sums dist/SHA256SUMS \ + --out dist/provenance.yaml # Record the published tag so the workflow_run follow-up # (runtime-pin-bump.yml) can read it back deterministically for both # trigger paths (tag push and manual dispatch). @@ -104,3 +128,4 @@ jobs: files: | dist/*.tar.gz dist/SHA256SUMS + dist/provenance.yaml diff --git a/README.md b/README.md index 62f302d..cfbb2ba 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,46 @@ identifiers to anything `tools/apply` writes into the tree. The local harness scripts tolerate bsdtar (macOS) without the clamp flags: their mirror is single-machine scratch consumed in the same run. +## Provenance ledger + +The authoritative provenance of a runtime build input is the tarball's +**content digest** (tebako-runtime-ruby records it as +`built_from.sources[].sha256`). A release *name* is only fetch metadata: +the rolling releases re-roll a version's tarball whenever its line's +patches move, and a deleted release leaves a recorded name dangling — so +the digest is what must be verifiable after the fact. + +Every release therefore carries **`provenance.yaml`**: the cumulative, +append-only ledger of every `(asset, sha256)` pair this factory has ever +published, each with the release that first carried those bytes +(`Tfs::Provenance`, built by `tools/provenance --build` in release-src's +publish job). Two publish-gate assertions make the invariant loud: + +- **append-only** — the previous release's ledger must be an exact prefix + of the new one: a re-rolled tarball appends an entry; a published digest + is never dropped, reordered, or re-attributed; +- **completeness** — every asset of the release being published is + recorded. + +Either failure fails the publish with a named error. The ledger chains +through the newest existing release (bootstrapped from every published +SHA256SUMS while none carries one): **never delete factory releases** — +deleting the newest one breaks the chain and blocks the next publish. + +Audit any recorded digest against the ledger (a runtime manifest's +`built_from.sources[].sha256` resolves against the `provenance.yaml` of +its own `built_from.release`, or of the newest release — the ledger only +ever grows): + +``` +tools/provenance --verify # newest release's ledger +tools/provenance --verify --ledger # offline / pinned ledger +``` + +An attested digest prints its asset and first-publishing release; an +unattested one is a named error, exit 1 — loud, never a silent dangle. + + ## Placeholders The gem computes the tebako static library list (`MAINLIBS`) dynamically per @@ -235,6 +275,14 @@ wires children with `autoload`): windows half of the patch set uncompiled); a missing cross compiler is a named error, not a silent host-tree smoke. Fails named: `FAIL (): did not compile`. +- `tools/provenance --build | --verify` — the src-tarball provenance + ledger (see "Provenance ledger"). `--build` (release-src's publish job; + `Tfs::ProvenanceFeed` over the releases API) chains the release's + `provenance.yaml` from the newest existing release's ledger — or + bootstraps it from every published SHA256SUMS when none carries one — + asserting append-only + completeness before the release is created. + `--verify ` is the audit lookup: resolves a digest to its asset + and first-publishing release, or fails named when unattested. CI: `.github/workflows/lint-patches.yml` lints every version on a matrix generated from versions.yml, then runs the same compile gate at change @@ -251,7 +299,8 @@ forward from the previous release as sha256-verified copies (`tools/copy_asset`), so the release's asset set stays complete (`linux-gnu` stays the unsuffixed back-compat asset; msys ships `-msys-pass1`/`-msys-pass2`). Built artifacts are verified against the -apply output, and the tarballs plus a `SHA256SUMS` are published to the +apply output, and the tarballs plus a `SHA256SUMS` plus the publish-gated +`provenance.yaml` ledger (see "Provenance ledger") are published to the release. Publish is gated on the compile-smoke matrix (roadmap 17.0; the v0.2.8 lesson — a patch release shipped apply-clean but uncompilable and broke every linux runtime leg): one representative leg per changed patch diff --git a/spec/tfs/provenance_feed_spec.rb b/spec/tfs/provenance_feed_spec.rb new file mode 100644 index 0000000..976f473 --- /dev/null +++ b/spec/tfs/provenance_feed_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require "json" + +RSpec.describe Tfs::ProvenanceFeed do + let(:api) { "https://api.test/repos/x/y/releases" } + let(:download) { "https://api.test/dl" } + + def release_json(tag, created_at, assets, draft: false) + { "tag_name" => tag, "created_at" => created_at, "draft" => draft, + "assets" => assets.map { |name| { "name" => name } } } + end + + # A fetcher stub keyed by exact url, capturing the headers each call got + # and raising like the real Tfs::HttpGet on unknown urls. + def stub_fetcher(bodies, captured) + lambda do |url, headers:| + captured << [url, headers] + bodies.fetch(url) { raise Tfs::HttpGet::Error, "HTTP 404 from #{url}" } + end + end + + it "lists published releases oldest-first with their asset names" do + captured = [] + bodies = { + "#{api}?per_page=100&page=1" => JSON.generate([ + release_json("v0.2.2", "2026-08-02T00:00:00Z", %w[SHA256SUMS provenance.yaml]), + release_json("v0.2.1", "2026-08-01T00:00:00Z", %w[SHA256SUMS]), + release_json("v0.2.3-draft", "2026-08-03T00:00:00Z", %w[SHA256SUMS], draft: true) + ]) + } + feed = described_class.new(api_url: api, download_url: download, fetcher: stub_fetcher(bodies, captured)) + expect(feed.releases).to eq([ + { "tag" => "v0.2.1", "created_at" => "2026-08-01T00:00:00Z", "assets" => %w[SHA256SUMS] }, + { "tag" => "v0.2.2", "created_at" => "2026-08-02T00:00:00Z", "assets" => %w[SHA256SUMS provenance.yaml] } + ]) + expect(captured.first[1]).to include("User-Agent" => Tfs::ProvenanceFeed::USER_AGENT) + expect(captured.first[1]).not_to have_key("Authorization") + end + + it "paginates until a short page and sends the token when given" do + captured = [] + full_page = Array.new(100) { |i| release_json("v0.0.#{i}", "2026-08-01T00:00:#{i.to_s.rjust(2, '0')}Z", %w[SHA256SUMS]) } + bodies = { + "#{api}?per_page=100&page=1" => JSON.generate(full_page), + "#{api}?per_page=100&page=2" => JSON.generate([release_json("v0.1.0", "2026-09-01T00:00:00Z", %w[SHA256SUMS])]) + } + feed = described_class.new(api_url: api, download_url: download, token: "sekrit", fetcher: stub_fetcher(bodies, captured)) + expect(feed.releases.size).to eq(101) + expect(feed.releases.last["tag"]).to eq("v0.1.0") + expect(captured.map { |(_url, headers)| headers["Authorization"] }.uniq).to eq(["Bearer sekrit"]) + end + + it "fails named on a malformed releases response" do + captured = [] + feed = described_class.new(api_url: api, download_url: download, + fetcher: stub_fetcher({ "#{api}?per_page=100&page=1" => "not json" }, captured)) + expect { feed.releases }.to raise_error(Tfs::ProvenanceFeed::Error, /cannot parse the releases API response/) + + feed = described_class.new(api_url: api, download_url: download, + fetcher: stub_fetcher({ "#{api}?per_page=100&page=1" => "{}" }, captured)) + expect { feed.releases }.to raise_error(Tfs::ProvenanceFeed::Error, /want an array/) + end + + it "fetches an asset body and wraps failures naming the release and asset" do + captured = [] + bodies = { "#{download}/v0.2.38/SHA256SUMS" => "abc tfs-ruby-3.3.12-src.tar.gz\n" } + feed = described_class.new(api_url: api, download_url: download, fetcher: stub_fetcher(bodies, captured)) + expect(feed.asset_body("v0.2.38", "SHA256SUMS")).to eq("abc tfs-ruby-3.3.12-src.tar.gz\n") + + expect { feed.asset_body("v0.2.38", "provenance.yaml") } + .to raise_error(Tfs::ProvenanceFeed::Error, /cannot fetch provenance\.yaml of release v0\.2\.38: HTTP 404/) + end +end diff --git a/spec/tfs/provenance_spec.rb b/spec/tfs/provenance_spec.rb new file mode 100644 index 0000000..23e10bd --- /dev/null +++ b/spec/tfs/provenance_spec.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +RSpec.describe Tfs::Provenance do + let(:sha_a) { "a" * 64 } + let(:sha_b) { "b" * 64 } + let(:sha_c) { "c" * 64 } + let(:entry_a) { { "asset" => "tfs-ruby-3.3.12-src.tar.gz", "sha256" => sha_a, "first_release" => "v0.2.21" } } + let(:entry_b) { { "asset" => "tfs-ruby-3.3.12-src.tar.gz", "sha256" => sha_b, "first_release" => "v0.2.27" } } + let(:entry_c) { { "asset" => "tfs-ruby-3.4.10-src.tar.gz", "sha256" => sha_c, "first_release" => "v0.2.21" } } + + describe ".parse + .serialize" do + it "round-trips entries through the published document form" do + entries = described_class.parse(described_class.serialize([entry_a, entry_b, entry_c])) + expect(entries).to eq([entry_a, entry_b, entry_c]) + end + + it "emits the header comment and the version/entries mapping" do + text = described_class.serialize([entry_a]) + expect(text).to start_with("# tfs-ruby src-tarball provenance ledger") + expect(text).to include("version: 1\n") + end + + it "rejects a document that is not exactly version/entries" do + expect { described_class.parse("entries: []\n") } + .to raise_error(Tfs::Provenance::Error, /must be a mapping of exactly version\/entries/) + expect { described_class.parse("version: 1\nentries: []\nextra: true\n") } + .to raise_error(Tfs::Provenance::Error, /must be a mapping of exactly version\/entries/) + end + + it "rejects a ledger of an unknown version" do + expect { described_class.parse("version: 2\nentries: []\n") } + .to raise_error(Tfs::Provenance::Error, /version 2.*understands 1/) + end + + it "rejects a non-array entries value" do + expect { described_class.parse("version: 1\nentries: {}\n") } + .to raise_error(Tfs::Provenance::Error, /entries must be an array/) + end + + it "rejects an entry with a malformed asset, sha256, or first_release" do + bad_asset = entry_a.merge("asset" => "ruby-3.3.12.tar.gz") + expect { described_class.serialize([bad_asset]) } + .to raise_error(Tfs::Provenance::Error, /entry #1: malformed asset/) + + bad_sha = entry_a.merge("sha256" => "abc") + expect { described_class.serialize([bad_sha]) } + .to raise_error(Tfs::Provenance::Error, /entry #1 \(tfs-ruby-3\.3\.12-src\.tar\.gz\): malformed sha256/) + + bad_tag = entry_a.merge("first_release" => "3.3.12") + expect { described_class.serialize([bad_tag]) } + .to raise_error(Tfs::Provenance::Error, /malformed first_release/) + + extra_key = entry_a.merge("note" => "hand-edited") + expect { described_class.serialize([extra_key]) } + .to raise_error(Tfs::Provenance::Error, /must be a mapping of exactly asset\/sha256\/first_release/) + end + + it "rejects a duplicate (asset, sha256) pair, naming both positions" do + expect { described_class.serialize([entry_a, entry_b, entry_a]) } + .to raise_error(Tfs::Provenance::Error, /duplicate provenance entry.*tfs-ruby-3\.3\.12-src\.tar\.gz/) + + text = "version: 1\nentries:\n" \ + "- {asset: #{entry_a['asset']}, sha256: #{sha_a}, first_release: v0.2.21}\n" \ + "- {asset: tfs-ruby-3.4.10-src.tar.gz, sha256: #{sha_c}, first_release: v0.2.21}\n" \ + "- {asset: #{entry_a['asset']}, sha256: #{sha_a}, first_release: v0.2.22}\n" + expect { described_class.parse(text) } + .to raise_error(Tfs::Provenance::Error, /duplicate provenance entry #1 and #3/) + end + end + + describe ".entries_from_sums" do + it "parses sha256sum lines, one entry per line pinned to the tag" do + entries = described_class.entries_from_sums("v0.2.38", "#{sha_a} tfs-ruby-3.3.12-src.tar.gz\n#{sha_c} tfs-ruby-3.4.10-src.tar.gz\n") + expect(entries).to eq([ + { "asset" => "tfs-ruby-3.3.12-src.tar.gz", "sha256" => sha_a, "first_release" => "v0.2.38" }, + { "asset" => "tfs-ruby-3.4.10-src.tar.gz", "sha256" => sha_c, "first_release" => "v0.2.38" } + ]) + end + + it "accepts the binary-mode marker and an uppercase sum" do + entries = described_class.entries_from_sums("v0.2.38", "#{sha_a.upcase} *tfs-ruby-3.3.12-src.tar.gz\n") + expect(entries).to eq([entry_a.merge("first_release" => "v0.2.38")]) + end + + it "rejects a malformed line, naming the release" do + expect { described_class.entries_from_sums("v0.2.38", "#{sha_a} not-a-tarball.txt\n") } + .to raise_error(Tfs::Provenance::Error, /malformed SHA256SUMS line in release v0\.2\.38/) + end + + it "rejects a malformed tag" do + expect { described_class.entries_from_sums("latest", "#{sha_a} tfs-ruby-3.3.12-src.tar.gz\n") } + .to raise_error(Tfs::Provenance::Error, /malformed release tag "latest"/) + end + end + + describe ".chain" do + it "appends new digests and keeps recorded pairs' first_release" do + merged = described_class.chain([entry_a], [entry_b]) + expect(merged).to eq([entry_a, entry_b]) + end + + it "does not duplicate a carried-forward pair" do + carry = entry_a.merge("first_release" => "v0.2.22") + merged = described_class.chain([entry_a], [carry]) + expect(merged).to eq([entry_a]) + end + + it "is a prefix extension of the previous ledger" do + previous = [entry_a, entry_c] + merged = described_class.chain(previous, [entry_b]) + expect(merged.first(previous.size)).to eq(previous) + end + end + + describe ".assert_prefix" do + it "passes when the previous ledger is an exact prefix" do + expect(described_class.assert_prefix([entry_a], [entry_a, entry_b])).to be(true) + expect(described_class.assert_prefix([], [entry_a])).to be(true) + end + + it "raises naming the dropped entry" do + expect { described_class.assert_prefix([entry_a, entry_b], [entry_a]) } + .to raise_error(Tfs::Provenance::Error, /not append-only: entry #2 .*was dropped/) + end + + it "raises on an altered entry, showing what it became" do + replaced = entry_b.merge("first_release" => "v0.9.9") + expect { described_class.assert_prefix([entry_a, entry_b], [entry_a, replaced]) } + .to raise_error(Tfs::Provenance::Error, /not append-only: entry #2 .*became/) + end + + it "raises on a reordered ledger" do + expect { described_class.assert_prefix([entry_a, entry_b], [entry_b, entry_a]) } + .to raise_error(Tfs::Provenance::Error, /not append-only: entry #1/) + end + end + + describe ".assert_complete" do + it "passes when every published asset is recorded" do + expect(described_class.assert_complete([entry_a, entry_b], [entry_b])).to be(true) + end + + it "raises naming the first unrecorded asset" do + expect { described_class.assert_complete([entry_a], [entry_b]) } + .to raise_error(Tfs::Provenance::Error, /incomplete: tfs-ruby-3\.3\.12-src\.tar\.gz.*\(release v0\.2\.27\) is published but unrecorded/) + end + end + + describe ".verify" do + it "returns every entry attesting the digest" do + expect(described_class.verify([entry_a, entry_b], sha_a)).to eq([entry_a]) + end + + it "returns multiple entries when one digest ships under two assets" do + twin = entry_a.merge("asset" => "tfs-ruby-3.3.12-src-linux-musl.tar.gz") + expect(described_class.verify([entry_a, twin], sha_a)).to eq([entry_a, twin]) + end + + it "raises a named error on an unattested digest — loud, never a silent dangle" do + expect { described_class.verify([entry_a], sha_b, source: "release v0.2.38") } + .to raise_error(Tfs::Provenance::Error, /sha256 #{sha_b} is unattested in release v0\.2\.38.*provenance is unresolvable/) + end + + it "rejects a malformed digest" do + expect { described_class.verify([entry_a], "ABC") } + .to raise_error(Tfs::Provenance::Error, /malformed sha256 "ABC"/) + end + end +end diff --git a/tools/lib/tfs.rb b/tools/lib/tfs.rb index 397a55e..10e7834 100644 --- a/tools/lib/tfs.rb +++ b/tools/lib/tfs.rb @@ -18,4 +18,6 @@ module Tfs autoload :BuildPlan, "tfs/build_plan" autoload :SmokePlan, "tfs/smoke_plan" autoload :ReleaseCopier, "tfs/release_copier" + autoload :Provenance, "tfs/provenance" + autoload :ProvenanceFeed, "tfs/provenance_feed" end diff --git a/tools/lib/tfs/http_get.rb b/tools/lib/tfs/http_get.rb index 6b21867..6ff9264 100644 --- a/tools/lib/tfs/http_get.rb +++ b/tools/lib/tfs/http_get.rb @@ -11,11 +11,13 @@ class Error < StandardError; end MAX_REDIRECTS = 5 - def self.body(url) + def self.body(url, headers: {}) uri = URI.parse(url) MAX_REDIRECTS.times do Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http| - response = http.request(Net::HTTP::Get.new(uri)) + request = Net::HTTP::Get.new(uri) + headers.each { |name, value| request[name] = value } + response = http.request(request) case response when Net::HTTPSuccess return response.body diff --git a/tools/lib/tfs/provenance.rb b/tools/lib/tfs/provenance.rb new file mode 100644 index 0000000..4da0a80 --- /dev/null +++ b/tools/lib/tfs/provenance.rb @@ -0,0 +1,195 @@ +# frozen_string_literal: true + +require "yaml" + +module Tfs + # The src-tarball provenance ledger — a cumulative, append-only YAML + # record of every (asset, sha256) pair this factory has ever published, + # carried as the `provenance.yaml` asset of every release. + # + # The model (ruby#100): the authoritative provenance of a runtime build is + # the CONTENT DIGEST of each consumed tarball (tebako-runtime-ruby records + # it as built_from.sources[].sha256; the deterministic-roll rule makes it + # content-addressed). A release NAME is only fetch metadata — rolling + # releases re-roll a tarball under the same version pin whenever the + # line's patches move, and a deleted release leaves the recorded name + # dangling. The ledger is what makes the digest verifiable after the + # fact: once a pair is recorded it is never removed or altered, so a + # re-rolled (or even deleted) tarball cannot silently invalidate a + # published runtime's provenance. + # + # Invariants the publish gate asserts (release-src.yml): + # - append-only: the previous release's ledger is an exact PREFIX of the + # new one (nothing dropped, nothing reordered, nothing re-attributed); + # - completeness: every asset of the release being published is recorded. + # + # This class is the pure model: no network, no filesystem. Entry shape is + # a plain hash — "asset", "sha256", "first_release" — so entries + # serialize to YAML directly. + class Provenance + # Raised on any ledger violation, naming the offending entry. + class Error < StandardError; end + + VERSION = 1 + ASSET_PATTERN = /\Atfs-ruby-\d+\.\d+\.\d+-src(?:-[a-z0-9-]+)?\.tar\.gz\z/ + SHA256_PATTERN = /\A[0-9a-f]{64}\z/ + TAG_PATTERN = /\Av\d+\.\d+\.\d+\z/ + + HEADER = <<~TEXT + # tfs-ruby src-tarball provenance ledger — cumulative, append-only. + # Every (asset, sha256) pair this factory has ever published, with the + # release that FIRST carried those bytes. Published tarballs are + # content-addressed (deterministic roll), so the sha256 is the + # authoritative provenance of any downstream build input; this ledger + # attests that the factory published it. Entries are never removed or + # altered — the publish gate rejects a ledger that does not extend the + # previous release's as an exact prefix. + TEXT + + class << self + # Parses and strictly validates a ledger document, returning its + # entries. Every violation is a named error — a malformed ledger is + # never silently trusted. + def parse(text) + document = YAML.safe_load(text) + unless document.is_a?(Hash) && document.keys.sort == %w[entries version] + raise Error, "provenance ledger must be a mapping of exactly version/entries" + end + unless document["version"] == VERSION + raise Error, "provenance ledger version #{document['version'].inspect} (this reader understands #{VERSION})" + end + + entries = document["entries"] + raise Error, "provenance ledger entries must be an array" unless entries.is_a?(Array) + + seen = {} + entries.each_with_index.map do |entry, index| + validate_entry(entry, index) + key = [entry["asset"], entry["sha256"]] + if seen.key?(key) + raise Error, "duplicate provenance entry ##{seen[key] + 1} and ##{index + 1}: " \ + "#{entry['asset']} sha256 #{entry['sha256'][0, 12]}…" + end + seen[key] = index + entry + end + end + + # Serializes entries to the published ledger form: the header comment + # over the version/entries mapping, keys in canonical order. + def serialize(entries) + seen = {} + entries.each_with_index do |entry, index| + validate_entry(entry, index) + key = [entry["asset"], entry["sha256"]] + raise Error, "duplicate provenance entry: #{entry['asset']} sha256 #{entry['sha256'][0, 12]}…" if seen.key?(key) + + seen[key] = index + end + document = { + "version" => VERSION, + "entries" => entries.map do |entry| + { + "asset" => entry["asset"], + "sha256" => entry["sha256"], + "first_release" => entry["first_release"] + } + end + } + HEADER + YAML.dump(document) + end + + # The entries one release's SHA256SUMS contributes: one per line, + # first_release pinned to that release's tag. Malformed lines are a + # named error (the published SHA256SUMS is machine-generated — a bad + # line is a bug, never something to skip). + def entries_from_sums(tag, sums_text) + unless tag.is_a?(String) && tag.match?(TAG_PATTERN) + raise Error, "malformed release tag #{tag.inspect} (want v..)" + end + + sums_text.each_line.map do |line| + sha256, file = line.strip.split(/\s+/, 2) + asset = file&.sub(/\A\*/, "") + unless sha256&.match?(/\A[0-9a-f]{64}\z/i) && asset&.match?(ASSET_PATTERN) + raise Error, "malformed SHA256SUMS line in release #{tag}: #{line.strip.inspect}" + end + + { "asset" => asset, "sha256" => sha256.downcase, "first_release" => tag } + end + end + + # Chains a previous ledger with one release's entries: pairs already + # recorded keep their first_release; new pairs append (a re-rolled + # tarball appends a new entry — the old bytes stay recorded forever). + def chain(previous_entries, new_entries) + recorded = previous_entries.map { |entry| [entry["asset"], entry["sha256"]] }.to_h { |key| [key, true] } + appended = new_entries.reject { |entry| recorded.key?([entry["asset"], entry["sha256"]]) } + previous_entries + appended + end + + # The append-only assertion: the previous ledger must be an exact + # prefix of the merged one. Raises naming the first divergence. + def assert_prefix(previous_entries, merged_entries) + previous_entries.each_with_index do |entry, index| + next if merged_entries[index] == entry + + got = merged_entries[index] + raise Error, + "provenance ledger is not append-only: entry ##{index + 1} " \ + "(#{entry['asset']} sha256 #{entry['sha256'][0, 12]}…, first_release #{entry['first_release']}) " \ + "#{got.nil? ? 'was dropped' : "became #{got['asset']} sha256 #{got['sha256'][0, 12]}…"} — " \ + "a published digest must never be removed or altered" + end + true + end + + # The completeness assertion: every entry of the release being + # published must be recorded in the ledger. Raises naming the first + # unrecorded asset. + def assert_complete(ledger_entries, sums_entries) + recorded = ledger_entries.map { |entry| [entry["asset"], entry["sha256"]] }.to_h { |key| [key, true] } + sums_entries.each do |entry| + next if recorded.key?([entry["asset"], entry["sha256"]]) + + raise Error, + "provenance ledger incomplete: #{entry['asset']} sha256 #{entry['sha256'][0, 12]}… " \ + "(release #{entry['first_release']}) is published but unrecorded" + end + true + end + + # The audit lookup: every entry attesting this digest. Raises a named + # error when the digest is unattested — loud, never a silent dangle. + def verify(entries, sha256, source: "the ledger") + unless sha256.is_a?(String) && sha256.match?(SHA256_PATTERN) + raise Error, "malformed sha256 #{sha256.inspect} (want 64 lowercase hex)" + end + + matches = entries.select { |entry| entry["sha256"] == sha256 } + return matches unless matches.empty? + + raise Error, + "sha256 #{sha256} is unattested in #{source}: no factory release ever published " \ + "these bytes — provenance is unresolvable, do not trust a build input claiming it" + end + + private + + def validate_entry(entry, index) + unless entry.is_a?(Hash) && entry.keys.sort == %w[asset first_release sha256] + raise Error, "provenance entry ##{index + 1} must be a mapping of exactly asset/sha256/first_release" + end + unless entry["asset"].is_a?(String) && entry["asset"].match?(ASSET_PATTERN) + raise Error, "provenance entry ##{index + 1}: malformed asset #{entry['asset'].inspect}" + end + unless entry["sha256"].is_a?(String) && entry["sha256"].match?(SHA256_PATTERN) + raise Error, "provenance entry ##{index + 1} (#{entry['asset']}): malformed sha256 #{entry['sha256'].inspect}" + end + unless entry["first_release"].is_a?(String) && entry["first_release"].match?(TAG_PATTERN) + raise Error, "provenance entry ##{index + 1} (#{entry['asset']}): malformed first_release #{entry['first_release'].inspect}" + end + end + end + end +end diff --git a/tools/lib/tfs/provenance_feed.rb b/tools/lib/tfs/provenance_feed.rb new file mode 100644 index 0000000..6b5fafe --- /dev/null +++ b/tools/lib/tfs/provenance_feed.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require "json" + +module Tfs + # The GitHub-facing feed for the provenance ledger: lists this + # factory's published releases (oldest first) with their asset names and + # fetches per-release SHA256SUMS / provenance.yaml bodies. All HTTP goes + # through an injectable fetcher (Tfs::HttpGet in production, a stub in + # specs) — the class itself is pure request planning and response + # parsing. Every failure is a named error naming the release. + class ProvenanceFeed + # Raised on any feed failure, naming the release where relevant. + class Error < StandardError; end + + API_URL = "https://api.github.com/repos/tamatebako/ruby/releases" + DOWNLOAD_URL = "https://github.com/tamatebako/ruby/releases/download" + USER_AGENT = "tfs-ruby-provenance (tamatebako/ruby)" + PER_PAGE = 100 + + # token: optional GitHub token (Authorization header for the releases + # API; the public rate limit already covers the handful of + # calls a publish makes). + # fetcher: #call(url, headers:) -> String body. + def initialize(token: nil, fetcher: nil, api_url: API_URL, download_url: DOWNLOAD_URL) + @fetcher = fetcher || HttpGet.method(:body) + @api_url = api_url + @download_url = download_url + @headers = { "User-Agent" => USER_AGENT } + @headers["Authorization"] = "Bearer #{token}" if token + end + + # Every published release, oldest first: {"tag", "created_at", + # "assets"}. Drafts are not publications and never attest anything. + def releases + page = 1 + listing = [] + loop do + batch = JSON.parse(@fetcher.call("#{@api_url}?per_page=#{PER_PAGE}&page=#{page}", headers: @headers)) + raise Error, "unexpected releases API response (want an array)" unless batch.is_a?(Array) + + listing.concat(batch.reject { |release| release["draft"] }) + break if batch.size < PER_PAGE + + page += 1 + end + listing.sort_by { |release| release.fetch("created_at") }.map do |release| + { + "tag" => release.fetch("tag_name"), + "created_at" => release.fetch("created_at"), + "assets" => release.fetch("assets").map { |asset| asset.fetch("name") } + } + end + rescue JSON::ParserError => e + raise Error, "cannot parse the releases API response: #{e.message}" + rescue KeyError => e + raise Error, "unexpected releases API response (missing key #{e.key})" + end + + # The named asset's body from one release's download area. + def asset_body(tag, name) + @fetcher.call("#{@download_url}/#{tag}/#{name}", headers: @headers) + rescue HttpGet::Error => e + raise Error, "cannot fetch #{name} of release #{tag}: #{e.message}" + end + end +end diff --git a/tools/provenance b/tools/provenance new file mode 100755 index 0000000..3c0c6e5 --- /dev/null +++ b/tools/provenance @@ -0,0 +1,124 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Usage: +# tools/provenance --build --tag --sums --out +# tools/provenance --verify [--ledger |-] +# +# --build (release-src's publish gate): chains the new release's +# provenance.yaml ledger from the newest existing release's ledger — or +# BOOTSTRAPS it from every published release's SHA256SUMS when none carries +# one yet (the one-time recovery path) — then asserts the invariants and +# writes the ledger the release attaches: +# append-only the previous ledger is an exact prefix of the new one; +# completeness every asset of this release's SHA256SUMS is recorded. +# Either failure is named and fails the publish: a re-rolled tarball never +# silently drops a published runtime's provenance (Tfs::Provenance). +# +# --verify (the audit path): resolves a recorded digest against a ledger — +# a local file, "-" for stdin, or by default the newest release's +# published provenance.yaml. Prints every attesting entry; an unattested +# digest is a named error, exit 1. +# +# GH_TOKEN/GITHUB_TOKEN (optional) authorizes the releases-API calls. + +$LOAD_PATH.unshift(File.expand_path("lib", __dir__)) +require "tfs" + +USAGE = "usage: tools/provenance --build --tag --sums --out \n" \ + " tools/provenance --verify [--ledger |-]" + +def option(arguments, name) + index = arguments.index(name) + return nil unless index + + value = arguments[index + 1] + abort "tools/provenance: #{name} needs a value\n#{USAGE}" if value.nil? || value.start_with?("--") + + value +end + +def feed + Tfs::ProvenanceFeed.new(token: ENV["GH_TOKEN"] || ENV.fetch("GITHUB_TOKEN", nil)) +end + +def build(arguments) + tag = option(arguments, "--tag") + sums = option(arguments, "--sums") + out = option(arguments, "--out") + abort "tools/provenance: --build needs --tag, --sums and --out\n#{USAGE}" if [tag, sums, out].any?(&:nil?) + abort "tools/provenance: cannot read #{sums}" unless File.file?(sums) + + releases = feed.releases + base = releases.last + chained = base && base["assets"].include?("provenance.yaml") + if chained + previous = Tfs::Provenance.parse(feed.asset_body(base["tag"], "provenance.yaml")) + mode = "chained from #{base['tag']} (#{previous.size} entries)" + else + # One-time recovery path: no published release carries a ledger yet, so + # derive the history from every published SHA256SUMS, oldest first — + # first appearance owns first_release. + previous = [] + releases.each do |release| + sums_entries = Tfs::Provenance.entries_from_sums(release["tag"], feed.asset_body(release["tag"], "SHA256SUMS")) + previous = Tfs::Provenance.chain(previous, sums_entries) + end + mode = "bootstrap over #{releases.size} published releases (#{previous.size} entries; " \ + "no previous ledger to prefix-check)" + end + + current = Tfs::Provenance.entries_from_sums(tag, File.read(sums)) + merged = Tfs::Provenance.chain(previous, current) + Tfs::Provenance.assert_prefix(previous, merged) if chained + Tfs::Provenance.assert_complete(merged, current) + File.write(out, Tfs::Provenance.serialize(merged)) + + added = merged.size - previous.size + puts "provenance: #{mode}" + puts "provenance: #{tag} adds #{added} entr#{added == 1 ? 'y' : 'ies'} (of #{current.size} assets); " \ + "ledger now #{merged.size} entries" + puts "provenance: append-only + completeness assertions passed; wrote #{out}" +end + +def verify(arguments) + sha256 = arguments[1] + abort "tools/provenance: --verify needs a sha256\n#{USAGE}" if sha256.nil? || sha256.start_with?("--") + + ledger = option(arguments, "--ledger") + if ledger == "-" + text = $stdin.read + source = "stdin" + elsif ledger + abort "tools/provenance: cannot read #{ledger}" unless File.file?(ledger) + + text = File.read(ledger) + source = ledger + else + latest = feed.releases.last + abort "tools/provenance: no published release to read the ledger from" if latest.nil? + unless latest["assets"].include?("provenance.yaml") + abort "tools/provenance: the newest release #{latest['tag']} carries no provenance.yaml " \ + "(pre-ledger era) — pass --ledger explicitly" + end + + text = feed.asset_body(latest["tag"], "provenance.yaml") + source = "release #{latest['tag']}" + end + + matches = Tfs::Provenance.verify(Tfs::Provenance.parse(text), sha256.downcase, source: source) + matches.each do |entry| + puts "#{entry['sha256']} #{entry['asset']} (first published: #{entry['first_release']})" + end + puts "provenance: attested by #{source}" +end + +begin + case ARGV[0] + when "--build" then build(ARGV) + when "--verify" then verify(ARGV) + else abort USAGE + end +rescue Tfs::Provenance::Error, Tfs::ProvenanceFeed::Error => e + abort "tools/provenance: #{e.message}" +end