Skip to content

Commit 22e5973

Browse files
authored
Add the regression corpus tool and the guide to writing migrations (#4866)
* Add the regression corpus tool and the guide to writing migrations * Update website migration guide
1 parent f04bbe1 commit 22e5973

13 files changed

Lines changed: 1062 additions & 2 deletions

File tree

‎.github/workflows/check.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,62 @@ jobs:
186186
--package document-graph-storage
187187
--package document-format
188188
--package graphene-cli
189+
--package regression
190+
191+
# Render the documents of the regression corpus and compare them with their reference images
192+
regression-corpus:
193+
needs: skip-check
194+
if: ${{ !cancelled() && needs.skip-check.outputs.skip != 'true' }}
195+
runs-on: ${{ github.event.pull_request.head.repo.fork && 'ubuntu-latest' || vars.LINUX_RUNNER || 'ubuntu-latest' }}
196+
permissions:
197+
contents: read
198+
env:
199+
CARGO_INCREMENTAL: 0
200+
RUSTC_WRAPPER: sccache
201+
SCCACHE_GHA_ENABLED: "true"
202+
# Only master pushes write to the compilation cache, so unmerged code can't plant entries that later master builds reuse
203+
SCCACHE_GHA_RW_MODE: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/master') && 'READ_WRITE' || 'READ_ONLY' }}
204+
# NOTICE: keep in sync with the test job above and the web job in `build.yml`
205+
SCCACHE_VERSION: "v0.19.0"
206+
RUSTFLAGS: "-Dwarnings --cfg=web_sys_unstable_apis"
207+
steps:
208+
- name: 📥 Clone repository
209+
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
210+
211+
- name: 📥 Clone regression corpus
212+
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
213+
with:
214+
repository: tagtraumen/graphite-corpus
215+
path: target/regression-corpus
216+
lfs: true
217+
218+
- name: 🦀 Install Rust
219+
uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
220+
with:
221+
toolchain: stable
222+
override: true
223+
cache: false
224+
rustflags: ""
225+
226+
- name: 📦 Install sccache
227+
uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
228+
with:
229+
version: ${{ env.SCCACHE_VERSION }}
230+
231+
# GitHub's runners have no GPU, so rendering falls back to Mesa's software Vulkan driver
232+
- name: 🎮 Install a software Vulkan driver
233+
run: sudo apt-get update && sudo apt-get install --yes mesa-vulkan-drivers
234+
235+
- name: 🖼️ Check the corpus renders against its references
236+
run: cargo run --package regression -- --corpus target/regression-corpus check --output target/regression
237+
238+
- name: 📤 Upload the renders and diffs of mismatches
239+
if: ${{ failure() }}
240+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
241+
with:
242+
name: regression-mismatches
243+
path: target/regression
244+
if-no-files-found: ignore
189245

190246
# Stored types, node version lineages, and node APIs against the base commit. Only needs the snapshot tool, so it runs on a GitHub runner
191247
migrations:

‎Cargo.lock‎

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ members = [
4141
"tools/third-party-licenses",
4242
"tools/editor-message-tree",
4343
"tools/node-docs",
44+
"tools/regression",
4445
"tools/web-editor-driver",
4546
]
4647
default-members = [

‎tools/cargo-run/src/lib.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ pub enum Action {
1111
Explore(Option<String>),
1212
Drive,
1313
Migration,
14+
Regression,
1415
}
1516

1617
pub enum Target {
@@ -54,6 +55,16 @@ impl Task {
5455
});
5556
}
5657

58+
// Everything after `regression` belongs to the document regression tool
59+
if args.first() == Some(&"regression") {
60+
return Some(Task {
61+
action: Action::Regression,
62+
target: Target::Cli,
63+
profile: Profile::Default,
64+
args: args[1..].iter().map(|s| s.to_string()).collect(),
65+
});
66+
}
67+
5768
let split = args.iter().position(|a| *a == "--").unwrap_or(args.len());
5869
let passthru_args = args[split..].iter().skip(1).map(|s| s.to_string()).collect();
5970
let args = &args[..split];

‎tools/cargo-run/src/main.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ fn usage() {
1919
println!(" explore Open an assortment of tools for exploring the codebase");
2020
println!(" drive Drive the web editor in an isolated browser (see `cargo run drive help`)");
2121
println!(" migration Freeze the stored format or scaffold node migrations (see `cargo run migration help`)");
22+
println!(" regression Upgrade documents and check renders of the regression corpus (see `cargo run regression help`)");
2223
println!(" help Show this message");
2324
println!("<target>:");
2425
println!(" [web] Web app (default)");
@@ -55,6 +56,9 @@ fn main() -> ExitCode {
5556
if matches!(task.action, Action::Migration) {
5657
return delegate("migration-snapshot", &task);
5758
}
59+
if matches!(task.action, Action::Regression) {
60+
return delegate("regression", &task);
61+
}
5862

5963
if let Err(e) = run_task(&task) {
6064
eprintln!("Error: {e}");
@@ -161,7 +165,7 @@ fn run_task(task: &Task) -> Result<(), Error> {
161165
profile = match action {
162166
Action::Run => &Profile::Debug,
163167
Action::Build => &Profile::Release,
164-
Action::Explore(_) | Action::Drive | Action::Migration => unreachable!(),
168+
Action::Explore(_) | Action::Drive | Action::Migration | Action::Regression => unreachable!(),
165169
}
166170
}
167171

@@ -188,7 +192,7 @@ fn run_task(task: &Task) -> Result<(), Error> {
188192
(Action::Build, Target::Cli, Profile::Debug) => cmd!("cargo", "build", "-p", "graphene-cli").run()?,
189193
(Action::Build, Target::Cli, Profile::Release | Profile::Default) => cmd!("cargo", "build", "-r", "-p", "graphene-cli").run()?,
190194

191-
(Action::Explore(_) | Action::Drive | Action::Migration, _, _) => unreachable!(),
195+
(Action::Explore(_) | Action::Drive | Action::Migration | Action::Regression, _, _) => unreachable!(),
192196
}
193197
Ok(())
194198
}

‎tools/regression/Cargo.toml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
[package]
2+
name = "regression"
3+
description = "Upgrades Graphite documents through their migrations and checks renders of a document corpus against reference images"
4+
version.workspace = true
5+
license.workspace = true
6+
authors.workspace = true
7+
edition.workspace = true
8+
repository.workspace = true
9+
readme.workspace = true
10+
homepage.workspace = true
11+
rust-version.workspace = true
12+
publish.workspace = true
13+
14+
[lints]
15+
workspace = true
16+
17+
[dependencies]
18+
# Local dependencies
19+
graphene-cli = { path = "../../node-graph/graphene-cli" }
20+
document-format = { workspace = true, features = ["zip", "xz", "migrations"] }
21+
22+
# Workspace dependencies
23+
clap = { workspace = true, features = ["derive"] }
24+
fern = { workspace = true }
25+
image = { workspace = true }
26+
log = { workspace = true }
27+
serde = { workspace = true }
28+
serde_json = { workspace = true }
29+
tokio = { workspace = true }

‎tools/regression/src/compare.rs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
//! Compares a render to its reference pixel by pixel.
2+
3+
use crate::corpus::Tolerance;
4+
use image::RgbaImage;
5+
6+
pub(crate) enum Comparison {
7+
Matches,
8+
SizeChanged {
9+
reference: (u32, u32),
10+
actual: (u32, u32),
11+
},
12+
/// More pixels than the tolerance allows differ, highlighted in `diff`.
13+
Differs {
14+
differing: usize,
15+
total: usize,
16+
diff: RgbaImage,
17+
},
18+
}
19+
20+
/// O(pixels).
21+
pub(crate) fn compare(reference: &RgbaImage, actual: &RgbaImage, tolerance: Tolerance) -> Comparison {
22+
if reference.dimensions() != actual.dimensions() {
23+
return Comparison::SizeChanged {
24+
reference: reference.dimensions(),
25+
actual: actual.dimensions(),
26+
};
27+
}
28+
29+
// Differing pixels show red over a faded copy of the reference, so they stand out in context
30+
let mut diff = RgbaImage::new(actual.width(), actual.height());
31+
let mut differing = 0;
32+
for ((expected, rendered), marked) in reference.pixels().zip(actual.pixels()).zip(diff.pixels_mut()) {
33+
let largest_difference = expected.0.iter().zip(rendered.0).map(|(a, b)| a.abs_diff(b)).max().unwrap_or_default();
34+
if largest_difference > tolerance.channel {
35+
differing += 1;
36+
*marked = image::Rgba([255, 0, 0, 255]);
37+
} else {
38+
let [red, green, blue, alpha] = expected.0;
39+
let luminance = ((red as u32 * 3 + green as u32 * 6 + blue as u32) / 10) as u8;
40+
*marked = image::Rgba([luminance, luminance, luminance, alpha / 4]);
41+
}
42+
}
43+
44+
let total = (actual.width() * actual.height()) as usize;
45+
if differing as f64 > tolerance.pixels * total as f64 {
46+
Comparison::Differs { differing, total, diff }
47+
} else {
48+
Comparison::Matches
49+
}
50+
}
51+
52+
#[cfg(test)]
53+
mod tests {
54+
use super::*;
55+
56+
fn image_of(pixels: &[[u8; 4]]) -> RgbaImage {
57+
RgbaImage::from_raw(pixels.len() as u32, 1, pixels.concat()).unwrap()
58+
}
59+
60+
#[test]
61+
fn small_differences_and_few_pixels_are_tolerated() {
62+
let reference = image_of(&[[10, 10, 10, 255]; 4]);
63+
let close = image_of(&[[12, 9, 10, 255]; 4]);
64+
assert!(matches!(compare(&reference, &close, Tolerance::default()), Comparison::Matches));
65+
66+
let one_off = image_of(&[[10, 10, 10, 255], [10, 10, 10, 255], [10, 10, 10, 255], [200, 10, 10, 255]]);
67+
let lenient = Tolerance { channel: 2, pixels: 0.25 };
68+
assert!(matches!(compare(&reference, &one_off, lenient), Comparison::Matches));
69+
70+
let Comparison::Differs { differing, total, diff } = compare(&reference, &one_off, Tolerance::default()) else {
71+
panic!("one pixel in four exceeds the default tolerance");
72+
};
73+
assert_eq!((differing, total), (1, 4));
74+
assert_eq!(diff.get_pixel(3, 0).0, [255, 0, 0, 255]);
75+
}
76+
77+
#[test]
78+
fn a_size_change_never_matches() {
79+
let comparison = compare(&image_of(&[[0; 4]; 2]), &image_of(&[[0; 4]; 3]), Tolerance::default());
80+
assert!(matches!(comparison, Comparison::SizeChanged { reference: (2, 1), actual: (3, 1) }));
81+
}
82+
}

0 commit comments

Comments
 (0)