diff --git a/.stkr/project.toml b/.stkr/project.toml index 4f2b88cd7..97da3a032 100644 --- a/.stkr/project.toml +++ b/.stkr/project.toml @@ -37,9 +37,13 @@ include = ["**/*.rs", "**/*.py", "**/Cargo.toml", "mise.toml", ".stkr/project.to exclude = ["crates/dokime-gui/src-tauri/src/rpc/generated.rs"] [barrels] -# Deliberate public façades, not accidental re-export files: the authoring -# prelude and the one namespace a sidecar author imports from. -exclude = ["crates/dokime-runtime/src/prelude.rs", "crates/dokime-runtime/src/plugin/sidecar.rs"] +# Deliberate Public Import Surfaces: Authoring, Criterion Evidence And GUI Test Ports. +exclude = [ + "crates/dokime-runtime/src/prelude.rs", + "crates/dokime-runtime/src/plugin/sidecar.rs", + "crates/dokime-runtime/src/evidence.rs", + "crates/dokime-gui/src-tauri/src/testing.rs", +] [markdown] include = ["**/README.md", "CHANGELOG.md", "CONTRIBUTING.md", "SECURITY.md", "LICENSE.md", ".github/**", "openspec/**", "docs/**/*.md"] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb17f28fd..dd65542bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -146,17 +146,23 @@ restores generated contracts and extracts the Rust archive once before concurren readers start. Standalone Rust lanes also reuse one extraction for listing and execution; standalone system runs unpack their binaries and shared runtime inputs without the other test layers, keeping extraction within hosted disk capacity. -The entry runs all ten -lanes even if one fails and returns failure if any lane fails. Full lane logs, +The aggregate owns a native mise task lock across preparation, consumers and +collection. Its referenced tasks share that permit and run concurrently; the +ordinary one-task default remains in place for standalone language tasks. +Canonical mise task references schedule all eleven product lanes concurrently, +plus the tooling integration tests. Each prepared leaf records its actual exit +code; the final collection returns failure for any failed or missing result. Full lane logs, exit codes, wall times and an aggregate `time-summary.md`/`.json` live under `target/test-reports/run--/`; every lane also keeps its own time summary under `target/test-reports/-/`. Individual lane tasks remain independently usable; avoid running the same lane twice in one checkout. +Run tooling alone with `mise run test:python:integration --project tools/python`; +add `--select schedule` to select its scheduling tests. Only one native desktop lane may use an interactive desktop session at a time, including runs from other worktrees. Reserve an idle test host for wall-clock comparisons; simultaneous external test runs can trip existing liveness guards. -The scheduler reserves CPU headroom for the desktop and divides the remaining +Shared preparation reserves CPU headroom for the desktop and divides the remaining logical CPU slots across the four Rust nextest lanes, Python integration and both Vitest lanes. All seven pools use that worker count. Standalone Rust lanes keep nextest's CPU-sized pool; standalone Python integration uses diff --git a/crates/dokime-packaging/src/installer/manifest.rs b/crates/dokime-packaging/src/installer/manifest.rs index 9211f08cc..727a3ef04 100644 --- a/crates/dokime-packaging/src/installer/manifest.rs +++ b/crates/dokime-packaging/src/installer/manifest.rs @@ -423,32 +423,6 @@ name='x'" ); } - #[test] - fn license_file_containment() { - let pack = tempfile::tempdir().unwrap(); - let outside = tempfile::tempdir().unwrap(); - std::fs::create_dir(pack.path().join("terms")).unwrap(); - std::fs::write(pack.path().join("terms/EULA.txt"), "terms").unwrap(); - std::fs::write(outside.path().join("EULA.txt"), "terms").unwrap(); - let escape = format!("../{}/EULA.txt", outside.path().file_name().unwrap().to_str().unwrap()); - - let mut license = License { - spdx: Some("LicenseRef-Acme-Proprietary".into()), - file: Some("terms/EULA.txt".into()), - }; - license.resolve(pack.path()).unwrap(); - assert_eq!(license.file.as_deref(), Some(pack.path().join("terms/EULA.txt").canonicalize().unwrap().as_path())); - - for file in [escape.as_str(), "/EULA.txt", "C:/EULA.txt", r"terms\EULA.txt", "missing.txt", "terms"] { - let mut license = License { - spdx: Some("LicenseRef-Acme-Proprietary".into()), - file: Some(file.into()), - }; - let Err(Error::License { field, .. }) = license.resolve(pack.path()) else { panic!("{file}") }; - assert_eq!(field, "file", "{file}"); - } - } - #[test] fn native_formats() { let format = if cfg!(target_os = "windows") { "nsis" } else { "deb" }; diff --git a/crates/dokime-packaging/tests/integration/installer.rs b/crates/dokime-packaging/tests/integration/installer.rs index 510976889..f718c514e 100644 --- a/crates/dokime-packaging/tests/integration/installer.rs +++ b/crates/dokime-packaging/tests/integration/installer.rs @@ -9,6 +9,8 @@ /* === Modules === */ +mod license_file_containment; + /* === Re-Exports === */ /* === Imports === */ diff --git a/crates/dokime-packaging/tests/integration/installer/license_file_containment.rs b/crates/dokime-packaging/tests/integration/installer/license_file_containment.rs new file mode 100644 index 000000000..a40136716 --- /dev/null +++ b/crates/dokime-packaging/tests/integration/installer/license_file_containment.rs @@ -0,0 +1,69 @@ +/* *** + * + * Dokime + * + * SPDX-License-Identifier: Apache-2.0 + * + * *** + */ + +// Terms File Containment Through The Public Manifest Reader + +/* === Modules === */ + +/* === Re-Exports === */ + +/* === Imports === */ + +// Standard Library Imports + +// Third Party Imports + +// Local Imports +use dokime_packaging::{Error, installer::Manifest}; + +/* === Constants === */ + +/* === Macros === */ + +/* === Type Aliases === */ + +/* === Structs === */ + +/* === Enums === */ + +/* === Traits === */ + +/* === Implementations === */ + +/* === Public Functions === */ + +/* === Private Functions === */ + +/* === Tests === */ + +#[test] +fn license_file_containment() { + let pack = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::create_dir(pack.path().join("terms")).unwrap(); + std::fs::write(pack.path().join("terms/EULA.txt"), "terms").unwrap(); + std::fs::write(outside.path().join("EULA.txt"), "terms").unwrap(); + let escape = format!("../{}/EULA.txt", outside.path().file_name().unwrap().to_str().unwrap()); + + let manifest = pack.path().join("installer.toml"); + let read = |file: &str| { + let text = format!( + "[installer]\nversion='1.2.3'\nformats=[]\n[dokime]\nversion='1.2.3'\n[license]\nspdx='LicenseRef-Acme-Proprietary'\nfile={file:?}" + ); + std::fs::write(&manifest, text).unwrap(); + Manifest::read(&manifest) + }; + let license = read("terms/EULA.txt").unwrap().license.unwrap(); + assert_eq!(license.file.as_deref(), Some(pack.path().join("terms/EULA.txt").canonicalize().unwrap().as_path())); + + for file in [escape.as_str(), "/EULA.txt", "C:/EULA.txt", r"terms\EULA.txt", "missing.txt", "terms"] { + let Err(Error::License { field, .. }) = read(file) else { panic!("{file}") }; + assert_eq!(field, "file", "{file}"); + } +} diff --git a/crates/dokime-runtime/src/artifacts/directory.rs b/crates/dokime-runtime/src/artifacts/directory.rs index 76bc7b4a8..f904ed25b 100644 --- a/crates/dokime-runtime/src/artifacts/directory.rs +++ b/crates/dokime-runtime/src/artifacts/directory.rs @@ -396,48 +396,3 @@ fn read(path: &Path, from: u64, pending: u64, sweeping: bool) -> io::Result 20); assert_eq!(candidate_bytes, baseline_bytes); - eprintln!("Fixture Report Save: before={before:?}, after={after:?}, bytes={}", candidate_bytes.len()); } } diff --git a/crates/dokime-runtime/tests/integration/artifacts/mod.rs b/crates/dokime-runtime/tests/integration/artifacts/mod.rs index a6cd45fc0..7f0c97c12 100644 --- a/crates/dokime-runtime/tests/integration/artifacts/mod.rs +++ b/crates/dokime-runtime/tests/integration/artifacts/mod.rs @@ -13,6 +13,7 @@ pub(crate) mod diagnostics; pub(crate) mod publication; +mod temporary_cleanup; /* === Re-Exports === */ diff --git a/crates/dokime-runtime/tests/integration/artifacts/temporary_cleanup.rs b/crates/dokime-runtime/tests/integration/artifacts/temporary_cleanup.rs new file mode 100644 index 000000000..a0d2583b2 --- /dev/null +++ b/crates/dokime-runtime/tests/integration/artifacts/temporary_cleanup.rs @@ -0,0 +1,86 @@ +/* *** + * + * Dokime + * + * SPDX-License-Identifier: Apache-2.0 + * + * *** + */ + +// Artifact Scratch Ownership Across Clones And Unwinding + +/* === Modules === */ + +/* === Re-Exports === */ + +/* === Imports === */ + +// Standard Library Imports +use std::sync::Arc; + +// Third Party Imports + +// Local Imports +use dokime_runtime::{ + artifacts::{ArtifactSink, Artifacts}, + transport::ArtifactChunk, +}; + +/* === Constants === */ + +/* === Macros === */ + +/* === Type Aliases === */ + +/* === Structs === */ + +struct Sink; + +/* === Enums === */ + +/* === Traits === */ + +/* === Implementations === */ + +impl ArtifactSink for Sink { + fn push(&self, _: ArtifactChunk) {} +} + +/* === Public Functions === */ + +/* === Private Functions === */ + +/* === Tests === */ + +#[test] +fn temporary_artifacts_cleanup_without_sweep() { + for failure in [false, true] { + let artifacts = Artifacts::new("temporary-owner", Arc::new(Sink)); + let root = artifacts.root().unwrap(); + let area = root.parent().unwrap().to_owned(); + std::fs::write(root.join("evidence.bin"), b"captured evidence").unwrap(); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || { + let _owner = artifacts; + assert!(!failure, "injected artifact fixture failure"); + })); + assert_eq!(outcome.is_err(), failure); + assert!(!area.exists(), "artifact scratch survived owner teardown: {}", area.display()); + } +} + +#[test] +fn temporary_artifacts_follow_the_last_clone_and_isolate_equal_tokens() { + let first = Artifacts::new("same-token", Arc::new(Sink)); + let second = Artifacts::new("same-token", Arc::new(Sink)); + let first_root = first.root().unwrap(); + let second_root = second.root().unwrap(); + assert_ne!(first_root, second_root); + let retained = first.clone(); + drop(first); + assert!(first_root.exists()); + drop(retained); + assert!(!first_root.exists()); + assert!(second_root.exists()); + drop(second); + assert!(!second_root.exists()); +} diff --git a/mise.toml b/mise.toml index 0452294ff..6d7f9c382 100644 --- a/mise.toml +++ b/mise.toml @@ -195,7 +195,7 @@ run = [{ task = "check:repository" }, { task = "check:rust" }, { task = "check:p [tasks."check:python"] description = "Check Python" -run = [{ task = "format:python --check" }, { task = "lint:python" }, { task = "test:tooling" }] +run = [{ task = "format:python --check" }, { task = "lint:python" }, { task = "test:python:integration --project tools/python" }] [tasks."check:python:bindings"] description = "Check Built Python Bindings And Stubs" @@ -326,20 +326,33 @@ run = [ # === Test === # [tasks.test] +lock = true description = "Test Every Layer Concurrently from Built Artifacts" usage = 'flag "--profile " default="dev"' depends = ["prepare:desktop"] -run = "{{vars.tools}} dokime_tools.test.schedule --profile {{usage.profile}}" +run = [ + "{{vars.tools}} dokime_tools.test.schedule --prepare --profile {{usage.profile}}", + { tasks = [ + "test:python --profile {{usage.profile}} --prepared", + "test:rust --profile {{usage.profile}} --prepared", + "test:typescript --profile {{usage.profile}} --prepared", + ] }, + "{{vars.tools}} dokime_tools.test.schedule --finish --profile {{usage.profile}}", +] env = { CARGO_INCREMENTAL = "0" } [tasks."test:python"] description = "Test Python Layers" -usage = 'flag "--profile " default="dev"' +usage = ''' +flag "--profile " default="dev" +flag "--prepared" +''' depends = [ - "test:python:unit --profile {{usage.profile}}", - "test:python:integration --profile {{usage.profile}}", - "test:python:acceptance --profile {{usage.profile}}", - "test:python:system --profile {{usage.profile}}", + "test:python:integration --profile {{usage.profile}} --project tools/python{% if usage.prepared %} --prepared{% endif %}", + "test:python:unit --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", + "test:python:integration --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", + "test:python:acceptance --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", + "test:python:system --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", ] env = { CARGO_INCREMENTAL = "0" } @@ -348,18 +361,25 @@ description = "Test Unit Python" depends = ["prepare"] usage = ''' flag "--profile " default="dev" +flag "--prepared" flag "--select " ''' -run = '{{vars.tools}} dokime_tools.test.python --profile {{usage.profile}} --category integration {% if usage.select %}-- -k "{{usage.select}}"{% endif %}' +run = [ + '''{% if usage.prepared %}{{vars.tools}} dokime_tools.test.schedule --lane --profile {{usage.profile}} --category integration --language python --project {{usage.project}}{% elif usage.project == "tools/python" %}{{vars.python}} python -m unittest discover -s tools/python/tests/integration {% if usage.select %}-k "{{usage.select}}"{% endif %}{% else %}{{vars.tools}} dokime_tools.test.python --profile {{usage.profile}} --category integration {% if usage.select %}-- -k "{{usage.select}}"{% endif %}{% endif %}''', +] env = { CARGO_INCREMENTAL = "0" } [tasks."test:python:acceptance"] @@ -367,10 +387,13 @@ description = "Test Acceptance Python" depends = ["prepare"] usage = ''' flag "--profile " default="dev" +flag "--prepared" flag "--artifacts " help="Use An Explicit Native Client Fixture Root" flag "--select " help="Run Matching System Stories" ''' -run = '{{vars.tools}} dokime_tools.test.python --profile {{usage.profile}} --category system -- --build-profile {{usage.profile}} {% if usage.artifacts %}--artifacts "{{usage.artifacts}}"{% endif %} {% if usage.select %}-k "{{usage.select}}"{% endif %}' +run = [ + '''{% if usage.prepared %}{{vars.tools}} dokime_tools.test.schedule --lane --profile {{usage.profile}} --category system --language python{% else %}{{vars.tools}} dokime_tools.test.python --profile {{usage.profile}} --category system -- --build-profile {{usage.profile}} {% if usage.artifacts %}--artifacts "{{usage.artifacts}}"{% endif %} {% if usage.select %}-k "{{usage.select}}"{% endif %}{% endif %}''', +] env = { CARGO_INCREMENTAL = "0" } [tasks."test:python:coverage"] @@ -403,12 +429,15 @@ run = "{{vars.tools}} dokime_tools.test.wheel --profile {{usage.profile}}{% if u [tasks."test:rust"] description = "Test Rust Layers" -usage = 'flag "--profile " default="dev"' +usage = ''' +flag "--profile " default="dev" +flag "--prepared" +''' depends = [ - "test:rust:unit --profile {{usage.profile}}", - "test:rust:integration --profile {{usage.profile}}", - "test:rust:acceptance --profile {{usage.profile}}", - "test:rust:system --profile {{usage.profile}}", + "test:rust:unit --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", + "test:rust:integration --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", + "test:rust:acceptance --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", + "test:rust:system --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", ] env.CARGO_INCREMENTAL = "0" @@ -416,9 +445,12 @@ env.CARGO_INCREMENTAL = "0" description = "Test Unit Rust" usage = ''' flag "--profile " default="dev" +flag "--prepared" flag "--select " help="Narrow the Nextest Filterset" ''' -run = "{{vars.lane}} --profile {{usage.profile}} --category integration --language rust -- -E \"not binary_id(dokime-macros::compile)\" {% if usage.select %}-E \"{{usage.select}}\"{% endif %}" +run = [ + '''{% if usage.prepared %}{{vars.tools}} dokime_tools.test.schedule --lane --profile {{usage.profile}} --category integration --language rust{% else %}{{vars.lane}} --profile {{usage.profile}} --category integration --language rust -- -E "not binary_id(dokime-macros::compile)" {% if usage.select %}-E "{{usage.select}}"{% endif %}{% endif %}''', +] env.CARGO_INCREMENTAL = "0" [tasks."test:rust:acceptance"] @@ -436,12 +471,13 @@ description = "Test Acceptance Rust" depends = ["prepare"] usage = ''' flag "--profile " default="dev" +flag "--prepared" flag "--select " help="Narrow the Nextest Filterset" ''' run = [ - "{{vars.tools}} dokime_tools.test.archive --install-wheels --profile {{usage.profile}}", - '{{vars.lane}} --profile {{usage.profile}} --category system --language rust {% if usage.select %}-- -E "{{usage.select}}"{% endif %}', + '''{% if usage.prepared %}{{vars.tools}} dokime_tools.test.schedule --lane --profile {{usage.profile}} --category system --language rust{% else %}{{vars.tools}} dokime_tools.test.archive --install-wheels --profile {{usage.profile}}{% endif %}''', + '''{% if not usage.prepared %}{{vars.lane}} --profile {{usage.profile}} --category system --language rust {% if usage.select %}-- -E "{{usage.select}}"{% endif %}{% endif %}''', ] env.CARGO_INCREMENTAL = "0" @@ -486,56 +523,57 @@ usage = 'flag "--profile " default="release"' run = "{{vars.tools}} dokime_tools.build.machine -- python -m dokime_tools.test.rust.starter --profile {{usage.profile}}" env.CARGO_INCREMENTAL = "0" -[tasks."test:tooling"] - -description = "Test Project Tooling" -run = "{{vars.python}} python -m unittest discover -s tools/python/tests/integration" - [tasks."test:typescript"] description = "Test TypeScript Layers" -usage = 'flag "--profile " default="dev"' +usage = ''' +flag "--profile " default="dev" +flag "--prepared" +''' depends = [ - "test:typescript:unit --profile {{usage.profile}}", - "test:typescript:integration --profile {{usage.profile}}", - "test:typescript:system --profile {{usage.profile}}", + "test:typescript:unit --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", + "test:typescript:integration --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", + "test:typescript:system --profile {{usage.profile}}{% if usage.prepared %} --prepared{% endif %}", ] env = { CARGO_INCREMENTAL = "0" } [tasks."test:typescript:unit"] description = "Test Unit TypeScript" -depends = ["prepare:generated"] +depends = ["{% if usage.prepared %}prepare{% else %}prepare:generated{% endif %}"] usage = ''' flag "--profile " default="dev" +flag "--prepared" flag "--select " ''' run = [ - "{% if not usage.select %}{{vars.tools}} dokime_tools.prepare.generated --profile {{usage.profile}}{% endif %}", - "{{vars.lane}} --profile {{usage.profile}} --category integration --language typescript --project crates/dokime-gui -- node ./node_modules/vitest/vitest.mjs run {% if usage.select %}{{usage.select}}{% else %}tests/integration{% endif %} --reporter=default --reporter=junit --outputFile.junit={junit} --testTimeout={timeout_ms}", + '''{% if usage.prepared %}{{vars.tools}} dokime_tools.test.schedule --lane --profile {{usage.profile}} --category integration --language typescript{% else %}{% if not usage.select %}{{vars.tools}} dokime_tools.prepare.generated --profile {{usage.profile}}{% endif %}{% endif %}''', + '''{% if not usage.prepared %}{{vars.lane}} --profile {{usage.profile}} --category integration --language typescript --project crates/dokime-gui -- node ./node_modules/vitest/vitest.mjs run {% if usage.select %}{{usage.select}}{% else %}tests/integration{% endif %} --reporter=default --reporter=junit --outputFile.junit={junit} --testTimeout={timeout_ms}{% endif %}''', ] env = { CARGO_INCREMENTAL = "0" } [tasks."test:typescript:system"] description = "Test System TypeScript" -depends = ["prepare:desktop", "prepare:generated"] +depends = ["prepare:desktop", "{% if usage.prepared %}prepare{% else %}prepare:generated{% endif %}"] usage = ''' flag "--profile " default="dev" +flag "--prepared" flag "--select " help="Run One Native Desktop Story" ''' run = [ - "{{vars.tools}} dokime_tools.prepare.generated --profile {{usage.profile}}", - "{{vars.lane}} --profile {{usage.profile}} --category system --language typescript --project crates/dokime-gui -- node e2e/system.mjs --profile {{usage.profile}} --junit {junit} {% if usage.select %}--select {{usage.select}}{% endif %}", + '''{% if usage.prepared %}{{vars.tools}} dokime_tools.test.schedule --lane --profile {{usage.profile}} --category system --language typescript{% else %}{{vars.tools}} dokime_tools.prepare.generated --profile {{usage.profile}}{% endif %}''', + '''{% if not usage.prepared %}{{vars.lane}} --profile {{usage.profile}} --category system --language typescript --project crates/dokime-gui -- node e2e/system.mjs --profile {{usage.profile}} --junit {junit} {% if usage.select %}--select {{usage.select}}{% endif %}{% endif %}''', ] env = { CARGO_INCREMENTAL = "0" } diff --git a/tools/python/src/dokime_tools/test/schedule.py b/tools/python/src/dokime_tools/test/schedule.py index 1f7b41a9a..dd72f8d0d 100644 --- a/tools/python/src/dokime_tools/test/schedule.py +++ b/tools/python/src/dokime_tools/test/schedule.py @@ -7,23 +7,24 @@ # *** -"""Schedule Independent Test Lanes After Preparing Shared Read-Only Inputs.""" +"""Prepare Shared Inputs And Collect Results For The Native Mise Test Graph.""" # === Imports === # # Standard Library Imports import argparse +import json import os +import shutil import subprocess import sys +import tempfile import time -from collections.abc import Mapping, Sequence -from concurrent.futures import ThreadPoolExecutor, as_completed +from collections.abc import Sequence from pathlib import Path # Third Party Imports # Local Imports -from .. import temporary from ..build import paths from ..prepare import generated from ..process import spawn_flags @@ -94,98 +95,115 @@ def command(category: str, language: str, profile: str, workers: int, archive_di return arguments + (["--", *extra] if extra else []) -def execute(directory: Path, commands: Mapping[str, Sequence[str]], destination: Path) -> list[dict]: - """Keep Each Lane's Full Output And Finish Every Lane Despite Failures.""" - environment = dict(os.environ) +def prepare(directory: Path, profile: str) -> None: + """Prepare One Shared Read-Only Input Set Before Mise Starts Any Leaf.""" + started = time.perf_counter() + lanes = policy.lanes(directory) + if not lanes: + raise policy.TestError("no declared test lanes") + worker_pools = sum( + language == "rust" or language == "typescript" and category != "system" or language == "python" and category == "integration" + for category, language in lanes + ) + desktop = int(("system", "typescript") in lanes) + workers = max(1, ((os.process_cpu_count() or 1) - desktop) // max(1, worker_pools)) + destination = report.native_directory(directory, Path("target/test-reports") / f"run-{profile}-{paths.target()}") + for name in ("time-summary.md", "time-summary.json", "prepared.json"): + (destination / name).unlink(missing_ok=True) + for category, language in lanes: + (destination / f"{category}-{language}.json").unlink(missing_ok=True) + (destination / "integration-python-tooling.json").unlink(missing_ok=True) + + archive.install_wheels(directory, profile) + generated.restore(profile) + extracted = Path(tempfile.mkdtemp(prefix="inputs-", dir=destination)) + if any(language == "rust" for _, language in lanes): + archive.extract(directory, profile, extracted) + report.dump( + destination / "prepared.json", + { + "profile": profile, + "worker_pools": worker_pools, + "workers_per_pool": workers, + "archive_directory": str(extracted), + "preparation_seconds": time.perf_counter() - started, + "execution_started": time.time(), + }, + ) + + +def lane(directory: Path, profile: str, category: str, language: str, project: str) -> None: + """Record The Real Exit Status; The Final Aggregate Fails After All Leaves Finish.""" + destination = directory / "target/test-reports" / f"run-{profile}-{paths.target()}" + context = json.loads((destination / "prepared.json").read_text(encoding="utf-8")) + extracted = Path(context["archive_directory"]) if language == "rust" else None + key = f"{category}-{language}" + if project == "tools/python": + key += "-tooling" + arguments = [sys.executable, "-m", "unittest", "discover", "-s", "tools/python/tests/integration"] + else: + arguments = command(category, language, profile, context["workers_per_pool"], extracted) + environment = dict(os.environ, PYTHONUNBUFFERED="1") environment.pop("UV_PROJECT_ENVIRONMENT", None) - # Children Use The Prepared Interpreter Without uv Sync Rewriting Shared Inputs. - environment["PYTHONUNBUFFERED"] = "1" - results = [] - with ThreadPoolExecutor(max_workers=len(commands)) as executor: - pending = {} - for key, arguments in commands.items(): - print(f"START {key}", flush=True) - future = executor.submit(_lane, directory, key, arguments, destination, environment) - pending[future] = key - for future in as_completed(pending): - row = future.result() - results.append(row) - print( - f"{'PASS' if row['exit_code'] == 0 else 'FAIL'} {row['lane']}: {row['seconds']:.3f} s (exit {row['exit_code']})", - flush=True, - ) - if row["exit_code"] != 0: - log = directory / row["log"] - print("\n".join(log.read_text(encoding="utf-8", errors="replace").splitlines()[-30:]), flush=True) - return sorted(results, key=lambda row: row["lane"]) + row = _lane(directory, key, arguments, destination, environment) + report.dump(destination / f"{key}.json", row) + print(f"{'PASS' if row['exit_code'] == 0 else 'FAIL'} {key}: {row['seconds']:.3f} s (exit {row['exit_code']})", flush=True) + if row["exit_code"]: + print((directory / row["log"]).read_text(encoding="utf-8", errors="replace"), flush=True) + + +def finish(directory: Path, profile: str) -> int: + """Missing Results And Failed Leaves Both Fail The Aggregate.""" + destination = directory / "target/test-reports" / f"run-{profile}-{paths.target()}" + context = json.loads((destination / "prepared.json").read_text(encoding="utf-8")) + keys = [f"{category}-{language}" for category, language in policy.lanes(directory)] + keys.append("integration-python-tooling") + rows = [] + for key in keys: + result = destination / f"{key}.json" + if not result.is_file(): + rows.append({"lane": key, "exit_code": 1, "seconds": 0, "log": ""}) + print(f"FAIL {key}: missing lane result", file=sys.stderr) + continue + rows.append(json.loads(result.read_text(encoding="utf-8"))) + context["lanes"] = sorted(rows, key=lambda row: row["lane"]) + context["execution_seconds"] = time.time() - context.pop("execution_started") + context["total_seconds"] = context["preparation_seconds"] + context["execution_seconds"] + extracted = Path(context.pop("archive_directory")) + if extracted.resolve().parent != destination.resolve() or not extracted.name.startswith("inputs-"): + raise policy.TestError("prepared archive escapes its owned report directory") + shutil.rmtree(extracted) + report.dump(destination / "time-summary.json", context) + lines = ["# Concurrent Test Execution", "", "| Lane | Exit | Seconds | Log |", "| --- | ---: | ---: | --- |"] + for row in context["lanes"]: + lines.append(f"| {row['lane']} | {row['exit_code']} | {row['seconds']:.3f} | [{row['lane']}.log]({row['lane']}.log) |") + report.write(destination / "time-summary.md", "\n".join(lines) + "\n") + print(f"Logs And Summary: {destination.relative_to(directory).as_posix()}", flush=True) + return int(any(row["exit_code"] != 0 for row in rows)) def main(arguments: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--profile", choices=policy.PROFILES, default="dev") + actions = parser.add_mutually_exclusive_group(required=True) + actions.add_argument("--prepare", action="store_true") + actions.add_argument("--lane", action="store_true") + actions.add_argument("--finish", action="store_true") + parser.add_argument("--category", choices=policy.CATEGORIES) + parser.add_argument("--language", choices=policy.LANGUAGES) + parser.add_argument("--project", default="sdk/python") options = parser.parse_args(arguments) - started = time.perf_counter() try: directory = policy.root() - lanes = policy.lanes(directory) - if not lanes: - raise policy.TestError("no declared test lanes") - # Budget Every Worker Pool And Leave Headroom For The Interactive Desktop. - rust_lanes = sum(language == "rust" for _, language in lanes) - worker_pools = sum( - language == "rust" or language == "typescript" and category != "system" or language == "python" and category == "integration" - for category, language in lanes - ) - desktop = int(("system", "typescript") in lanes) - workers = max(1, ((os.process_cpu_count() or 1) - desktop) // max(1, worker_pools)) - destination = report.native_directory(directory, Path("target/test-reports") / f"run-{options.profile}-{paths.target()}") - for name in ("time-summary.md", "time-summary.json"): - (destination / name).unlink(missing_ok=True) - - print("Preparing Shared Test Inputs", flush=True) - archive.install_wheels(directory, options.profile) - generated.restore(options.profile) - with temporary.directory() as extracted: - if rust_lanes: - archive.extract(directory, options.profile, extracted) - commands = { - f"{category}-{language}": command(category, language, options.profile, workers, extracted if language == "rust" else None) - for category, language in lanes - } - preparation = time.perf_counter() - started - print( - f"Running {len(lanes)} Lanes Concurrently ({workers} Workers Per Rust Lane / Python Integration / Vitest Lane)", flush=True - ) - execution = time.perf_counter() - rows = execute(directory, commands, destination) - seconds = time.perf_counter() - execution - total = time.perf_counter() - started - result = { - "profile": options.profile, - "worker_pools": worker_pools, - "workers_per_pool": workers, - "preparation_seconds": preparation, - "execution_seconds": seconds, - "total_seconds": total, - "lanes": rows, - } - report.dump(destination / "time-summary.json", result) - lines = [ - "# Concurrent Test Execution", - "", - f"Preparation: {preparation:.3f} s; execution: {seconds:.3f} s; total: {total:.3f} s.", - "", - "| Lane | Exit | Seconds | Log | Time Summary |", - "| --- | ---: | ---: | --- | --- |", - ] - for row in rows: - lines.append( - f"| {row['lane']} | {row['exit_code']} | {row['seconds']:.3f} | [{row['lane']}.log]({row['lane']}.log) | [Summary](../{row['lane']}/time-summary.md) |" - ) - report.write(destination / "time-summary.md", "\n".join(lines) + "\n") - print(f"Execution: {seconds:.3f} s; With Shared Preparation: {total:.3f} s", flush=True) - print(f"Logs And Summary: {destination.relative_to(directory).as_posix()}", flush=True) - return int(any(row["exit_code"] != 0 for row in rows)) + if options.prepare: + prepare(directory, options.profile) + elif options.finish: + return finish(directory, options.profile) + else: + if not options.category or not options.language: + parser.error("--lane requires --category and --language") + lane(directory, options.profile, options.category, options.language, options.project) + return 0 except (policy.TestError, OSError, ValueError, subprocess.CalledProcessError) as error: print(f"test: {error}", file=sys.stderr) return 1 diff --git a/tools/python/tests/integration/test_schedule.py b/tools/python/tests/integration/test_schedule.py index 1d056b8d0..e5d394292 100644 --- a/tools/python/tests/integration/test_schedule.py +++ b/tools/python/tests/integration/test_schedule.py @@ -15,9 +15,13 @@ import io import json import os +import socket +import subprocess import sys import tarfile import tempfile +import threading +import tomllib import unittest from pathlib import Path from unittest.mock import patch @@ -127,105 +131,144 @@ def execute(command, cwd, environment): self.assertTrue((evidence / "junit.xml").is_file()) self.assertEqual(list(evidence.rglob("fixture-only.dat")), []) - def test_concurrent_lanes(self): + def test_native_task_graph_keeps_all_lanes_concurrent_and_reports_failure(self): with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary).resolve() - script = directory / "lane.py" - # Each Child Waits For The Other: A Serial Scheduler Cannot Complete. - script.write_text( - "import os, pathlib, sys, time\n" - "root = pathlib.Path(sys.argv[1])\n" - "name = sys.argv[2]\n" - "assert (root / 'prepared').is_file()\n" + repository = Path(__file__).resolve().parents[4] + configuration = tomllib.loads((repository / "mise.toml").read_text(encoding="utf-8")) + # Project The Authored Graph Without Product Prerequisites Or Native Test Binaries. + source = (repository / "mise.toml").read_text(encoding="utf-8") + start = source.index("[tasks.test]") + stop = source.index("# === Bench === #", start) + tasks = source[start:stop] + (directory / "tools/python/tests/integration").mkdir(parents=True) + (directory / "tools/project.toml").write_text("", encoding="utf-8") + (directory / "tools/python/tests/integration/test_prepared.py").write_text( + "import pathlib, unittest\n" + "class Prepared(unittest.TestCase):\n" + " def test_inputs(self):\n" + " self.assertTrue(pathlib.Path('wheel-ready').is_file())\n" + " self.assertTrue(pathlib.Path('contracts-ready').is_file())\n", + encoding="utf-8", + ) + native = directory / "native.py" + native.write_text( + "import os, pathlib, socket, sys\n" + "root = pathlib.Path.cwd()\n" + "assert (root / 'wheel-ready').is_file()\n" + "assert (root / 'contracts-ready').is_file()\n" "assert 'UV_PROJECT_ENVIRONMENT' not in os.environ\n" - "(root / name).touch()\n" - "deadline = time.monotonic() + 15\n" - "while not all((root / peer).exists() for peer in ('left', 'right')):\n" - " if time.monotonic() > deadline: raise RuntimeError('lanes did not overlap')\n" - " time.sleep(0.01)\n" - "print(name)\n" - "sys.exit(int(sys.argv[3]))\n", + "if '--archive-directory' in sys.argv:\n" + " shared = pathlib.Path(sys.argv[sys.argv.index('--archive-directory') + 1])\n" + " assert (shared / 'ready').read_text() == 'extracted'\n" + " (root / sys.argv[1]).write_text(str(shared))\n" + "with socket.create_connection(('127.0.0.1', int(os.environ['DOKIME_GRAPH_PORT']))) as peer:\n" + " peer.sendall(b'ready')\n" + " assert peer.recv(1) == b'1'\n" + "print(sys.argv[1])\n" + "sys.exit(7 if sys.argv[1] == 'system-rust' else 0)\n", encoding="utf-8", ) - (directory / "prepared").touch() - destination = directory / "reports" - destination.mkdir() - commands = { - "left": [sys.executable, str(script), str(directory), "left", "7"], - "right": [sys.executable, str(script), str(directory), "right", "0"], - "missing": [str(directory / "absent-executable")], - } - with patch.dict(os.environ, {"UV_PROJECT_ENVIRONMENT": "must-not-inherit"}): - rows = schedule.execute(directory, commands, destination) - results = {row["lane"]: row for row in rows} - self.assertEqual(results["left"]["exit_code"], 7) - self.assertEqual(results["right"]["exit_code"], 0) - self.assertNotEqual(results["missing"]["exit_code"], 0) - self.assertEqual((directory / results["left"]["log"]).read_text().strip(), "left") - self.assertEqual((directory / results["right"]["log"]).read_text().strip(), "right") - - def test_aggregate_failure(self): - with tempfile.TemporaryDirectory() as temporary: - directory = Path(temporary).resolve() - installed = directory / "wheel-ready" - generated = directory / "contracts-ready" - fixture = ( - "import pathlib, sys; " - f"assert pathlib.Path({str(installed)!r}).is_file(); " - f"assert pathlib.Path({str(generated)!r}).is_file(); " - "print('fixture complete'); sys.exit(3)" + wrapper = directory / "wrapper.py" + wrapper.write_text( + "import pathlib, sys\n" + f"sys.path.insert(0, {str(repository / 'tools/python/src')!r})\n" + "from dokime_tools.test import schedule\n" + "root = pathlib.Path.cwd()\n" + "if sys.argv[1] == 'prerequisite':\n" + " assert not (root / 'wheel-ready').exists()\n" + " sys.exit(0)\n" + "schedule.paths.target = lambda: 'fixture'\n" + "schedule.archive.install_wheels = lambda *_: (root / 'wheel-ready').touch()\n" + "schedule.generated.restore = lambda *_: (root / 'contracts-ready').touch()\n" + "def extract(directory, profile, destination):\n" + " assert not (root / 'extractions').exists()\n" + " (root / 'extractions').write_text('1')\n" + " (destination / 'ready').write_text('extracted')\n" + "schedule.archive.extract = extract\n" + "def command(category, language, profile, workers, extracted):\n" + " result = [sys.executable, str(root / 'native.py'), f'{category}-{language}']\n" + " return result + (['--archive-directory', str(extracted)] if extracted else [])\n" + "schedule.command = command\n" + "sys.exit(schedule.main(sys.argv[2:]))\n", + encoding="utf-8", ) - with ( - patch.object(schedule.policy, "root", return_value=directory), - patch.object(schedule.policy, "lanes", return_value=[("integration", "python")]), - patch.object(schedule.archive, "install_wheels", side_effect=lambda *_: installed.touch()), - patch.object(schedule.generated, "restore", side_effect=lambda *_: generated.touch()), - patch.object(schedule, "command", return_value=[sys.executable, "-c", fixture]), - ): - status = schedule.main([]) - self.assertNotEqual(status, 0) + tool = f'"{sys.executable}" "{wrapper}"' + header = f"[settings]\njobs={configuration['settings']['jobs']}\n[vars]\ntools={json.dumps(tool)}\npython={json.dumps(tool)}\nlane=''\n" + prerequisite = ( + "\n[tasks.prepare]\nrun=" + + json.dumps(tool + " prerequisite") + + "\n[tasks.'prepare:generated']\nrun=''\n[tasks.'prepare:desktop']\ndepends=['prepare']\nrun=''\n" + ) + config = directory / "mise.toml" + config.write_text(header + tasks + prerequisite, encoding="utf-8") + environment = dict(os.environ, UV_PROJECT_ENVIRONMENT="must-not-inherit") + # The Isolated Fixture Owns Different Resources From Its Calling Test Aggregate. + environment.pop("MISE_TASK_LOCKS", None) + subprocess.run(["mise", "trust", str(config)], cwd=directory, env=environment, capture_output=True, check=True) + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + listener.listen() + environment["DOKIME_GRAPH_PORT"] = str(listener.getsockname()[1]) + peers = [] + + def rendezvous(): + # Release Only After Every Lane Arrives: Serial Execution Cannot Complete. + for _ in schedule.policy.lanes(directory): + peer, _ = listener.accept() + peers.append(peer) + for peer in peers: + with peer.makefile("rb") as incoming: + self.assertEqual(incoming.read(5), b"ready") + peer.sendall(b"1") + peer.close() + + barrier = threading.Thread(target=rendezvous, daemon=True) + barrier.start() + result = subprocess.run( + ["mise", "run", "test"], cwd=directory, env=environment, capture_output=True, text=True, timeout=60, check=False + ) + barrier.join() + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) summary = next(directory.glob("target/test-reports/run-*/time-summary.json")) - result = json.loads(summary.read_text(encoding="utf-8")) - self.assertEqual(result["lanes"][0]["exit_code"], 3) - self.assertIn("fixture complete", (directory / result["lanes"][0]["log"]).read_text()) + document = json.loads(summary.read_text(encoding="utf-8")) + results = {row["lane"]: row for row in document["lanes"]} + self.assertEqual(results["system-rust"]["exit_code"], 7) + self.assertTrue(all(row["exit_code"] == 0 for name, row in results.items() if name != "system-rust")) + self.assertEqual(len(results), len(schedule.policy.lanes(directory)) + 1) + for row in results.values(): + self.assertTrue((directory / row["log"]).is_file()) self.assertTrue(summary.with_suffix(".md").is_file()) + self.assertEqual((directory / "unit-rust").read_text(), (directory / "system-rust").read_text()) + self.assertFalse(Path((directory / "unit-rust").read_text()).exists()) - def test_shared_archive(self): + def test_missing_result_fails_the_aggregate(self): with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary).resolve() - fixture = ( - "import pathlib, sys; " - "shared = pathlib.Path(sys.argv[sys.argv.index('--archive-directory') + 1]); " - "assert (shared / 'ready').read_text() == 'extracted'; " - f"(pathlib.Path({str(directory)!r}) / sys.argv[1]).write_text(str(shared))" - ) - - def extract(root, profile, destination): - (destination / "ready").write_text("extracted", encoding="utf-8") + with ( + patch.object(schedule.policy, "lanes", return_value=[("unit", "rust")]), + patch.object(schedule.archive, "install_wheels"), + patch.object(schedule.archive, "extract"), + patch.object(schedule.generated, "restore"), + ): + schedule.prepare(directory, "dev") + self.assertEqual(schedule.finish(directory, "dev"), 1) + def test_lane_retains_process_start_failure(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary).resolve() with ( - patch.object(schedule.policy, "root", return_value=directory), - patch.object(schedule.policy, "lanes", return_value=[("unit", "rust"), ("system", "rust")]), + patch.object(schedule.policy, "lanes", return_value=[("unit", "rust")]), patch.object(schedule.archive, "install_wheels"), - patch.object(schedule.archive, "extract", side_effect=extract) as extraction, + patch.object(schedule.archive, "extract"), patch.object(schedule.generated, "restore"), - patch.object( - schedule, - "command", - side_effect=lambda category, language, profile, workers, extracted: [ - sys.executable, - "-c", - fixture, - category, - "--archive-directory", - str(extracted), - ], - ), + patch.object(schedule, "command", return_value=[str(directory / "absent-executable")]), ): - status = schedule.main([]) - self.assertEqual(status, 0) - extraction.assert_called_once() - self.assertEqual((directory / "unit").read_text(), (directory / "system").read_text()) + schedule.prepare(directory, "dev") + schedule.lane(directory, "dev", "unit", "rust", "sdk/python") + self.assertEqual(schedule.finish(directory, "dev"), 1) + result = next(directory.glob("target/test-reports/run-*/unit-rust.json")) + self.assertEqual(json.loads(result.read_text(encoding="utf-8"))["exit_code"], 1) # === Public Functions === #