diff --git a/CLAUDE.md b/CLAUDE.md index 3676ea5..b9bd02d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,6 +13,27 @@ Requires: LLVM 19, clang-cpp, g++-13, C++20. ## Usage +### With YAML config (recommended) + +```bash +$TOOL --init-config > .stable-abi.yaml # generate starter config +# Edit: set pytorch_root, project_root + +$TOOL # audit (default mode) +$TOOL --mode=plan # migration plan +$TOOL --mode=rewrite --dry-run # preview changes +$TOOL --mode=rewrite # transform + auto-verify +$TOOL --mode=verify # verify stable-only +``` + +CLI flags override the config file. Positional arguments override the `transform` field: + +```bash +$TOOL --mode=rewrite ./csrc/cache.cu # transform one file using config settings +``` + +### Without config (direct CLI) + ```bash TOOL=build/stable-abi-transform PYTORCH=/path/to/pytorch # source tree or install root @@ -58,14 +79,7 @@ $TOOL --mode=plan --project-root=./csrc -- [same flags] ### YAML config -Instead of passing flags on every invocation, create `.stable-abi.yaml`: - -```bash -$TOOL --init-config # generates starter config -$TOOL # auto-loads .stable-abi.yaml -``` - -Config fields mirror CLI flags: `mode`, `format`, `pytorch_root`, `project_root`, `verify_method`, `cuda_include`, `extra_includes`, `sources`, `include_paths`, `output_dir`. Sources can be files or directories (directories are recursively walked for .cpp/.cu/.cuh files). PyTorch include paths are auto-derived from `pytorch_root` — set it to `auto` to detect from pip-installed torch. +Config fields mirror CLI flags: `mode`, `format`, `pytorch_root`, `project_root`, `verify_method`, `cuda_include`, `extra_includes`, `transform`, `include_paths`, `output_dir`, `jobs`. `transform` lists files or directories to process (directories are recursively walked for .cpp/.cu/.cuh files); if omitted, all files under `project_root` are auto-discovered. PyTorch include paths are auto-derived from `pytorch_root` — set it to `auto` to detect from pip-installed torch. ### .cuh files @@ -97,7 +111,7 @@ python3 scripts/migration_progress.py --audit-json audit.json --source-dir csrc RESOURCE_DIR=/usr/lib/clang/19 bash test/run_tests.sh ``` -13 test cases: input/expected pairs in test/inputs/ and test/expected/. +14 test cases: input/expected pairs in test/inputs/ and test/expected/. ## Architecture diff --git a/README.md b/README.md index 91a5a3f..7d906d2 100644 --- a/README.md +++ b/README.md @@ -20,17 +20,23 @@ mkdir -p build && cmake -GNinja -B build -S . && cmake --build build # Generate config ./build/stable-abi-transform --init-config > .stable-abi.yaml -# Edit: set pytorch_root, project_root, include_paths +# Edit: set pytorch_root, project_root # Audit (read-only) ./build/stable-abi-transform # mode: audit +# Migration plan (dependency-aware grouping) +./build/stable-abi-transform --mode=plan + # Preview changes -# set mode: rewrite in .stable-abi.yaml -./build/stable-abi-transform --dry-run +./build/stable-abi-transform --mode=rewrite --dry-run # Rewrite + auto-verify -./build/stable-abi-transform +./build/stable-abi-transform --mode=rewrite + +# Or without a config file — pass flags directly: +./build/stable-abi-transform --mode=audit file.cu -- -std=c++20 \ + -I/path/to/pytorch/torch/csrc/api/include -I/path/to/pytorch ``` Before/after — the tool rewrites this: @@ -58,13 +64,16 @@ void foo(const torch::stable::Tensor& input) { Patterns it can't auto-rewrite (TensorOptions decomposition, PYBIND11_MODULE replacement, project dispatch macros) are flagged with actionable guidance. +Supports `.cpp`, `.cu`, and `.cuh` files. Audit and plan modes run in parallel (`--jobs`) for large projects. Use the `transform` config field to target specific files for incremental migration. See the [demo walkthrough](docs/demo-walkthrough.md) for a real-world example analyzing vLLM (~50k lines of C++/CUDA). + ## Verification Compile-based verification creates a shadow include tree with *only* stable headers. If the file compiles against it, migration is provably complete — zero false positives, zero false negatives. See [docs/user-guide.md](docs/user-guide.md) for details. ## Documentation -- **[User Guide](docs/user-guide.md)** — full reference: modes, config, flags, verification, architecture, rule regeneration, CI integration +- **[Demo Walkthrough](docs/demo-walkthrough.md)** — real-world case study (vLLM, 235 files) +- **[User Guide](docs/user-guide.md)** — full reference: modes, config, flags, verification, architecture - **[Claude Skill](docs/claude-skill.md)** — Claude Code skill for agent-assisted migration - **[CLAUDE.md](CLAUDE.md)** — project context for Claude Code diff --git a/docs/demo-walkthrough.md b/docs/demo-walkthrough.md new file mode 100644 index 0000000..340fd29 --- /dev/null +++ b/docs/demo-walkthrough.md @@ -0,0 +1,295 @@ +# stable-abi-transform: Demo Walkthrough + +Analyzing vllm's C++ extension codebase (235 files, ~50k lines) as a real-world case study. + +## Setup + +### Generate a project config + +From your project root: + +```bash +cd /path/to/vllm +stable-abi-transform --init-config > .stable-abi.yaml +``` + +Edit `.stable-abi.yaml` to match your project: + +```yaml +mode: audit +format: text +pytorch_root: auto # detect from pip-installed torch +project_root: ./csrc + +compiler_flags: + - -std=c++20 + +# include_paths: # only needed for project-specific paths +# - ./csrc/inc # (PyTorch paths are auto-derived from pytorch_root) +``` + +Now every invocation auto-loads this config — no long CLI flags needed: + +```bash +stable-abi-transform # audit (default mode) +stable-abi-transform --mode=plan # override just the mode +stable-abi-transform --format=json # override just the format +``` + +CLI flags override the config file, so the YAML captures your project defaults and the CLI handles per-invocation choices. + +### Incremental migration with `transform` + +For a large project, you don't want to rewrite 200+ files at once. Use the `transform` field to target a subset while keeping the full project scope for include analysis: + +```yaml +project_root: ./csrc # full scope — all headers visible +transform: # only these get transformed + - ./csrc/cache_kernels.cu + - ./csrc/attention/ +``` + +Or on the CLI (positional args serve as transform targets): + +```bash +stable-abi-transform --mode=rewrite ./csrc/cache_kernels.cu +``` + +Headers included from the target files that live under `project_root` are still in rewrite scope — shared headers get transformed correctly. Files outside `transform` are untouched. + +Omit `transform` to process everything under `project_root`. + +## Step 1: Audit — What needs to change? + +```bash +stable-abi-transform +``` + +The tool auto-discovers all `.cpp`, `.cu`, and `.cuh` files under `project_root` and reports every usage of unstable PyTorch internals. Audit runs in parallel by default (`--jobs=0` auto-detects cores): + +``` +[MACRO] csrc/cache_kernels.cu:45:5 TORCH_CHECK -> STD_TORCH_CHECK +[TYPE ] csrc/sampler.cu:12:18 torch::Tensor -> torch::stable::Tensor +[DPTR ] csrc/sampler.cu:30:22 .data_ptr() -> .mutable_data_ptr() +[GUARD] csrc/sampler.cu:8:5 at::cuda::OptionalCUDAGuard -> DeviceGuard +[STRM ] csrc/sampler.cu:9:22 at::cuda::getCurrentCUDAStream() -> aoti_torch_get_current_cuda_stream() +[M2F ] csrc/attention/paged_attention_v2.cu:190:36 dtype -> scalar_type +[INCL ] csrc/moe/moe_ops.h:3:1 #include -> (removed, no torch API usage) +[FLAG ] csrc/cpu/sgl-kernels/vec.h:20:1 at::vec::Vectorized -> no stable equivalent +``` + +Each finding is either: +- **Auto-rewritable** — the tool can transform this automatically +- **Flagged** — requires manual review (no stable equivalent, or inside a macro body) + +### vllm results (snapshot, May 2026) + +``` +Summary: 2,554 auto-rewritable, 429 flagged for manual review +``` + +| Category | Description | Auto | Flag | Total | +|----------|-------------|------|------|-------| +| MACRO | `TORCH_CHECK` -> `STD_TORCH_CHECK`, etc. | 841 | 83 | 924 | +| DPTR | `.data_ptr()` -> `.mutable_data_ptr()` | 593 | 2 | 595 | +| TYPE | `at::Tensor` -> `torch::stable::Tensor`, etc. | 499 | 72 | 571 | +| FLAG | Unstable APIs with no stable equivalent | 0 | 209 | 209 | +| INCL | `#include ` -> stable headers | 152 | 26 | 178 | +| STYPE | `at::kFloat` -> `torch::headeronly::ScalarType::Float` | 146 | 29 | 175 | +| FUNC | `at::empty(...)` -> `torch::stable::empty(...)` | 101 | 4 | 105 | +| M2F | `.clone()` -> `torch::stable::clone(t)` | 93 | 0 | 93 | +| STRM | CUDA stream acquisition | 69 | 4 | 73 | +| GUARD | `CUDAGuard` -> `DeviceGuard` | 60 | 0 | 60 | + +For CI integration or migration tracking, use JSON output: + +```bash +stable-abi-transform --format=json +``` + +## Step 2: Plan — How to sequence the migration + +```bash +stable-abi-transform --mode=plan +``` + +Plan mode analyzes the include dependency graph and partitions files into groups that must migrate together (they share headers with unstable API usage). Like audit, it runs in parallel by default: + +``` +Migration plan: 13 groups (all independent — transform in parallel) + +Round 1: + Group A (76 files, 1523 findings): + csrc/activation_kernels.cu + csrc/cache_kernels.cu + csrc/sampler.cu + csrc/torch_bindings.cpp + ... (46 sources, 30 headers) + + Group B (1 file, 37 findings): + csrc/attention/vertical_slash_index.cu + + Group C (14 files, 97 findings): + csrc/cpu/activation.cpp + csrc/cpu/shm.cpp + ... (10 sources, 4 headers) + + Group D (15 files, 943 findings): + csrc/cpu/sgl-kernels/conv.cpp + csrc/cpu/sgl-kernels/gemm.cpp + ... (10 sources, 5 headers) + + Group E (1 file, 20 findings): csrc/cuda_view.cu + Group F (1 file, 53 findings): csrc/custom_all_reduce.cu + Group G (1 file, 4 findings): csrc/custom_quickreduce.cu + Group H (2 files, 149 findings): csrc/mamba/mamba_ssm/selective_scan_fwd.cu + Group I (1 file, 53 findings): csrc/moe/dynamic_4bit_int_moe_cpu.cpp + Group J (1 file, 35 findings): csrc/moe/grouped_topk_kernels.cu + Group K (1 file, 11 findings): csrc/moe/moe_wna16.cu + Group L (1 file, 11 findings): csrc/quantization/gptq/q_gemm.cu + Group M (1 file, 54 findings): csrc/topk.cu +``` + +Each group lists the specific unstable APIs it uses, so reviewers know the scope before starting. Groups are independent — they can be separate PRs merged in any order. + +Use `--format=json` for machine-readable plan output (dependency-aware PR grouping in CI). + +## Step 3: Preview — Dry-run before committing + +```bash +stable-abi-transform --mode=rewrite --dry-run +``` + +Shows a unified diff of every change the tool would make, without modifying any files: + +```diff +- TORCH_CHECK(group_size <= MAX_SHM_RANK_NUM); ++ STD_TORCH_CHECK(group_size <= MAX_SHM_RANK_NUM); +- TORCH_CHECK_EQ(ptr->thread_num, thread_num); ++ STD_TORCH_CHECK((ptr->thread_num) == (thread_num)); +``` + +To preview a single file instead of the full project: + +```bash +stable-abi-transform --mode=rewrite --dry-run csrc/cpu/shm.cpp +``` + +## Step 4: Rewrite — Apply transformations + +### In-place + +```bash +stable-abi-transform --mode=rewrite +``` + +Transforms all files under `project_root` and runs post-rewrite verification automatically. + +### Out-of-place (keep originals intact) + +Add `output_dir` to your config: + +```yaml +mode: rewrite +output_dir: ./csrc-stable +``` + +Or on the CLI: + +```bash +stable-abi-transform --mode=rewrite --output-dir=./csrc-stable +``` + +Transformed files are written to `csrc-stable/` preserving directory structure. Originals are untouched. + +## Step 5: Verify — Confirm no unstable API remains + +After rewriting, verify the result: + +```bash +stable-abi-transform --mode=verify +``` + +**Compile-based verification** (default, requires `pytorch_root`): Creates a shadow include tree with symlinks to only the stable PyTorch headers, then compiles the file against it. If it compiles, the file uses only stable ABI. If not, the compiler errors *are* the gap analysis — precise, zero false positives. + +**Regex-based verification** (no PyTorch source needed): + +```bash +stable-abi-transform --mode=verify --verify-method=regex +``` + +## What the tool does automatically + +| Transformation | Before | After | +|----------------|--------|-------| +| Macro rename | `TORCH_CHECK(x)` | `STD_TORCH_CHECK(x)` | +| Comparison macro | `TORCH_CHECK_EQ(a, b)` | `STD_TORCH_CHECK((a) == (b))` | +| Type rename | `at::Tensor` | `torch::stable::Tensor` | +| data_ptr (mutable) | `t.data_ptr()` | `t.mutable_data_ptr()` | +| data_ptr (const) | `ct.data_ptr()` | `ct.const_data_ptr()` | +| Method to function | `t.clone()` | `torch::stable::clone(t)` | +| Method rename | `t.dtype()` | `t.scalar_type()` | +| nbytes | `t.nbytes()` | `t.numel() * t.element_size()` | +| sizes indexing | `t.sizes()[0]` | `t.size(0)` | +| Free function | `at::empty({3}, at::kFloat)` | `torch::stable::empty({3}, ...)` | +| Scalar type | `at::kFloat` | `torch::headeronly::ScalarType::Float` | +| Include | `#include ` | stable header set | +| CUDA guard | `at::cuda::OptionalCUDAGuard` | `torch::stable::accelerator::DeviceGuard` | +| CUDA stream | `at::cuda::getCurrentCUDAStream()` | `aoti_torch_get_current_cuda_stream()` | +| nullopt/optional | `c10::optional` / `c10::nullopt` | `std::optional` / `std::nullopt` | +| Dispatch macro | `AT_DISPATCH_FLOATING_TYPES` | `THO_DISPATCH_V2` + `AT_FLOATING_TYPES` | +| Op registration | `TORCH_LIBRARY` | `STABLE_TORCH_LIBRARY` | + +For vllm: **2,554 automatic transformations** across 235 files. + +## What requires manual work + +### No stable equivalent (209 findings in vllm) + +PyTorch internal APIs that have not been stabilized yet: + +- `at::vec::Vectorized` — SIMD vectorization (CPU kernels) +- `at::native::cpublas::brgemm` — CPU BLAS routines +- `at::ones` — factory function not yet in stable ops +- `at::stack`, `at::clamp`, `at::sigmoid`, `at::silu` — higher-level ops +- `c10::InferenceMode` — inference mode guard + +These require waiting for PyTorch to stabilize the API, or restructuring code to use stable primitives. + +### Inside macro bodies (184 findings in vllm) + +The tool detects unstable API usage inside `#define` bodies but cannot auto-rewrite them: + +```cpp +#define MY_CHECK(x) TORCH_CHECK(x, "failed") // tool flags but can't rewrite +``` + +The tool reports exactly which macro and what the replacement should be. The edit is manual but straightforward. + +### Type boundaries across files + +`at::Tensor` and `torch::stable::Tensor` are distinct C++ types. Changing a function's signature in one file without updating all callers causes compilation errors. The plan mode identifies which files share headers and must migrate together, but verifying that function signatures are consistent across translation units requires human review. + +Use the plan groups as PR boundaries. Within each group, ensure functions declared in shared headers have consistent signatures. + +## Typical workflow + +```bash +# 1. One-time setup +stable-abi-transform --init-config > .stable-abi.yaml +vim .stable-abi.yaml # set pytorch_root, project_root, transform + +# 2. Assess the scope +stable-abi-transform # audit — see what needs to change +stable-abi-transform --mode=plan # plan — see how to group the work + +# 3. Review before touching code +stable-abi-transform --mode=rewrite --dry-run # preview all changes + +# 4. Apply and verify +stable-abi-transform --mode=rewrite # transform +stable-abi-transform --mode=verify # confirm stable-only + +# 5. Build and test +make -j30 && python -m pytest tests/ +``` diff --git a/docs/user-guide.md b/docs/user-guide.md index e398eba..1fad76b 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -5,9 +5,10 @@ ### Prerequisites - Linux (tested on Fedora 41) -- LLVM/Clang 19 development libraries, CUDA 12.9 headers +- LLVM/Clang 19 development libraries - C++20 compiler - PyTorch headers (via `pip install torch`, libtorch download, or source tree) +- CUDA headers (optional, for `.cu`/`.cuh` analysis) ### Build @@ -40,12 +41,11 @@ project_root: ./csrc compiler_flags: - -std=c++20 -include_paths: - - /usr/local/cuda/include - - ./csrc/inc # project-specific headers +# include_paths: # only needed for project-specific paths +# - ./csrc/inc # (PyTorch paths are auto-derived) -extra_includes: # for verify mode - - ./csrc +# extra_includes: # for verify mode +# - ./csrc ``` PyTorch include paths are auto-derived from `pytorch_root` — you only need project-specific paths in `include_paths`. Set `pytorch_root` to `auto` (detects from `pip install torch`), or an explicit path to a PyTorch source tree or libtorch download. @@ -56,17 +56,53 @@ When `.stable-abi.yaml` exists in the working directory, all commands auto-load stable-abi-transform # uses .stable-abi.yaml ``` +CLI flags override the config file, so the YAML captures your project defaults and the CLI handles per-invocation choices. + +### Incremental migration with `transform` + +For large projects, you don't want to transform everything at once. Use the `transform` field to target specific files or directories while keeping the full project scope for include analysis: + +```yaml +project_root: ./csrc # full scope — all headers visible +transform: # only these get transformed + - ./csrc/attention/ + - ./csrc/cache_kernels.cu +``` + +Or on the CLI (positional args serve as transform targets): + +```bash +stable-abi-transform --mode=rewrite ./csrc/cache_kernels.cu +``` + +Headers included from the target files that live under `project_root` are still in rewrite scope — shared headers get transformed correctly. Files outside `transform` are untouched. + +Omit `transform` to process everything under `project_root`. + ## Modes | Mode | What it does | Modifies files? | |------|-------------|-----------------| | `audit` (default) | Reports all unstable API usage with suggested replacements | No | +| `plan` | Analyzes include dependencies and partitions files into independent migration groups | No | | `rewrite` | Transforms source files in-place, then auto-verifies | Yes | | `rewrite` + `--dry-run` | Shows unified diff of what rewrite would produce | No | | `verify` | Checks if a file uses only stable APIs | No | Switch modes by editing `mode:` in `.stable-abi.yaml` or passing `--mode=` on the command line. +### Plan mode + +Plan mode analyzes the include dependency graph and partitions files into groups that must migrate together (they share headers with unstable API usage): + +```bash +stable-abi-transform --mode=plan +``` + +Groups are independent — they can be separate PRs merged in any order. Each group lists the specific unstable APIs it uses, so reviewers know the scope before starting. + +Use `--format=json` for machine-readable plan output (dependency-aware PR grouping in CI). + ### Output formats - `--format=text` (default) — human-readable report @@ -79,6 +115,7 @@ Switch modes by editing `mode:` in `.stable-abi.yaml` or passing `--mode=` on th | Includes | `torch/all.h` -> stable headers | Yes | | Types | `at::Tensor` -> `torch::stable::Tensor` | Yes | | Macros | `TORCH_CHECK` -> `STD_TORCH_CHECK` | Yes | +| Comparison macros | `TORCH_CHECK_EQ(a, b)` -> `STD_TORCH_CHECK((a) == (b))` | Yes | | Scalar types | `at::kFloat` -> `torch::headeronly::ScalarType::Float` | Yes | | Device types | `at::kCUDA` -> `torch::headeronly::DeviceType::CUDA` | Yes | | data_ptr | `t.data_ptr()` -> `mutable_data_ptr` or `const_data_ptr` (AST const-analysis) | Yes | @@ -88,7 +125,6 @@ Switch modes by editing `mode:` in `.stable-abi.yaml` or passing `--mode=` on th | CUDA streams | `getCurrentCUDAStream()` -> `aoti_torch_get_current_cuda_stream()` | Yes | | Op registration | `TORCH_LIBRARY` -> `STABLE_TORCH_LIBRARY` | Yes | | Dispatch macros | `AT_DISPATCH_SWITCH` -> `THO_DISPATCH_SWITCH` | Yes | -| Comparison macros | `TORCH_CHECK_EQ(a, b)` -> `STD_TORCH_CHECK((a) == (b))` | Yes | | `c10::optional` | -> `std::optional` | Yes | | `c10::nullopt` | -> `std::nullopt` | Yes | | `c10::ArrayRef` | -> `torch::headeronly::HeaderOnlyArrayRef` | Yes | @@ -106,6 +142,45 @@ Switch modes by editing `mode:` in `.stable-abi.yaml` or passing `--mode=` on th The tool has AST-based catch-all matchers that flag ANY remaining `at::`, `c10::`, or unstable `torch::` usage. No unstable pattern can silently pass through unreported. +## .cuh files + +`.cuh` (CUDA header) files are automatically discovered and analyzed alongside `.cpp` and `.cu` files. They are parsed as C++ — no CUDA toolchain is required. CUDA device-only constructs may cause parse errors that are reported per-file but do not block analysis of host-side API calls. + +## Parallel execution + +Audit and plan modes process translation units in parallel using a thread pool: + +```bash +stable-abi-transform --jobs=8 # 8 threads +stable-abi-transform --jobs=0 # auto-detect (default) +stable-abi-transform --jobs=1 # sequential +``` + +Or in config: + +```yaml +jobs: 0 # 0 = auto-detect, 1 = sequential +``` + +Rewrite mode always runs sequentially (parallel rewrite with shared header deduplication is planned for v0.3.0). + +## Out-of-place rewrite + +To keep originals intact, write transformed files to a separate directory: + +```yaml +mode: rewrite +output_dir: ./csrc-stable +``` + +Or on the CLI: + +```bash +stable-abi-transform --mode=rewrite --output-dir=./csrc-stable +``` + +Only modified files are written, preserving directory structure relative to `project_root`. + ## Verification ### Compile-based (default) @@ -134,21 +209,72 @@ verify_method: regex # in .stable-abi.yaml Pattern-matches against known unstable namespaces. Faster but less precise — use compile-based when possible. +## Migration progress + +`scripts/migration_progress.py` measures how close a project is to stable ABI adoption: + +```bash +# Audit then report progress +python3 scripts/migration_progress.py /path/to/project --source-dir csrc + +# From pre-computed audit JSON +python3 scripts/migration_progress.py --audit-json audit.json --source-dir csrc + +# With dependency-aware PR grouping (from --mode=plan --format=json) +python3 scripts/migration_progress.py --audit-json audit.json --source-dir csrc \ + --plan-json plan.json + +# Machine-readable +python3 scripts/migration_progress.py --audit-json audit.json --source-dir csrc \ + --format json +``` + ## CLI reference | Flag | Description | |------|-------------| -| `--mode` | `audit`, `rewrite`, or `verify` | -| `--pytorch-root` | Path to PyTorch source/install root | -| `--project-root` | Process all C++/CUDA files under this directory | +| `--mode` | `audit` (default), `rewrite`, `verify`, or `plan` | +| `--pytorch-root` | Path to PyTorch source/install root, or `auto` to detect from pip | +| `--project-root` | Project root directory; auto-discovers source files and bounds rewrites | +| `--output-dir` | Write transformed files here instead of in-place (requires `--project-root`) | | `--extra-includes` | Project-specific `-I` paths for verify mode | | `--cuda-include` | CUDA include path (auto-detected from `/usr/local/cuda/include`) | | `--verify-method` | `compile` (default) or `regex` | | `--format` | `text` (default) or `json` | | `--dry-run` | Preview rewrite as unified diff (rewrite mode only) | +| `--jobs` | Parallel TU threads: 0 = auto (default), 1 = sequential. Audit and plan only. | | `--config` | Path to YAML config file (default: `.stable-abi.yaml`) | | `--init-config` | Generate a starter config and exit | +Positional arguments are treated as transform targets (files or directories to process). + +## CLI without a config file + +The tool works without `.stable-abi.yaml` — pass source files and compiler flags directly: + +```bash +TOOL=build/stable-abi-transform +PYTORCH=/path/to/pytorch + +# Audit a single file +$TOOL file.cu -- -std=c++20 -I$PYTORCH/torch/csrc/api/include \ + -I$PYTORCH -I$PYTORCH/torch/include + +# Rewrite in-place +$TOOL --mode=rewrite --pytorch-root=$PYTORCH file.cu -- -std=c++20 \ + -I$PYTORCH/torch/csrc/api/include -I$PYTORCH -I$PYTORCH/torch/include + +# Verify +$TOOL --mode=verify --pytorch-root=$PYTORCH \ + --extra-includes=./csrc file.cu -- -std=c++20 + +# Plan (requires --project-root) +$TOOL --mode=plan --project-root=./csrc -- -std=c++20 \ + -I$PYTORCH/torch/csrc/api/include -I$PYTORCH -I$PYTORCH/torch/include +``` + +Everything after `--` is passed to clang. The config file eliminates the repeated flags — it's recommended for any project you'll run the tool on more than once. + ## CI integration ```bash @@ -156,23 +282,29 @@ Pattern-matches against known unstable namespaces. Faster but less precise — u # audit/dry-run: 0 = no unstable API found, 1 = findings exist # rewrite: 0 = all auto-fixed, 1 = flagged items remain # verify: 0 = stable ABI only, 1 = unstable API detected +# plan: always 0 -stable-abi-transform --mode=audit --format=json src/*.cu -- [flags] +# With config (recommended) +stable-abi-transform --mode=audit --format=json -# Or verify already-migrated files +# Without config +stable-abi-transform --mode=audit --format=json src/*.cu -- -std=c++20 \ + -I$PYTORCH/torch/csrc/api/include -I$PYTORCH -I$PYTORCH/torch/include + +# Verify already-migrated files stable-abi-transform --mode=verify --pytorch-root=$PYTORCH src/*.cu -- -std=c++20 ``` -### Using a config file with CLI overrides +### CLI overrides -When `.stable-abi.yaml` exists, the tool auto-loads it. You can still override which files to process on the command line: +When `.stable-abi.yaml` exists, the tool auto-loads it. CLI flags override config values, and positional arguments override `transform`: ```bash # Process a specific file using config's compiler settings stable-abi-transform src/my_kernel.cu -# Config controls compiler_flags and include_paths; -# -- flags on the CLI are ignored in config mode +# Override mode for this invocation +stable-abi-transform --mode=plan ``` ## Regenerating rules @@ -189,16 +321,21 @@ Parses `torch/csrc/stable/` and `torch/headeronly/` to extract the complete stab ``` src/ - main.cpp CLI, mode dispatch, ClangTool configuration + main.cpp CLI, mode dispatch, parallel execution, ClangTool config StableAbiAction.{h,cpp} Clang FrontendAction + ASTConsumer wiring TransformerRules.cpp AST-based rewrite rules (Clang Transformer) AstCallbacks.{h,cpp} Manual AST matchers (CUDA guard/stream) PreprocessorCallbacks.cpp Include and macro rewrites Reporter.{h,cpp} Finding accumulation, text/JSON output Verifier.{h,cpp} Compile-based and regex-based verification - Rules.h Auto-generated transformation tables Config.{h,cpp} YAML config support - Helpers.h Shared utilities + DepGraph.{h,cpp} Include DAG analysis, migration partitioning, plan output + Rules.h Auto-generated transformation tables + Helpers.h Shared utilities (project scope check, path helpers) + +scripts/ + gen_rules.py Derives Rules.h from PyTorch stable ABI headers + migration_progress.py Migration progress reporting from audit/plan JSON ``` ### How verification works internally @@ -220,4 +357,4 @@ PYTORCH_DIR=/path/to/pytorch RESOURCE_DIR=/usr/lib/clang/19 bash test/run_tests. `PYTORCH_DIR` is required (points to your PyTorch source tree). `RESOURCE_DIR` defaults to `/usr/lib/clang/19`. -13 test cases with input/expected pairs in `test/inputs/` and `test/expected/`. The test suite runs four passes: rewrite correctness, regex verification, exit code validation, and compile-based verification. +14 test cases with input/expected pairs in `test/inputs/` and `test/expected/`. The test suite runs four passes: rewrite correctness, regex verification, exit code validation, and compile-based verification. diff --git a/src/Config.cpp b/src/Config.cpp index d7afed8..547062b 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -89,10 +89,10 @@ bool loadConfig(const std::string &path, Config &out, std::string &error) { makeAbsolute(p, base); warnUnexpanded(p, "extra_includes"); } - for (auto &p : out.sources) { + for (auto &p : out.transform) { expandVars(p, out); makeAbsolute(p, base); - warnUnexpanded(p, "sources"); + warnUnexpanded(p, "transform"); } warnUnexpanded(out.pytorch_root, "pytorch_root"); warnUnexpanded(out.project_root, "project_root"); @@ -178,7 +178,7 @@ format: text pytorch_root: auto # Project root — rewrites files under this path (headers included). -# Also auto-discovers .cpp/.cu source files when 'sources' is omitted. +# Also auto-discovers .cpp/.cu source files when 'transform' is omitted. project_root: ./csrc # Compiler flags passed to clang @@ -195,9 +195,11 @@ project_root: ./csrc # extra_includes: # - ./csrc -# Explicit source files or directories (optional — auto-discovered from project_root if omitted) +# Files or directories to transform (optional — auto-discovered from project_root if omitted). # Directory entries are recursively walked for .cpp/.cu/.cuh files. -# sources: +# Use this for incremental migration: set project_root for full include scope, +# then list only the files you want to transform in this run. +# transform: # - csrc/attention/ # walk entire directory # - csrc/cache.cu # single file diff --git a/src/Config.h b/src/Config.h index 8dcb34d..9fd31d8 100644 --- a/src/Config.h +++ b/src/Config.h @@ -18,7 +18,7 @@ struct Config { std::vector compiler_flags; std::vector include_paths; std::vector extra_includes; - std::vector sources; + std::vector transform; VerifyMethod verify_method = VerifyMethod::Compile; std::string cuda_include; std::string output_dir; @@ -68,7 +68,7 @@ template <> struct MappingTraits { io.mapOptional("compiler_flags", c.compiler_flags); io.mapOptional("include_paths", c.include_paths); io.mapOptional("extra_includes", c.extra_includes); - io.mapOptional("sources", c.sources); + io.mapOptional("transform", c.transform); io.mapOptional("verify_method", c.verify_method); io.mapOptional("cuda_include", c.cuda_include); io.mapOptional("output_dir", c.output_dir); diff --git a/src/Helpers.h b/src/Helpers.h index e8d5e19..fbb0b8c 100644 --- a/src/Helpers.h +++ b/src/Helpers.h @@ -101,14 +101,21 @@ inline void addReplacement(FileReplacements &fileRepls, inline std::string jsonEscape(std::string_view s) { std::string out; out.reserve(s.size()); - for (char c : s) { + for (unsigned char c : s) { switch (c) { case '"': out += "\\\""; break; case '\\': out += "\\\\"; break; case '\n': out += "\\n"; break; case '\r': out += "\\r"; break; case '\t': out += "\\t"; break; - default: out += c; + default: + if (c < 0x20) { + char buf[8]; + snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out += static_cast(c); + } } } return out; diff --git a/src/Reporter.cpp b/src/Reporter.cpp index dbb7955..b6d7531 100644 --- a/src/Reporter.cpp +++ b/src/Reporter.cpp @@ -121,6 +121,16 @@ void Reporter::printParseWarnings() const { llvm::errs() << " " << file << ": " << count << " error(s)\n"; } +void Reporter::sortFindings() { + std::sort(findings_.begin(), findings_.end(), + [](const Finding &a, const Finding &b) { + if (a.file != b.file) return a.file < b.file; + if (a.line != b.line) return a.line < b.line; + if (a.col != b.col) return a.col < b.col; + return a.old_text < b.old_text; + }); +} + void Reporter::suppressRedundantFlags() { std::set> covered; for (const auto &f : findings_) diff --git a/src/Reporter.h b/src/Reporter.h index 44e9eed..22ae619 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -53,6 +53,7 @@ class Reporter { void printJson() const; void suppressRedundantFlags(); + void sortFindings(); [[nodiscard]] size_t rewriteCount() const { return rewrite_count_; } [[nodiscard]] size_t flagCount() const { return flag_count_; } diff --git a/src/StableAbiAction.cpp b/src/StableAbiAction.cpp index a33f93e..2251065 100644 --- a/src/StableAbiAction.cpp +++ b/src/StableAbiAction.cpp @@ -187,8 +187,15 @@ void StableAbiFrontendAction::EndSourceFileAction() { llvm::StringRef srcPath = optRef->getName(); llvm::SmallString<256> relPath(srcPath); - llvm::sys::path::replace_path_prefix( - relPath, opts_.project_root, ""); + if (!llvm::sys::path::replace_path_prefix( + relPath, opts_.project_root, "")) { + llvm::errs() << "warning: skipping file outside project root: " + << srcPath << "\n"; + continue; + } + while (!relPath.empty() && + llvm::sys::path::is_separator(relPath[0])) + relPath.erase(relPath.begin()); llvm::SmallString<256> outPath(opts_.output_dir); llvm::sys::path::append(outPath, relPath); diff --git a/src/main.cpp b/src/main.cpp index a16f8c8..bf2dbd5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -61,7 +61,7 @@ static llvm::cl::opt static llvm::cl::opt ProjectRoot("project-root", llvm::cl::desc("Project root directory — rewrites files under this path (not just main file). " - "Also auto-discovers .cpp/.cu source files when no sources given."), + "Also auto-discovers .cpp/.cu source files when no transform targets given."), llvm::cl::init(""), llvm::cl::cat(ToolCategory)); static llvm::cl::opt @@ -302,7 +302,7 @@ static int runWithConfig(stable_abi::Config &cfg, projectRoot = std::string(abs); } - std::vector sources = expandSources(cfg.sources); + std::vector sources = expandSources(cfg.transform); if (sources.empty() && !projectRoot.empty()) { if (!llvm::sys::fs::is_directory(projectRoot)) { llvm::errs() << "error: project root is not a directory: " @@ -466,6 +466,7 @@ static int runWithConfig(stable_abi::Config &cfg, return 0; } + reporter.sortFindings(); reporter.suppressRedundantFlags(); if (json) { reporter.printJson(); @@ -551,7 +552,7 @@ int main(int argc, const char **argv) { } auto &cliSources = OptionsParser.getSourcePathList(); if (!cliSources.empty()) - cfg.sources = cliSources; + cfg.transform = cliSources; return runWithConfig(cfg); } @@ -583,6 +584,6 @@ int main(int argc, const char **argv) { llvm::errs() << "error: " << resolveErr << "\n"; return 1; } - cfg.sources = cliSources; + cfg.transform = cliSources; return runWithConfig(cfg, &OptionsParser.getCompilations()); } diff --git a/test/run_tests.sh b/test/run_tests.sh index 7e23915..52dd3ff 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -29,6 +29,21 @@ COMMON_ARGS=( -I"$CUDA_INCLUDE" ) +echo "--- Input integrity check ---" +for input_file in "$INPUTS"/*.cpp "$INPUTS"/*.cu; do + [ -f "$input_file" ] || continue + basename="$(basename "$input_file")" + expected_file="$EXPECTED/$basename" + [ -f "$expected_file" ] || continue + if diff -q "$input_file" "$expected_file" > /dev/null 2>&1; then + echo "CORRUPT $basename: input is identical to expected output" + echo " Run: git checkout HEAD -- test/inputs/$basename" + exit 1 + fi +done +echo "All inputs differ from expected outputs." +echo "" + passed=0 failed=0 skipped=0