diff --git a/.github/workflows/TestingCI.yml b/.github/workflows/TestingCI.yml index aba81b4..66d42c1 100644 --- a/.github/workflows/TestingCI.yml +++ b/.github/workflows/TestingCI.yml @@ -74,5 +74,58 @@ jobs: restore-keys: ${{ runner.os }}-cargo- - name: Build run: cargo build --verbose + # Serialized like the Linux jobs. The REPL tests spawn the binary as a + # subprocess and several tests share fixtures under tests/data/, so + # running them in parallel here made macOS and Windows flaky in a way + # Linux never showed. - name: Run tests - run: cargo test --verbose \ No newline at end of file + run: cargo test --verbose + env: + RUST_TEST_THREADS: 1 + + lint: + name: Format and Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-lint-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-lint- + - name: Check formatting + run: cargo fmt --all -- --check + - name: Clippy + run: cargo clippy --all-targets -- -D warnings + + msrv: + name: Minimum Supported Rust Version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # Pinned to the rust-version declared in Cargo.toml. That field claimed + # 1.70 for a long time while the dependency tree had moved well past it, + # because nothing ever checked. + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@1.88 + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-msrv-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo-msrv- + # Library and binaries only: rust-version is a promise to consumers, and + # dev-dependencies are not part of it. + - name: Check build at MSRV + run: cargo check --verbose diff --git a/CLAUDE.md b/CLAUDE.md index 70966aa..1053081 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,10 +50,22 @@ SQL execution uses a bytecode VM inspired by SQLite's architecture: ### VM Module (`src/vm/`) -- `bytecode.rs`: Defines bytecode instruction set -- `compiler.rs`: Compiles SQL AST to bytecode +- `bytecode.rs`: Bytecode instruction set and register/program types +- `compiler.rs`: SQL AST to bytecode. Holds `code_expr`, the single expression + compiler (the analogue of SQLite's `sqlite3ExprCode`) used from the SELECT + list, WHERE, HAVING, ORDER BY and SET clauses alike, plus `NameCtx`, which + carries the FROM sources a name can resolve against +- `compiler_aggregate.rs`: aggregates, GROUP BY, HAVING +- `compiler_join.rs`: joins +- `compiler_dml.rs`: INSERT / UPDATE / DELETE +- `compiler_ddl.rs`: CREATE / DROP / ALTER / TRUNCATE +- `compiler_window.rs`: window functions +- `ast_compat.rs`: thin accessors normalizing sqlparser AST shapes - `engine.rs`: Executes bytecode instructions +Each statement is compiled and executed separately, so a statement observes the +effects of the ones before it. + ### File Handling - **`file_handler.rs`**: Manages loading files into database tables @@ -62,9 +74,10 @@ SQL execution uses a bytecode VM inspired by SQLite's architecture: ### SQL Features -- `aggregate.rs`: COUNT, SUM, AVG, MIN, MAX functions -- `join.rs`: Cross join and INNER JOIN implementations -- `string_functions.rs`: UPPER, LOWER, TRIM, SUBSTR, REPLACE +Expression compilation, including all scalar and string functions, lives in +`src/vm/compiler.rs` behind `code_expr`. Aggregates are compiled in +`src/vm/compiler_aggregate.rs` and executed by the `AggStep`/`AggFinal` +opcodes; `src/aggregate.rs` retains only the function-name lookup. ### Safe Writeback Model @@ -79,4 +92,15 @@ Tests are in `tests/` organized by functionality: - `helpers/`: Test utilities - `common.rs`: Shared test helpers including REPL script runner +Two suites carry the correctness contract: + +- `tests/golden/`: exact-output characterization tests. They assert the + COMPLETE stdout -- every row, in order -- via `assert_query`. The rest of the + suite uses `predicate::str::contains`, which is substring matching: a test + expecting `"Engineering,3"` passes even when extra wrong rows are emitted and + regardless of row order. Prefer `assert_query` for new tests. +- `tests/defects/`: `#[ignore]`d tests encoding CORRECT behaviour for known + defects. Run with `cargo test --test mod defects -- --ignored`. Remove the + `#[ignore]` when a defect is fixed; the list only shrinks. + Tests use `assert_cmd` for CLI testing and `tempfile` for temporary test data. diff --git a/Cargo.lock b/Cargo.lock index 4e5e4b2..7d926f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,15 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + [[package]] name = "assert_cmd" version = "2.1.2" @@ -105,6 +114,16 @@ dependencies = [ "serde", ] +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -238,6 +257,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + [[package]] name = "float-cmp" version = "0.10.0" @@ -355,6 +380,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -406,6 +440,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "quote" version = "1.0.43" @@ -431,6 +475,26 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "regex" version = "1.12.2" @@ -530,6 +594,12 @@ dependencies = [ "syn", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "smallvec" version = "1.15.1" @@ -555,11 +625,25 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.36.1" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eaa1e88e78d2c2460d78b7dc3f0c08dbb606ab4222f9aff36f420d36e307d87" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", + "recursive", +] + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d6c0cc1..ac98a9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,13 +10,24 @@ homepage = "https://github.com/jgarzik/sqawk" readme = "README.md" keywords = ["csv", "tsv", "sql", "delimited", "awk"] categories = ["command-line-utilities", "text-processing", "parser-implementations"] -# Minimum version of Rust required -rust-version = "1.70.0" -exclude = ["tests/", "doc/", ".github/", "CLAUDE.md"] +# Minimum supported Rust version. +# +# Verified by building with that toolchain, not inferred. The binding +# constraints are transitive: `home` (via rustyline) and `psm` (via +# sqlparser -> recursive -> stacker) both require 1.88. +rust-version = "1.88" +exclude = [ + "tests/", + "doc/", + ".github/", + "CLAUDE.md", + "generated-icon.png", + ".replit", +] [dependencies] clap = { version = "4", features = ["derive"] } -sqlparser = "0.36" +sqlparser = "0.62" csv = "1" anyhow = "1" regex = "1" diff --git a/README.md b/README.md index 9ecb126..b8b3a53 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ Sqawk is an SQL-based command-line tool for processing delimiter-separated files - **Joins** - INNER, LEFT, RIGHT, FULL OUTER, and CROSS joins with ON conditions - **Aggregates** - COUNT, SUM, AVG, MIN, MAX with GROUP BY support - **Functions** - String (UPPER, LOWER, SUBSTR, REPLACE, etc.), math (ABS, ROUND, etc.), date/time +- **Subqueries** - Scalar, `IN (SELECT ...)`, and `EXISTS`, including correlated +- **Set Operations** - UNION, UNION ALL, INTERSECT, EXCEPT +- **Window Functions** - ROW_NUMBER, RANK, DENSE_RANK, LAG, LEAD, and aggregates with `OVER (PARTITION BY ... ORDER BY ...)` +- **DDL** - CREATE TABLE, CREATE TABLE AS SELECT, DROP, ALTER TABLE ADD COLUMN, TRUNCATE +- **Expressions** - CASE, CAST, COALESCE, NULLIF, BETWEEN, IN, LIKE/ILIKE, `||`, arithmetic - **File Formats** - CSV, TSV, and custom delimiters; headerless files via `--tabledef` - **Safe by Default** - Files unchanged unless `--write` flag is specified - **Interactive REPL** - Explore data interactively with `-i` flag @@ -38,6 +43,29 @@ sqawk -s "SELECT department, AVG(salary) FROM employees GROUP BY department" emp sqawk -s "UPDATE data SET status = 'archived' WHERE year < 2020" data.csv --write ``` +## `tsq` - test data generator + +`cargo install sqawk` also installs `tsq`, which generates deterministic +multi-table CSV data plus a corpus of SQL queries for exercising sqawk. + +```sh +tsq --seed 42 --rows 1000 --output-dir /tmp/sqawk-test +sqawk -s "SELECT * FROM customers LIMIT 10" /tmp/sqawk-test/data/customers.csv +``` + +It writes `data/` (customers, products, orders, order_items, reviews, with +realistic foreign-key relationships), `queries/` (numbered `.sql` files +covering selects, joins, aggregates, subqueries and window functions), +`verify/run_verification.sh`, and a `metadata.json` recording the seed and row +counts. The same seed always produces the same data. + +| Option | Meaning | +| --- | --- | +| `-s`, `--seed` | Seed for reproducible generation; a random one is printed if omitted | +| `-r`, `--rows` | Base customer row count; other tables scale proportionally (default 100000) | +| `-o`, `--output-dir` | Where to write the generated tree (required) | +| `-v`, `--verbose` | Show generation progress | + ## Documentation - [User Guide](doc/user_guide.md) - Installation, CLI options, and examples diff --git a/c.txt b/c.txt new file mode 100644 index 0000000..224ef20 --- /dev/null +++ b/c.txt @@ -0,0 +1,68 @@ +src/vm/mod.rs +Comment on lines +147 to +151 + let applied = apply_modifications(database, modifications, verbose)?; + modified_tables.extend(applied.modified_tables); + if applied.affected_rows > 0 { + affected_rows = applied.affected_rows; + } +Copilot commented now +Copilot +Copilot +AI +1m ago +Medium +affected_rows is only updated when applied.affected_rows > 0. This means a DML statement that affects 0 rows (e.g., UPDATE ... WHERE matches nothing) will incorrectly keep the previous non-zero count, so .changes/row-change reporting can be wrong. + +Suggested changeset 1 (1) +src/vm/mod.rs +Copilot uses AI. Check for mistakes. +@jgarzik Reply... +tests/tsq/mod.rs +Comment on lines +113 to +117 + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("40"), + "expected 40 customers, got: {stdout}" + ); +Copilot commented now +Copilot +Copilot +AI +1m ago +Low +This test uses stdout.contains("40"), which can produce false positives (e.g. output 140 would still pass). Given the repository guidance to prefer exact-output assertions for new coverage, this should assert the full result more precisely. + +Suggested changeset 1 (1) +tests/tsq/mod.rs +Original file line number Diff line number Diff line change + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("40"), + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!( + lines, + ["COUNT", "40"], + "expected 40 customers, got: {stdout}" + ); + Ok(()) +Copilot generated this review using guidance from repository custom instructions. +@jgarzik Reply... +src/vm/bytecode.rs +Comment on lines 234 to +237 + // Window function operations + WindowAggStep, // P1 = func type, P2 = value reg, P3 = accum reg, P4 = window spec. Step window aggregate. + WindowValue, // P1 = accum reg, P2 = dest reg, P3 = func type. Get current window value. + WindowValue, + WindowFinalize, // Back-fill result column P1 with each partition's final value // P1 = accum reg, P2 = dest reg, P3 = func type. Get current window value. +Copilot commented now +Copilot +Copilot +AI +1m ago +Low +The inline comment for WindowFinalize appears to be copy/pasted from WindowValue and describes the wrong operands, which makes the opcode contract unclear for future changes. + +Copilot uses AI. Check for mistakes. + diff --git a/doc/database.md b/doc/database.md index e8c9ebd..462abc5 100644 --- a/doc/database.md +++ b/doc/database.md @@ -30,7 +30,15 @@ Tables maintain an internal mapping of column names to their indices for efficie - **Rows**: Represented as vectors of values with one element per column - **Columns**: Identified by name, with automatic type inference based on content - **Schema**: Dynamically determined from the file's header row -- **Column Types**: Column types are not explicitly declared but inferred at runtime +- **Column Types**: Not declared; inferred at runtime **per value**, so a single + column may hold integers, floats and text +- **NULL**: An empty field reads as NULL. Comparisons follow SQL three-valued + logic, so a comparison involving NULL is UNKNOWN and its row is filtered out + rather than raising an error. Sorting and grouping treat NULL as equal to + NULL, which SQL comparison must not — the two use separate paths deliberately +- **Coercion**: A comparison mixing a number and a string converts the string + when it parses as a number, and compares textually otherwise. See + [SQL Reference](sql_reference.md) for the exact rules ### Table Lifecycle @@ -59,14 +67,43 @@ The `Table` struct represents an in-memory table with: - Methods for accessing and manipulating rows - Projection capabilities (selecting subsets of columns) -### SQL Executor +### SQL Execution: a bytecode VM -The `SqlExecutor` implements SQL parsing and execution: -- Uses `sqlparser` crate to parse SQL statements -- Converts parsed AST to operations on in-memory tables -- Handles WHERE clause evaluation -- Maintains a set of modified table names (`modified_tables`) to track changes -- Provides `save_modified_tables()` method that only writes back tables that were actually modified +SQL is not interpreted against the tables directly. It is compiled to bytecode +and run on a register virtual machine, following SQLite's architecture: + +1. **Parse** — the `sqlparser` crate produces an AST +2. **Compile** — `vm::compiler` lowers the AST to a bytecode program +3. **Execute** — `vm::engine` runs that program against table cursors + +`SqlExecutor` is a thin wrapper over this pipeline: it forwards to the VM, +tracks which tables were modified, and provides `save_modified_tables()`, which +writes back only tables that actually changed. + +The compiler has one recursive expression compiler, `code_expr` (the analogue +of SQLite's `sqlite3ExprCode`), used from every position an expression can +appear: the SELECT list, WHERE, HAVING, ORDER BY, SET clauses, join conditions, +function arguments and correlated subqueries. Name resolution is carried +separately in a `NameCtx`, which models the visible FROM sources, whether a +column lives behind a cursor or in an already-loaded register, and the +inner/outer scopes a correlated subquery needs. A node supported in one +position is therefore supported in all of them. + +### Statement Execution Model + +A script is parsed once, then each statement is compiled and executed on its +own, with its modifications applied before the next statement compiles. Two +consequences follow: + +- Each statement produces its own result set, with its own column schema. They + are not merged. +- A statement observes the database as the previous one left it, which is what + `CREATE TABLE t; INSERT INTO t ...; SELECT * FROM t` requires. + +A subquery in `FROM` (a derived table) is materialized before its enclosing +statement compiles: the subquery runs, its result is registered under the +alias, and the statement is rewritten to reference that name. The registration +lasts only for that statement. ### File Handlers @@ -152,6 +189,11 @@ The storage backend is selected automatically based on the data source: - **Stdin/pipes**: Uses traditional in-memory storage - **Write operations**: Mmap tables convert to memory on first modification +### Row Identity Across Updates + +UPDATE replaces a row in place rather than deleting and re-appending it, so a +table keeps its row order and `--write` does not reorder the user's file. + ### Write Operation Behavior When a table with mmap storage is modified (INSERT, UPDATE, DELETE): @@ -197,9 +239,15 @@ The database engine supports combining data from multiple tables through SQL-sta ### Join Syntax -- **ON constraints**: Supported for specifying join conditions (`JOIN t2 ON t1.id = t2.id`) -- **USING constraints**: Not supported +- **ON constraints**: Supported, and the condition is a full expression, not + just a single comparison: `ON a.id = b.id AND UPPER(a.name) = 'X'` +- **Table aliases**: Supported (`FROM users u JOIN orders o ON u.id = o.user_id`) +- **USING / NATURAL**: Not supported - **Multi-table joins**: Supports chaining 3+ tables in a single query +- **Aggregates over joins**: Supported for inner joins, which are rewritten to + the equivalent comma join. Deliberately NOT applied to outer joins, since + moving the ON condition into WHERE would discard the NULL-extended rows an + outer join exists to produce ### Technical Implementation @@ -213,9 +261,16 @@ The database engine supports combining data from multiple tables through SQL-sta The database engine has several architectural limitations: - **No Index Structure**: All operations perform full table scans -- **No Transaction Support**: Operations are applied immediately with no rollback capability -- **Schema Flexibility**: Types are inferred rather than enforced +- **No Transaction Support**: A statement's modifications are applied once it + completes; there is no BEGIN/COMMIT and no rollback across statements +- **Schema Flexibility**: Types are inferred per value rather than enforced - **No Constraints System**: Referential integrity not enforced +- **Whole-dataset residency**: A table must fit in memory, or be addressable + via mmap + +Unsupported SQL of note: CTEs (`WITH`), explicit window frames +(`ROWS BETWEEN ...`), `USING`/`NATURAL` joins, and aggregates over outer +joins. --- diff --git a/doc/sql_reference.md b/doc/sql_reference.md index 47ef66f..0c68d6f 100644 --- a/doc/sql_reference.md +++ b/doc/sql_reference.md @@ -125,6 +125,63 @@ SELECT * FROM t1 JOIN t3 ON t2.id = t3.t2_id ``` +## Subqueries + +```sql +-- Scalar subquery +SELECT name FROM employees WHERE salary = (SELECT MAX(salary) FROM employees) + +-- IN / NOT IN +SELECT name FROM users WHERE id IN (SELECT user_id FROM orders) +SELECT name FROM users WHERE id NOT IN (SELECT user_id FROM orders) + +-- EXISTS / NOT EXISTS +SELECT name FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id) + +-- Correlated: the inner query references the outer row +SELECT name FROM employees e + WHERE salary > (SELECT AVG(salary) FROM employees WHERE department = e.department) +``` + +```sql +-- Derived table: a subquery in FROM, which must be given an alias +SELECT name FROM (SELECT name, salary FROM employees WHERE salary > 70000) t +SELECT COUNT(*) FROM (SELECT department FROM employees GROUP BY department) t +``` + +A derived table is materialized before the outer query runs and exists only for +the statement that declares it. Its alias may not shadow a real table. + +## Window Functions + +```sql +SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) FROM employees +SELECT name, RANK() OVER (ORDER BY department) FROM employees +SELECT name, DENSE_RANK() OVER (ORDER BY department) FROM employees +SELECT name, LAG(salary) OVER (ORDER BY salary) FROM employees +SELECT name, LEAD(salary) OVER (ORDER BY salary) FROM employees + +-- Aggregates over a window +SELECT name, SUM(salary) OVER (PARTITION BY department) FROM employees +``` + +Supported functions: `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `LAG`, `LEAD`, and the +aggregates `COUNT`, `SUM`, `AVG`, `MIN`, `MAX`. + +The frame follows the standard: **without** `ORDER BY` in the `OVER` clause the +frame is the whole partition, so every row sees the partition total; **with** +`ORDER BY` the frame grows row by row, giving a running value. + +```sql +-- 210000 on every Engineering row +SELECT name, SUM(salary) OVER (PARTITION BY department) FROM employees + +-- 65000, 135000, 210000 across the Engineering rows +SELECT name, SUM(salary) OVER (PARTITION BY department ORDER BY salary) FROM employees +``` + +Explicit frame clauses (`ROWS BETWEEN ...`) are not supported. + ## Set Operations ```sql @@ -191,6 +248,28 @@ ALTER TABLE name ADD COLUMN col_name TYPE TRUNCATE TABLE name ``` +## NULL handling + +An empty field reads as NULL. + +Comparison follows SQL three-valued logic: any comparison involving NULL is +UNKNOWN, and a row whose `WHERE` evaluates to UNKNOWN is not returned. This +means a NULL row satisfies neither `x > 5` nor `x <= 5`, and `NULL = NULL` is +UNKNOWN rather than true. Use `IS NULL` / `IS NOT NULL` to test for NULL. + +Aggregates skip NULLs: `COUNT(*)` counts rows while `COUNT(col)` counts +non-NULL values. `ORDER BY` sorts NULLs first. + +## Type coercion + +Values are typed per cell as they are read, so one column may hold integers, +floats and text. + +When a comparison mixes a number and a string, the string is converted to a +number if it parses as one, and the two are compared textually otherwise. This +matches arithmetic, so `x + '1'` and `x > '1'` agree — awk-like rather than +strict SQL, which suits untyped delimited input. + ## Writeback Modifications (INSERT, UPDATE, DELETE) remain in-memory unless `--write` flag is specified. diff --git a/doc/user_guide.md b/doc/user_guide.md index 2cd0f94..0ff9633 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -52,6 +52,8 @@ Sqawk is an SQL-based command-line tool for processing delimiter-separated files - No database setup or schema definition required - Automatic type inference and cross-file operations - Powerful SQL dialect including joins, sorting, filtering, and aggregations +- Subqueries (scalar, `IN`, `EXISTS`, correlated) and derived tables +- Set operations (UNION, INTERSECT, EXCEPT) and window functions - Interactive REPL mode for SQL exploration and execution - Safe operation with explicit write-back control @@ -75,7 +77,7 @@ To build from source: 1. Clone the repository: ```sh - git clone https://github.com/username/sqawk.git + git clone https://github.com/jgarzik/sqawk.git cd sqawk ``` @@ -178,13 +180,22 @@ This opens an interactive SQL prompt where you can: | Command | Description | |---------|-------------| -| `.exit` or `.quit` | Exit the REPL | -| `.help` | Display help information about available commands | -| `.save [table]` | Immediately save changes to all modified tables or a specific table | -| `.schema [table]` | Show schema for a specific table or all tables | -| `.tables` | List all available tables | -| `.verbose [on/off]` | Toggle verbose mode on/off | -| `.write [on/off]` | Toggle write mode on/off (default is off) | +| `.cd DIRECTORY` | Change the working directory | +| `.changes [on\|off]` | Show the number of rows changed by each statement | +| `.exit [CODE]` | Exit the REPL, optionally with an exit code | +| `.help` | List the available commands | +| `.load [TABLE=]FILE` | Load `FILE`, optionally under the name `TABLE` | +| `.print STRING...` | Print a literal string | +| `.quit` | Exit the REPL | +| `.save [TABLE]` | Save changes to all modified tables, or just `TABLE` | +| `.schema [TABLE]` | Show the schema for `TABLE`, or for every table | +| `.show [WHAT]` | Show current settings and status | +| `.stats [on\|off]` | Toggle statistics display | +| `.tables [PATTERN]` | List tables, optionally matching a LIKE pattern | +| `.version` | Show source, library and compiler versions | +| `.write [on\|off]` | Toggle writing changes to files (default off) | + +`.help` inside the REPL always lists the authoritative set. **Example REPL Session:** @@ -663,7 +674,10 @@ sqawk> .exit ## Working with Large Files -Sqawk loads all data into memory, which provides excellent performance but requires consideration when working with large files: +Sqawk holds a whole table at once. On-disk files are memory-mapped, so loading +is zero-copy and read-only queries allocate almost nothing; a table is copied to +the heap the first time it is modified. Either way the dataset must fit, which +is worth planning for with large files: **Tips for handling large files:** @@ -687,7 +701,7 @@ Sqawk loads all data into memory, which provides excellent performance but requi 4. **Monitor memory usage**: Particularly when joining large tables, be aware of memory constraints ```sh # Using a more targeted join condition reduces memory requirements - sqawk -s "SELECT a.id, b.name FROM large_a INNER JOIN large_b ON a.id = b.id WHERE a.region = 'West'" large_a.csv large_b.csv + sqawk -s "SELECT a.id, b.name FROM large_a a INNER JOIN large_b b ON a.id = b.id WHERE a.region = 'West'" large_a.csv large_b.csv ``` ## Troubleshooting @@ -704,12 +718,22 @@ Sqawk loads all data into memory, which provides excellent performance but requi - For tab-delimited files, use `-F '\t'` - Ensure consistent delimiters throughout your files -3. **Type conversion errors**: - - Sqawk automatically infers types but sometimes needs hints - - Use explicit casts in SQL when needed: `CAST(value AS INT)` - - Check that numeric columns don't contain non-numeric characters - -4. **CSV parsing errors with malformed rows**: +3. **Unexpected results from comparisons**: + - Types are inferred per value, so one column can hold numbers and text + - Comparing a number against a numeric string coerces and compares + numerically, so `WHERE salary > '60000'` behaves as you would expect + - Use explicit casts when you want to force one interpretation: + `CAST(value AS INT)` + - See the SQL reference for the full coercion and NULL rules + +4. **Rows unexpectedly missing from results**: + - An empty field reads as NULL, and any comparison involving NULL is + UNKNOWN, so such rows satisfy neither `x > 5` nor `x <= 5` + - This follows SQL, and is not an error -- use `IS NULL` / `IS NOT NULL` + to test for NULL explicitly + - `COUNT(*)` counts rows while `COUNT(col)` counts non-NULL values + +5. **CSV parsing errors with malformed rows**: - Error messages about "field count mismatch" indicate rows with inconsistent numbers of fields - Error messages include line numbers to help locate problematic rows - Common causes include: @@ -718,27 +742,27 @@ Sqawk loads all data into memory, which provides excellent performance but requi - Newlines within quoted fields - Use the error recovery options described in the File Format Support section to handle malformed rows -5. **Issues with comment lines**: +6. **Issues with comment lines**: - Comments must start at the beginning of a line with the comment character - Comment characters appearing within data (not at the start of a line) are treated as regular data - If you're seeing unexpected parsing errors, check if comment lines are properly formatted -6. **Memory limitations**: +7. **Memory limitations**: - If processing very large files, filter data early in your queries - Consider processing in batches or using more targeted queries - Select only the columns you need rather than using SELECT * -7. **Changes not saved**: +8. **Changes not saved**: - Remember to use the `--write` flag to save changes - Only modified tables are written back - Check verbose output (`-v`) to confirm which tables were modified -8. **SQL syntax errors**: +9. **SQL syntax errors**: - Try running your query in interactive mode to get immediate feedback - Use the `-v` verbose flag to see the exact SQL being executed - Verify SQL statement syntax, particularly quotes, parentheses, and required clauses -9. **Special characters in files**: +10. **Special characters in files**: - For files with quotes or special characters, Sqawk follows CSV escaping rules - If encountering parsing issues, check for malformed CSV data @@ -766,14 +790,15 @@ For more help, use the verbose mode (`-v`) to see detailed information about pro 2. **Use version control** or backups before modifying important data files 3. **Qualify column names** with table names in multi-table queries 4. **Use verbose mode** (`-v`) when learning or debugging -5. **Chain SQL statements** for complex operations rather than using complex subqueries +5. **Chain SQL statements** with `-s` or `;` when it reads more clearly; each + statement produces its own result and sees the effects of the ones before it 6. **Test on sample data** before processing large files ### Additional Resources - [SQL Language Reference](sql_reference.md) - Complete guide to Sqawk's SQL dialect -- [GitHub Repository](https://github.com/username/sqawk) - Source code and issue tracking -- [Release Notes](https://github.com/username/sqawk/releases) - Latest features and bug fixes +- [GitHub Repository](https://github.com/jgarzik/sqawk) - Source code and issue tracking +- [Release Notes](https://github.com/jgarzik/sqawk/releases) - Latest features and bug fixes --- diff --git a/src/join.rs b/src/join.rs deleted file mode 100644 index 54a122c..0000000 --- a/src/join.rs +++ /dev/null @@ -1,172 +0,0 @@ -//! Join module for the sqawk query engine -//! -//! This module implements table join operations for SQL queries in the sqawk utility. -//! It provides functionality for: -//! -//! - Cross joins (Cartesian product of rows from both tables) -//! - Inner joins with filtering based on specified conditions -//! - A flexible join type system mapping SQL join types to internal representations -//! - Column naming with qualification to prevent ambiguity in join results -//! -//! Currently, the module fully implements cross joins and has a foundation for more -//! complex join types (inner, left, right, full) with placeholder implementations -//! that will be expanded in future versions. - -use sqlparser::ast::{Expr, JoinOperator}; - -use crate::error::{SqawkError, SqawkResult}; -use crate::table::Table; - -/// Join types supported by sqawk -#[derive(Debug, PartialEq, Clone, Copy)] -pub enum JoinType { - /// Inner join - returns rows when there is a match in both tables - Inner, - /// Left join - returns all rows from the left table and matching rows from the right - Left, - /// Right join - returns all rows from the right table and matching rows from the left - Right, - /// Full join - returns rows when there is a match in one of the tables - Full, - /// Cross join - returns the Cartesian product of rows from both tables - Cross, -} - -impl From<&JoinOperator> for JoinType { - fn from(op: &JoinOperator) -> Self { - match op { - JoinOperator::Inner(_) => JoinType::Inner, - JoinOperator::LeftOuter(_) => JoinType::Left, - JoinOperator::RightOuter(_) => JoinType::Right, - JoinOperator::FullOuter(_) => JoinType::Full, - JoinOperator::CrossJoin => JoinType::Cross, - // Default to inner join for other types - _ => JoinType::Inner, - } - } -} - -/// Executor for join operations -/// -/// This struct handles the execution of different types of joins between tables. -pub struct JoinExecutor { - // Placeholder for future join state -} - -impl Default for JoinExecutor { - fn default() -> Self { - Self::new() - } -} - -impl JoinExecutor { - /// Create a new join executor - pub fn new() -> Self { - JoinExecutor {} - } - - /// Execute a join operation between two tables - /// - /// # Arguments - /// * `left` - The left table - /// * `right` - The right table - /// * `join_type` - The type of join to perform - /// * `condition` - Optional join condition (for non-cross joins) - /// - /// # Returns - /// * The resulting joined table - pub fn execute_join( - &mut self, - left: &Table, - right: &Table, - join_type: JoinType, - condition: Option<&Expr>, - ) -> SqawkResult { - match join_type { - JoinType::Cross => self.execute_cross_join(left, right), - JoinType::Inner => { - if let Some(on_expr) = condition { - self.execute_inner_join(left, right, on_expr) - } else { - // Inner join without condition is equivalent to cross join - self.execute_cross_join(left, right) - } - } - _ => Err(SqawkError::UnsupportedSqlFeature(format!( - "Join type {:?} is not implemented yet", - join_type - ))), - } - } - - /// Execute a cross join (Cartesian product) - /// - /// # Arguments - /// * `left` - The left table - /// * `right` - The right table - /// - /// # Returns - /// * The resulting cross-joined table - fn execute_cross_join(&self, left: &Table, right: &Table) -> SqawkResult
{ - // Use the Table's cross_join method - left.cross_join(right) - } - - /// Execute an inner join with an ON condition - /// - /// # Arguments - /// * `left` - The left table - /// * `right` - The right table - /// * `on_expr` - The ON condition expression - /// - /// # Returns - /// * The resulting inner-joined table - fn execute_inner_join( - &self, - left: &Table, - right: &Table, - _on_expr: &Expr, - ) -> SqawkResult
{ - // For now, we'll just do a cross join - // In a real implementation, we would evaluate the ON condition for each row combination - - // Create a new table with combined columns from both tables - let mut result_columns = Vec::with_capacity(left.column_count() + right.column_count()); - - // Add qualified columns from left table - for col in left.columns() { - result_columns.push(format!("left.{}", col)); - } - - // Add qualified columns from right table - for col in right.columns() { - result_columns.push(format!("right.{}", col)); - } - - let mut result = Table::new("joined", result_columns, None); - - // The naive nested loop join approach - for left_row in left.rows().iter() { - for right_row in right.rows().iter() { - // Join the rows - let mut new_row = Vec::with_capacity(left_row.len() + right_row.len()); - - // Add values from left row - for value in left_row { - new_row.push(value.clone()); - } - - // Add values from right row - for value in right_row { - new_row.push(value.clone()); - } - - // Add the combined row to the result table - // TODO: Evaluate ON condition here - let _ = result.add_row(new_row); - } - } - - Ok(result) - } -} diff --git a/src/main.rs b/src/main.rs index f29eaf2..0aaeab5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -140,23 +140,21 @@ fn main() -> Result<()> { .execute(sql) .with_context(|| format!("Failed to execute SQL: {sql}"))?; - // Step 4: Output results to stdout (for SELECT queries) - match result { - // For SELECT queries that return data - Some(table) => { - if config.verbose() { - let row_count = table.row_count(); - println!("Query returned {row_count} rows"); - } - // Print the result table in delimiter-separated format - table.print_to_stdout()?; - } - // For statements that don't return data (UPDATE, DELETE, INSERT) - None => { - if config.verbose() { - println!("Query executed successfully (no results to display)"); - } + // Step 4: Output results to stdout (for SELECT queries). + // + // One result set per statement, each with its own header. They used to + // share a single buffer and schema, so `SELECT a; SELECT b` printed + // every row under the last statement's header. + for table in &result { + if config.verbose() { + let row_count = table.row_count(); + println!("Query returned {row_count} rows"); } + // Print the result table in delimiter-separated format + table.print_to_stdout()?; + } + if result.is_empty() && config.verbose() { + println!("Query executed successfully (no results to display)"); } } diff --git a/src/repl.rs b/src/repl.rs index 26b22d6..0eb31db 100644 --- a/src/repl.rs +++ b/src/repl.rs @@ -452,10 +452,11 @@ impl<'a> Repl<'a> { } else if self.show_changes { // For non-SELECT statements that don't return rows (INSERT, UPDATE, DELETE) // Try to display the number of affected rows if show_changes is enabled + // Reported even when zero. With `.changes on` a statement that + // matched nothing otherwise produced no output at all, leaving it + // ambiguous whether it had run. if let Ok(affected_rows) = self.executor.get_affected_row_count() { - if affected_rows > 0 { - println!("{} rows affected", affected_rows); - } + println!("{} rows affected", affected_rows); } } @@ -692,8 +693,12 @@ impl<'a> Repl<'a> { } /// Show version information + /// + /// Read from the manifest rather than written out, so it cannot drift. + /// This reported 0.1.1 for a long time while the crate was at 0.8.0. fn show_version(&self) -> Result<()> { - println!("Sqawk version 0.1.1"); + println!("Sqawk version {}", env!("CARGO_PKG_VERSION")); + println!("sqlparser {}", crate::vm::SQLPARSER_VERSION); Ok(()) } diff --git a/src/sql_executor.rs b/src/sql_executor.rs index e78542a..0dc8c69 100644 --- a/src/sql_executor.rs +++ b/src/sql_executor.rs @@ -69,8 +69,10 @@ impl<'a> SqlExecutor<'a> { /// * `sql` - SQL statement to execute /// /// # Returns - /// * `SqawkResult>` - Result of the operation, possibly containing a table - pub fn execute(&mut self, sql: &str) -> SqawkResult> { + /// * One result table per statement that produced rows, in order. A + /// multi-statement script yields one result set per statement; they are + /// deliberately NOT merged, since each has its own column schema. + pub fn execute(&mut self, sql: &str) -> SqawkResult> { if self.config.verbose() { println!("Executing SQL: {}", sql); } @@ -86,18 +88,15 @@ impl<'a> SqlExecutor<'a> { // Track affected rows from the last statement self.affected_rows = result.affected_rows; - // Set delimiter on result table to match config (for consistent output format) - let table = match result.table { - Some(mut t) => { - if let Some(delim) = self.config.field_separator() { - t.set_delimiter(delim); - } - Some(t) + // Set delimiter on each result table to match config + let mut tables = result.tables; + if let Some(delim) = self.config.field_separator() { + for t in &mut tables { + t.set_delimiter(delim.clone()); } - None => None, - }; + } - Ok(table) + Ok(tables) } /// Save all modified tables back to their source files @@ -198,7 +197,9 @@ impl<'a> SqlExecutor<'a> { pub fn execute_sql(&mut self, sql: &str) -> Result> { let result = self.execute(sql)?; - Ok(result.map(|table| ResultSet { + // The REPL shows one result set at a time; a multi-statement line + // reports the last statement's. + Ok(result.last().map(|table| ResultSet { columns: table.columns().to_vec(), rows: table.rows_as_strings(), })) diff --git a/src/table.rs b/src/table.rs index 880b733..64a2def 100644 --- a/src/table.rs +++ b/src/table.rs @@ -854,6 +854,14 @@ impl Table { /// Returns the name of the table as a string slice. /// This is useful for operations that need to access the table's name /// such as logging, error messages, or generating SQL output. + /// Rename the table. + /// + /// Used when a derived table is materialized and registered under its + /// alias, so the table reports the name the outer query refers to. + pub fn set_name(&mut self, name: String) { + self.name = name; + } + pub fn name(&self) -> &str { &self.name } @@ -1022,6 +1030,26 @@ impl Table { self.modified = true; } + /// Replace a single row in place, preserving its position. + /// + /// Used by UPDATE. Expressing an update as delete-plus-insert appends the + /// new row, which reorders the table -- and with --write, the user's file. + pub fn replace_row(&mut self, row_index: usize, row: Row) -> SqawkResult<()> { + self.storage.ensure_mutable(); + let rows = self.storage.rows_mut().ok_or_else(|| { + SqawkError::InvalidSqlQuery("Table storage is not mutable".to_string()) + })?; + if row_index >= rows.len() { + return Err(SqawkError::InvalidSqlQuery(format!( + "Row index {} out of range for table '{}'", + row_index, self.name + ))); + } + rows[row_index] = row; + self.modified = true; + Ok(()) + } + /// Add a column to the table with a specified data type #[cfg(test)] pub fn add_column(&mut self, name: String, data_type_str: String) { diff --git a/src/vm/ast_compat.rs b/src/vm/ast_compat.rs new file mode 100644 index 0000000..eaa569c --- /dev/null +++ b/src/vm/ast_compat.rs @@ -0,0 +1,185 @@ +//! Small accessors that normalize sqlparser AST shapes. +//! +//! sqlparser treats every AST change as breaking, and a few of its types have +//! grown structure that the compiler does not care about. Rather than spread +//! that structure across ~150 call sites, the compiler goes through the +//! accessors here. +//! +//! This is deliberately thin: it adapts *shape*, never *meaning*. Anything that +//! requires a semantic decision (which join kinds are supported, how DISTINCT +//! affects an aggregate) belongs in the compiler proper. + +use sqlparser::ast::{ + Assignment, AssignmentTarget, CreateTableOptions, DuplicateTreatment, Expr, FromTable, + Function, FunctionArg, FunctionArguments, GroupByExpr, LimitClause, ObjectName, OrderBy, + OrderByExpr, OrderByKind, Query, SqlOption, TableObject, TableWithJoins, +}; + +use crate::error::{SqawkError, SqawkResult}; + +/// The positional argument list of a function call. +/// +/// `FunctionArguments` distinguishes three cases that the compiler mostly does +/// not need to: no parentheses at all (`CURRENT_TIMESTAMP`), a bare subquery +/// argument, and an ordinary parenthesized list. Only the last carries +/// positional arguments; the other two yield an empty slice. +pub(crate) fn func_args(func: &Function) -> &[FunctionArg] { + match &func.args { + FunctionArguments::List(list) => &list.args, + FunctionArguments::None | FunctionArguments::Subquery(_) => &[], + } +} + +/// Whether a function call was written with `DISTINCT`, as in +/// `COUNT(DISTINCT department)`. +/// +/// Older sqlparser exposed this as a plain `Function::distinct` bool; it now +/// lives in the argument list as a `DuplicateTreatment`. `ALL` is the default +/// and means the same thing as omitting it. +pub(crate) fn func_is_distinct(func: &Function) -> bool { + match &func.args { + FunctionArguments::List(list) => { + matches!(list.duplicate_treatment, Some(DuplicateTreatment::Distinct)) + } + FunctionArguments::None | FunctionArguments::Subquery(_) => false, + } +} + +/// The GROUP BY key expressions. +/// +/// `GROUP BY ALL` has no explicit key list; it yields an empty slice here and +/// is rejected by the caller that cares. +pub(crate) fn group_by_exprs(group_by: &GroupByExpr) -> &[sqlparser::ast::Expr] { + match group_by { + GroupByExpr::Expressions(exprs, _) => exprs, + GroupByExpr::All(_) => &[], + } +} + +/// Whether this GROUP BY is the unsupported `GROUP BY ALL` form. +pub(crate) fn is_group_by_all(group_by: &GroupByExpr) -> bool { + matches!(group_by, GroupByExpr::All(_)) +} + +/// The ORDER BY key expressions of a query. +/// +/// `Query::order_by` is now an `Option` whose kind distinguishes an +/// explicit key list from `ORDER BY ALL`. Absent ORDER BY and `ORDER BY ALL` +/// both yield an empty slice; the caller that cares rejects the latter. +pub(crate) fn query_order_by(query: &Query) -> &[OrderByExpr] { + match &query.order_by { + Some(OrderBy { + kind: OrderByKind::Expressions(exprs), + .. + }) => exprs, + _ => &[], + } +} + +/// Whether this query uses the unsupported `ORDER BY ALL` form. +pub(crate) fn is_order_by_all(query: &Query) -> bool { + matches!( + &query.order_by, + Some(OrderBy { + kind: OrderByKind::All(_), + .. + }) + ) +} + +/// Sort direction for one ORDER BY key: `true` ascending, `false` descending. +/// +/// Ascending is the SQL default when the direction is omitted. `asc` moved +/// from `OrderByExpr` into a nested `OrderByOptions`. +pub(crate) fn order_by_is_asc(expr: &OrderByExpr) -> bool { + expr.options.asc.unwrap_or(true) +} + +/// The LIMIT expression, if any. +/// +/// `Query::limit` and `Query::offset` were merged into a single +/// `limit_clause` that also models MySQL's reversed `LIMIT , `. +pub(crate) fn query_limit(query: &Query) -> Option<&Expr> { + match &query.limit_clause { + Some(LimitClause::LimitOffset { limit, .. }) => limit.as_ref(), + Some(LimitClause::OffsetCommaLimit { limit, .. }) => Some(limit), + None => None, + } +} + +/// The OFFSET expression, if any. See [`query_limit`]. +pub(crate) fn query_offset(query: &Query) -> Option<&Expr> { + match &query.limit_clause { + Some(LimitClause::LimitOffset { offset, .. }) => offset.as_ref().map(|o| &o.value), + Some(LimitClause::OffsetCommaLimit { offset, .. }) => Some(offset), + None => None, + } +} + +/// The target table of an INSERT. +/// +/// `Insert::table` is now a `TableObject`, which can also be a table function. +pub(crate) fn insert_target(table: &TableObject) -> SqawkResult<&ObjectName> { + match table { + TableObject::TableName(name) => Ok(name), + TableObject::TableFunction(_) | TableObject::TableQuery(_) => Err( + SqawkError::UnsupportedSqlFeature("INSERT target must be a plain table name".into()), + ), + } +} + +/// The source tables of a DELETE. +/// +/// `Delete::from` distinguishes `DELETE FROM t` from `DELETE t`; sqawk treats +/// them the same. +pub(crate) fn delete_from(from: &FromTable) -> &[TableWithJoins] { + match from { + FromTable::WithFromKeyword(t) | FromTable::WithoutKeyword(t) => t, + } +} + +/// The assigned column of an UPDATE `SET` clause. +/// +/// `Assignment::id` (a `Vec`) became `target: AssignmentTarget`. +pub(crate) fn assignment_column(assignment: &Assignment) -> SqawkResult { + match &assignment.target { + AssignmentTarget::ColumnName(name) => Ok(object_name_last(name)), + AssignmentTarget::Tuple(_) => Err(SqawkError::UnsupportedSqlFeature( + "Tuple assignment in UPDATE is not supported".into(), + )), + } +} + +/// The `WITH (...)` options attached to a CREATE TABLE. +/// +/// Previously a plain `Vec`; the dialect-specific spellings are now +/// distinguished by `CreateTableOptions`, which sqawk does not care about. +pub(crate) fn create_table_options(options: &CreateTableOptions) -> &[SqlOption] { + match options { + CreateTableOptions::With(o) + | CreateTableOptions::Options(o) + | CreateTableOptions::Plain(o) + | CreateTableOptions::TableProperties(o) => o, + _ => &[], + } +} + +/// A CREATE TABLE option as a `(name, value)` pair, for the options sqawk +/// understands (`delimiter`, `header`, ...). Non key-value forms yield `None`. +pub(crate) fn sql_option_key_value(option: &SqlOption) -> Option<(String, String)> { + match option { + SqlOption::KeyValue { key, value } => Some((key.value.clone(), value.to_string())), + _ => None, + } +} + +/// The final segment of an `ObjectName` -- the bare table or function name. +pub(crate) fn object_name_last(name: &ObjectName) -> String { + match name.0.last() { + Some(part) => match part.as_ident() { + Some(ident) => ident.value.clone(), + None => part.to_string(), + }, + None => String::new(), + } +} diff --git a/src/vm/bytecode.rs b/src/vm/bytecode.rs index e0cc97b..e7f9dbe 100644 --- a/src/vm/bytecode.rs +++ b/src/vm/bytecode.rs @@ -24,6 +24,17 @@ pub const AGG_MIN: i64 = 3; /// MAX aggregate function type pub const AGG_MAX: i64 = 4; +/// Flag bit OR-ed into an AggStep/AggFinal function type to request DISTINCT. +/// +/// Carried as a flag rather than as five extra constants so that DISTINCT +/// composes with every aggregate: `COUNT(DISTINCT x)`, `SUM(DISTINCT x)` and +/// the rest all use the same mechanism. Mask with `AGG_TYPE_MASK` to recover +/// the base function. +pub const AGG_DISTINCT: i64 = 0x100; + +/// Mask selecting the base aggregate function out of a function-type word. +pub const AGG_TYPE_MASK: i64 = 0xFF; + /// A column in a result schema #[derive(Debug, Clone)] pub struct ResultColumn { @@ -90,6 +101,7 @@ pub enum OpCode { Column, // Read column value into register InsertRow, // Insert row from registers P2..P2+P3 into cursor P1 DeleteRow, // Delete current row at cursor P1 + UpdateRow, // Replace current row at cursor P1 with registers P2..P2+P3 IN PLACE // Data manipulation Integer, // Load integer constant @@ -123,6 +135,29 @@ pub enum OpCode { // Null operations IsNull, // Set P2 to 1 if P1 is NULL, 0 otherwise + // Logical negation + // + // Distinct from the comparison opcodes because it must propagate NULL: + // NOT NULL is NULL (UNKNOWN), not true. Replaces the several hand-rolled + // inversions used for NOT LIKE / NOT IN / IS NOT NULL and makes a bare + // `NOT ` expressible at all. + Not, // P2 = NULL if P1 is NULL, else 1 if P1 is falsy, else 0 + + // Vector comparison, modelled on SQLite's OP_Compare / OP_Jump. + // + // Compare sets an internal flag from a pairwise comparison of two register + // vectors; Jump then branches three ways on that flag. Together they make + // multi-column key comparison two instructions regardless of key width, + // which is what GROUP BY needs to detect a group change across ALL key + // columns rather than just the first. + // + // These use INTERNAL ordering, not SQL comparison: NULL sorts equal to + // NULL, so a NULL group key groups with other NULL keys. SQL `=` must not + // behave that way, which is exactly why this is a separate opcode rather + // than a reuse of Eq. + Compare, // Compare P3 registers starting at P1 against P3 starting at P2 + Jump, // Branch on the last Compare: P1 if Less, P2 if Equal, P3 if Greater + // Conditional jumps IfZ, // Jump to P2 if register P1 contains 0 IfPos, // Jump to P2 if register P1 is positive (> 0) @@ -132,7 +167,8 @@ pub enum OpCode { Noop, // No operation // Set operations - Distinct, // Remove duplicate rows from results + SortResults, // Sort accumulated results. P4 = "col:asc,col:desc,..." over RESULT columns. + Distinct, // Remove duplicate rows from results Limit, // Limit results to P1 rows. P2 = offset (rows to skip, already applied). Post-processing opcode. Intersect, // Keep only rows that exist in both left and right result sets Except, // Keep only rows from left that don't exist in right result set @@ -198,6 +234,8 @@ pub enum OpCode { // Window function operations WindowAggStep, // P1 = func type, P2 = value reg, P3 = accum reg, P4 = window spec. Step window aggregate. WindowValue, // P1 = accum reg, P2 = dest reg, P3 = func type. Get current window value. + WindowFinalize, // P1 = result column index. Give every row of a partition that partition's + // final window value, which for an unordered frame is the partition total. } /// A SQL VM instruction with opcode and parameters @@ -276,6 +314,13 @@ pub struct Program { pub instructions: Vec, /// Schema for the result set (built at compile time) pub result_schema: ResultSchema, + /// Number of registers the compiler allocated for this program. + /// + /// The engine sizes its register file from this. It must not be inferred + /// from the instruction stream: p1/p2/p3 hold literals as often as they + /// hold register numbers (`Integer 999999 -> r` being the obvious case), + /// so scanning for the maximum both over- and under-estimates. + pub register_count: i64, } impl Default for Program { @@ -289,6 +334,7 @@ impl Program { Self { instructions: Vec::with_capacity(DEFAULT_INSTRUCTION_CAPACITY), result_schema: ResultSchema::new(), + register_count: 0, } } diff --git a/src/vm/compiler.rs b/src/vm/compiler.rs index 3db2cc1..3490705 100644 --- a/src/vm/compiler.rs +++ b/src/vm/compiler.rs @@ -4,10 +4,14 @@ //! into bytecode instructions that can be executed by the SQL VM. //! It implements a visitor pattern to walk the AST generated by sqlparser. +use super::ast_compat::{ + create_table_options, delete_from, func_args, group_by_exprs, insert_target, is_group_by_all, + is_order_by_all, object_name_last, order_by_is_asc, query_limit, query_offset, query_order_by, +}; use sqlparser::ast::{ - BinaryOperator, DataType as SqlDataType, Expr, Function, FunctionArg, FunctionArgExpr, - ObjectName, Query, Select, SelectItem, SetExpr, SetOperator, SetQuantifier, Statement, - TableWithJoins, UnaryOperator, Value, + BinaryOperator, CaseWhen, DataType as SqlDataType, Expr, Function, FunctionArg, + FunctionArgExpr, Ident, JoinConstraint, ObjectName, Query, Select, SelectItem, SetExpr, + SetOperator, SetQuantifier, Statement, TableWithJoins, UnaryOperator, Value, ValueWithSpan, }; use sqlparser::dialect::HiveDialect; use sqlparser::parser::Parser; @@ -28,10 +32,237 @@ pub(crate) type MultiTableProjection = (Vec>, Vec<(usize, usize)>, Re /// Outer column reference found in a correlated subquery (Phase 4B) #[derive(Debug, Clone)] struct OuterColumnRef { - /// The table qualifier (alias or table name) + /// The table qualifier (alias or table name) the outer query used. + /// + /// Only the qualifier is needed: which column it names is resolved by + /// NameCtx against the outer table, so recording it here would be a second + /// copy of that lookup. qualifier: String, - /// The column name - column: String, +} + +/// A symbolic jump target. +/// +/// Jump destinations used to be computed arithmetically, as offsets from +/// `program.len()` counting instructions that had not been emitted yet -- e.g. +/// `let skip_offset = if limit_reg.is_some() { 3 } else { 2 }`. That couples +/// every jump to the exact shape of the code after it, so inserting, +/// reordering, or conditionally omitting an instruction silently retargets the +/// jump. The failure mode is a wrong answer, not a crash. +/// +/// A `Label` is created before it is known where it points, used as a jump +/// destination any number of times, and bound to an address later with +/// [`SqlCompiler::resolve`]. Unresolved labels are a compile-time bug and are +/// caught by an assertion when the program is finished. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Label(usize); + +/// One FROM source visible to expression compilation. +pub(crate) struct Src<'t> { + /// VM cursor id for this source. + pub cursor: i64, + /// Base register holding this source's columns, when the loop has already + /// materialized them. + /// + /// Multi-table joins load every column into registers before evaluating + /// join conditions, so a column reference there is a register offset + /// rather than a `Column` read from a cursor. Modelling both here means + /// one expression compiler serves both loop shapes. + pub reg_base: Option, + /// The table behind the cursor. + pub table: &'t Table, + /// How the query refers to it: the alias if there is one, else the table + /// name. Stored ASCII-lowercased, since SQL identifiers are matched + /// case-insensitively. + pub refname: String, +} + +/// A resolved column: where its value lives at run time. +#[derive(Clone, Copy, Debug)] +pub(crate) enum ColumnRef { + /// Read column `col` from cursor `cursor`. + Cursor { cursor: i64, col: usize }, + /// The value is already in this register. + Register(i64), +} + +impl ColumnRef { + fn new(src: &Src<'_>, col: usize) -> Self { + match src.reg_base { + Some(base) => ColumnRef::Register(base + col as i64), + None => ColumnRef::Cursor { + cursor: src.cursor, + col, + }, + } + } +} + +/// Name resolution scope for [`SqlCompiler::code_expr`]. +/// +/// The reason two expression compilers drifted apart is that both threaded a +/// bare `(table, cursor_idx)` pair, a shape that cannot express a join, an +/// alias, or a post-aggregation scope. Everything a name can resolve against +/// lives here instead, so one compiler serves the SELECT list, WHERE, HAVING, +/// ORDER BY and SET clauses. +pub(crate) struct NameCtx<'t> { + /// FROM sources in cursor order. Empty for `SELECT 1` (no FROM). + pub srcs: Vec>, + + /// Enclosing query's sources, for a correlated subquery. + /// + /// SQL resolves a name in the innermost scope first and only then looks + /// outward, so these are consulted only when `srcs` has no match. Keeping + /// them in a separate list rather than appending to `srcs` preserves that + /// precedence, and keeps the ambiguity check meaningful: two tables in one + /// FROM are ambiguous, but an inner name shadowing an outer one is not. + pub outer: Vec>, + + /// Aggregate results available to this expression, keyed by the rendered + /// call text (`"sum(salary)"`). + /// + /// After aggregation an aggregate is a value like any other, so an + /// expression may contain one -- `SUM(salary) + 1`. Binding them here lets + /// the ordinary expression compiler handle the surrounding arithmetic + /// instead of needing a separate post-aggregation compiler. + pub agg_bindings: Vec<(String, i64)>, +} + +impl<'t> NameCtx<'t> { + /// The common single-table scope, equivalent to the old + /// `(table, cursor_idx)` pair. + pub(crate) fn single(table: &'t Table, cursor: i64) -> Self { + NameCtx { + srcs: vec![Src { + cursor, + reg_base: None, + table, + refname: table.name().to_ascii_lowercase(), + }], + outer: Vec::new(), + agg_bindings: Vec::new(), + } + } + + /// A scope over several FROM sources, for joins. + pub(crate) fn multi(srcs: Vec>) -> Self { + NameCtx { + srcs, + outer: Vec::new(), + agg_bindings: Vec::new(), + } + } + + /// A scope with no FROM sources, for constant expressions. + #[allow(dead_code)] // used by constant-expression contexts + pub(crate) fn empty() -> Self { + NameCtx { + srcs: Vec::new(), + outer: Vec::new(), + agg_bindings: Vec::new(), + } + } + + /// Normalized key for an aggregate call, used to bind and to look up. + pub(crate) fn agg_key(expr: &Expr) -> String { + expr.to_string().to_ascii_lowercase() + } + + /// Register holding the result of an already-computed aggregate. + fn lookup_agg(&self, expr: &Expr) -> Option { + let key = Self::agg_key(expr); + self.agg_bindings + .iter() + .find(|(k, _)| *k == key) + .map(|(_, r)| *r) + } + + /// Resolve an unqualified column name across every source. + /// + /// Case-insensitive: `SELECT NAME` already worked while + /// `WHERE NAME = 'x'` did not, because the projection path lowercased and + /// `find_column_index` compared exactly. + /// + /// Returns `Err` if no source has the column, or if more than one does -- + /// silently picking the first would make the result depend on FROM order. + fn resolve_unqualified(&self, name: &str) -> SqawkResult { + let want = name.to_ascii_lowercase(); + let mut found: Option = None; + for src in &self.srcs { + for (i, col) in src.table.column_metadata().iter().enumerate() { + if col.name.to_ascii_lowercase() == want { + if found.is_some() { + return Err(SqawkError::InvalidSqlQuery(format!( + "Column '{}' is ambiguous across the tables in FROM", + name + ))); + } + found = Some(ColumnRef::new(src, i)); + break; + } + } + } + if let Some(col) = found { + return Ok(col); + } + // Not in this scope: look outward, nearest first. + for src in &self.outer { + for (i, col) in src.table.column_metadata().iter().enumerate() { + if col.name.to_ascii_lowercase() == want { + return Ok(ColumnRef::new(src, i)); + } + } + } + Err(SqawkError::ColumnNotFound(name.to_string())) + } + + /// Resolve a `table.column` (or `alias.column`) reference. + fn resolve_qualified(&self, qualifier: &str, name: &str) -> SqawkResult { + let qual = qualifier.to_ascii_lowercase(); + let want = name.to_ascii_lowercase(); + for src in &self.srcs { + if src.refname != qual { + continue; + } + for (i, col) in src.table.column_metadata().iter().enumerate() { + if col.name.to_ascii_lowercase() == want { + return Ok(ColumnRef::new(src, i)); + } + } + return Err(SqawkError::ColumnNotFound(format!( + "{}.{}", + qualifier, name + ))); + } + // Qualifier not in this scope: try the enclosing one, which is how a + // correlated subquery refers to its outer row. + for src in &self.outer { + if src.refname != qual { + continue; + } + for (i, col) in src.table.column_metadata().iter().enumerate() { + if col.name.to_ascii_lowercase() == want { + return Ok(ColumnRef::new(src, i)); + } + } + } + // Unknown qualifier entirely: fall back to an unqualified match so a + // single-table query can still say `employees.age` when the cursor was + // opened under a different refname. + self.resolve_unqualified(name) + } +} + +/// Classify a join operator for the inner-join rewrite. +/// +/// `Some(Some(c))` = inner join with constraint `c`; `Some(None)` = cross join; +/// `None` = not rewritable (outer joins and the exotic forms). +fn classify_join_for_rewrite(op: &sqlparser::ast::JoinOperator) -> Option> { + use sqlparser::ast::JoinOperator as J; + match op { + J::Join(c) | J::Inner(c) => Some(Some(c)), + J::CrossJoin(_) => Some(None), + _ => None, + } } /// SQL statement compiler that generates bytecode for the VM engine @@ -53,6 +284,17 @@ pub struct SqlCompiler<'a> { /// Current outer table alias for correlated subquery detection pub(crate) current_outer_alias: Option, + + /// Resolved address of each label, indexed by label id. `None` until + /// [`SqlCompiler::resolve`] binds it. + label_addrs: Vec>, + + /// Jumps awaiting resolution: (instruction address, target label). + pending_jumps: Vec<(usize, Label)>, + + /// Three-way `Jump`s awaiting resolution: + /// (instruction address, less, equal, greater). + pending_jumps_three: Vec<(usize, Label, Label, Label)>, } impl<'a> SqlCompiler<'a> { @@ -65,7 +307,102 @@ impl<'a> SqlCompiler<'a> { register_counter: 0, verbose, current_outer_alias: None, + label_addrs: Vec::new(), + pending_jumps: Vec::new(), + pending_jumps_three: Vec::new(), + } + } + + /// Create a new, unresolved jump target. + pub(crate) fn label(&mut self) -> Label { + self.label_addrs.push(None); + Label(self.label_addrs.len() - 1) + } + + /// Bind `label` to the next instruction to be emitted. + /// + /// Panics if the label was already resolved -- a label denotes exactly one + /// address, and rebinding one silently redirects every jump to it. + pub(crate) fn resolve(&mut self, label: Label) { + let addr = self.program.len(); + debug_assert!( + self.label_addrs[label.0].is_none(), + "label {:?} resolved twice (already at {:?}, now {})", + label, + self.label_addrs[label.0], + addr + ); + self.label_addrs[label.0] = Some(addr); + } + + /// Emit a jump instruction whose destination (p2) is `target`. + /// + /// The destination is patched in by [`SqlCompiler::finish_labels`], so the + /// target may be resolved before or after this call. + pub(crate) fn emit_jump_to( + &mut self, + opcode: OpCode, + p1: i64, + target: Label, + p3: i64, + p4: Option, + comment: &str, + ) { + let addr = self.program.len(); + // p2 is a placeholder; finish_labels overwrites it. + self.emit(opcode, p1, -1, p3, p4, comment); + self.pending_jumps.push((addr, target)); + } + + /// Emit a `Jump`: a three-way branch on the most recent `Compare`. + /// + /// All three destinations are labels, patched by `finish_labels`. + pub(crate) fn emit_jump_to_three( + &mut self, + less: Label, + equal: Label, + greater: Label, + comment: &str, + ) { + let addr = self.program.len(); + self.emit(OpCode::Jump, -1, -1, -1, None, comment); + self.pending_jumps_three.push((addr, less, equal, greater)); + } + + /// Patch every pending jump with its label's address. + /// + /// Called once when a program is finished. An unresolved label means a + /// code path forgot to mark its destination, which would leave a jump + /// pointing at the placeholder -- so it fails loudly rather than + /// generating a program that runs and returns nonsense. + pub(crate) fn finish_labels(&mut self) -> SqawkResult<()> { + for (addr, label) in std::mem::take(&mut self.pending_jumps) { + let target = self.label_addrs[label.0].ok_or_else(|| { + SqawkError::InvalidSqlQuery(format!( + "internal error: jump at {} targets unresolved label {:?}", + addr, label + )) + })?; + self.patch_jump(addr, target); + } + for (addr, less, equal, greater) in std::mem::take(&mut self.pending_jumps_three) { + let resolve = |l: Label| -> SqawkResult { + self.label_addrs[l.0].map(|a| a as i64).ok_or_else(|| { + SqawkError::InvalidSqlQuery(format!( + "internal error: Jump at {} targets unresolved label {:?}", + addr, l + )) + }) + }; + let (p1, p2, p3) = (resolve(less)?, resolve(equal)?, resolve(greater)?); + if let Some(inst) = self.program.instructions.get_mut(addr) { + inst.p1 = p1; + inst.p2 = p2; + inst.p3 = p3; + } } + self.label_addrs.clear(); + Ok(()) } /// Allocate a new register and return its index @@ -149,95 +486,6 @@ impl<'a> SqlCompiler<'a> { } } - /// Emit AND logic: result = left_reg && right_reg - /// Returns the register containing the result (1 if both true, 0 otherwise) - pub(crate) fn emit_and(&mut self, left_reg: i64, right_reg: i64) -> i64 { - let result_reg = self.allocate_register(); - - // Start with 0 (false) - self.emit( - OpCode::Integer, - 0, - result_reg, - 0, - None, - &format!("r[{}] = 0 (AND init)", result_reg), - ); - - // If left is 0, skip to end (result stays 0) - let skip_addr1 = self.program.len(); - self.emit(OpCode::IfZ, left_reg, 0, 0, None, "Skip if left is false"); - - // If right is 0, skip to end (result stays 0) - let skip_addr2 = self.program.len(); - self.emit(OpCode::IfZ, right_reg, 0, 0, None, "Skip if right is false"); - - // Both true, set result to 1 - self.emit( - OpCode::Integer, - 1, - result_reg, - 0, - None, - &format!("r[{}] = 1 (AND true)", result_reg), - ); - - // Patch skip addresses to end - let end_addr = self.program.len(); - self.patch_jump(skip_addr1, end_addr); - self.patch_jump(skip_addr2, end_addr); - - result_reg - } - - /// Emit OR logic: result = left_reg || right_reg - /// Returns the register containing the result (1 if either true, 0 otherwise) - pub(crate) fn emit_or(&mut self, left_reg: i64, right_reg: i64) -> i64 { - let result_reg = self.allocate_register(); - - // Start with 1 (optimistic - assume true) - self.emit( - OpCode::Integer, - 1, - result_reg, - 0, - None, - &format!("r[{}] = 1 (OR init)", result_reg), - ); - - // If left is non-zero, skip to end (result stays 1) - let skip_addr1 = self.program.len(); - self.emit(OpCode::IfPos, left_reg, 0, 0, None, "Skip if left is true"); - - // If right is non-zero, skip to end (result stays 1) - let skip_addr2 = self.program.len(); - self.emit( - OpCode::IfPos, - right_reg, - 0, - 0, - None, - "Skip if right is true", - ); - - // Both false, set result to 0 - self.emit( - OpCode::Integer, - 0, - result_reg, - 0, - None, - &format!("r[{}] = 0 (OR false)", result_reg), - ); - - // Patch skip addresses to end - let end_addr = self.program.len(); - self.patch_jump(skip_addr1, end_addr); - self.patch_jump(skip_addr2, end_addr); - - result_reg - } - /// Emit a comparison operation and return the result register pub(crate) fn emit_comparison( &mut self, @@ -346,11 +594,52 @@ impl<'a> SqlCompiler<'a> { // TODO: Add direct query compilation support /// Compile an SQL string into bytecode + /// Compile a single already-parsed statement into its own program. + /// + /// Each statement gets its own program so it can be executed, and its + /// modifications applied, before the next one is compiled -- which is what + /// lets a later statement see an earlier one's effects. + pub fn compile_statement_program(&mut self, statement: &Statement) -> SqawkResult { + self.program = Program::new(); + self.reset_registers(); + self.table_map.clear(); + self.label_addrs.clear(); + self.pending_jumps.clear(); + self.pending_jumps_three.clear(); + + let init_addr = self.program.len(); + self.emit(OpCode::Init, 0, 0, 0, None, "Start address filled in later"); + + self.compile_statement(statement)?; + + self.emit(OpCode::Halt, 0, 0, 0, None, "End execution"); + + let start = init_addr + 1; + if let Some(instruction) = self.program.instructions.get_mut(init_addr) { + instruction.p2 = start as i64; + instruction.comment = Some(format!("Start at {}", start).into()); + } + + self.finish_labels()?; + self.program.register_count = self.register_counter; + Ok(self.program.clone()) + } + + /// Parse and compile `sql` into a single program. + /// + /// Convenience wrapper used by the compiler unit tests, which examine + /// generated bytecode for one statement. Execution goes through + /// `compile_statement_program` per statement instead, so that each + /// statement can observe the previous one's effects. + #[allow(dead_code)] pub fn compile(&mut self, sql: &str) -> SqawkResult { // Reset state for new compilation self.program = Program::new(); self.reset_registers(); self.table_map.clear(); + self.label_addrs.clear(); + self.pending_jumps.clear(); + self.pending_jumps_three.clear(); // Parse the SQL statement let dialect = HiveDialect {}; @@ -390,6 +679,13 @@ impl<'a> SqlCompiler<'a> { instruction.comment = Some(format!("Start at {}", transaction_addr).into()); } + // Patch every symbolic jump now that all addresses are known. This + // errors rather than emitting a program with a placeholder target. + self.finish_labels()?; + + // Hand the engine the real register high-water mark. + self.program.register_count = self.register_counter; + Ok(self.program.clone()) } @@ -399,35 +695,39 @@ impl<'a> SqlCompiler<'a> { fn compile_statement(&mut self, statement: &Statement) -> SqawkResult<()> { match statement { Statement::Query(query) => self.compile_query(query), - Statement::Insert { - table_name, - columns, - source, - .. - } => self.compile_insert(table_name, columns, source), - Statement::Delete { - from, selection, .. - } => self.compile_delete(from, selection.as_ref()), - Statement::Update { - table, - assignments, - selection, - .. - } => self.compile_update(table, assignments, selection.as_ref()), - Statement::CreateTable { - name, - columns, - hive_formats, - location, - with_options, - query, - .. - } => { - if let Some(q) = query { + Statement::Insert(insert) => { + let source = insert.source.as_ref().ok_or_else(|| { + SqawkError::UnsupportedSqlFeature("INSERT without a source".into()) + })?; + // `Insert::columns` became Vec; sqawk only accepts + // bare column names, so flatten each to its last segment. + let columns: Vec = insert + .columns + .iter() + .map(|c| Ident::new(object_name_last(c))) + .collect(); + self.compile_insert(insert_target(&insert.table)?, &columns, source) + } + Statement::Delete(delete) => { + self.compile_delete(delete_from(&delete.from), delete.selection.as_ref()) + } + Statement::Update(update) => self.compile_update( + &update.table, + &update.assignments, + update.selection.as_ref(), + ), + Statement::CreateTable(create) => { + if let Some(q) = &create.query { // CREATE TABLE ... AS SELECT - self.compile_create_table_as_select(name, q) + self.compile_create_table_as_select(&create.name, q) } else { - self.compile_create_table(name, columns, hive_formats, location, with_options) + self.compile_create_table( + &create.name, + &create.columns, + &create.hive_formats, + &create.location, + create_table_options(&create.table_options), + ) } } Statement::Drop { @@ -436,10 +736,21 @@ impl<'a> SqlCompiler<'a> { if_exists, .. } => self.compile_drop(object_type, names, *if_exists), - Statement::AlterTable { - name, operation, .. - } => self.compile_alter_table(name, operation), - Statement::Truncate { table_name, .. } => self.compile_truncate(table_name), + Statement::AlterTable(alter) => { + // `operation` became `operations`; sqawk handles one at a time. + match alter.operations.as_slice() { + [operation] => self.compile_alter_table(&alter.name, operation), + _ => Err(SqawkError::UnsupportedSqlFeature( + "ALTER TABLE with multiple operations is not supported".into(), + )), + } + } + Statement::Truncate(truncate) => match truncate.table_names.as_slice() { + [target] => self.compile_truncate(&target.name), + _ => Err(SqawkError::UnsupportedSqlFeature( + "TRUNCATE of multiple tables is not supported".into(), + )), + }, _ => Err(SqawkError::UnsupportedSqlFeature(format!( "Unsupported SQL statement type: {:?}", statement @@ -449,9 +760,17 @@ impl<'a> SqlCompiler<'a> { /// Compile a SQL query fn compile_query(&mut self, query: &Query) -> SqawkResult<()> { + // `ORDER BY ALL` carries no key list, so it would otherwise look + // identical to "no ORDER BY" and be silently ignored. Reject it. + if is_order_by_all(query) { + return Err(SqawkError::UnsupportedSqlFeature( + "ORDER BY ALL is not supported".into(), + )); + } + // Check if we have ORDER BY, LIMIT, or OFFSET - let has_order_by = !query.order_by.is_empty(); - let has_limit = query.limit.is_some() || query.offset.is_some(); + let has_order_by = !query_order_by(query).is_empty(); + let has_limit = query_limit(query).is_some() || query_offset(query).is_some(); // Compile the body (SELECT or set operation) match &*query.body { @@ -554,7 +873,7 @@ impl<'a> SqlCompiler<'a> { "Keep only rows in both result sets", ); } - SetOperator::Except => { + SetOperator::Except | SetOperator::Minus => { self.emit( OpCode::Except, left_results_marker.unwrap_or(0), @@ -589,8 +908,79 @@ impl<'a> SqlCompiler<'a> { } } + /// Rewrite `A INNER JOIN B ON cond` into `FROM A, B WHERE cond`. + /// + /// The implicit (comma) join path already supports aggregates and GROUP BY; + /// the explicit-JOIN path does not, and rejects `COUNT(*)` in its + /// projection resolver. For an INNER join the two spellings mean the same + /// thing, so an aggregate query over an explicit join is answered by + /// handing it to the machinery that can already do it, rather than by + /// duplicating that machinery. + /// + /// Returns `None` when the rewrite does not apply: outer joins are NOT + /// equivalent to a comma join plus WHERE, since the WHERE would discard + /// exactly the NULL-extended rows the outer join exists to produce. + fn rewrite_inner_join_as_implicit(select: &Select) -> Option
, + /// One result table per statement that produced rows, in order. + /// + /// A multi-statement script produces one result set per statement; folding + /// them into a single table would print every statement's rows under one + /// header. + pub tables: Vec
, /// Names of tables that were modified (INSERT, UPDATE, DELETE, CREATE TABLE) pub modified_tables: HashSet, /// Number of rows affected by the last DML statement (INSERT, UPDATE, DELETE) @@ -53,10 +64,7 @@ pub fn execute_vm( ) -> SqawkResult { if verbose { println!("VM Engine: Executing SQL via bytecode: {}", sql); - } - // In verbose mode, print messages about SQL features being applied - if verbose { let sql_upper = sql.to_uppercase(); if sql_upper.contains("DISTINCT") { eprintln!("Applying DISTINCT"); @@ -75,41 +83,218 @@ pub fn execute_vm( } } - // PHASE 1: SQL → BYTECODE COMPILATION + // Parse once, then compile and execute ONE STATEMENT AT A TIME. + // + // Every statement used to be folded into a single bytecode program sharing + // one result buffer and one schema, which had two consequences: + // + // - `SELECT a; SELECT b` printed the rows of both statements under the + // LAST statement's header. + // - A statement could not see the effects of the one before it, because + // modifications are applied only after the whole program finishes. So + // `UPDATE ...; SELECT ...` ran the SELECT against the pre-update table + // and, sharing the buffer, emitted nothing useful. + // + // Executing per statement gives each its own result set and makes each + // statement observe the database as the previous one left it, which is + // also what `CREATE TABLE t; INSERT INTO t ...` requires. + let dialect = sqlparser::dialect::HiveDialect {}; + let statements = + sqlparser::parser::Parser::parse_sql(&dialect, sql).map_err(SqawkError::SqlParseError)?; + + if statements.is_empty() { + return Err(SqawkError::InvalidSqlQuery( + "No SQL statements found".to_string(), + )); + } - // Use the SqlCompiler to convert the SQL statement to bytecode - let mut compiler = compiler::SqlCompiler::new(database, verbose); - let program = compiler.compile(sql)?; + let mut tables: Vec
= Vec::new(); + let mut modified_tables = HashSet::with_capacity(DEFAULT_TABLE_CAPACITY); + let mut affected_rows: usize = 0; - if verbose { - println!("Phase 1 complete - Generated bytecode:"); - println!("{}", program); - } + for statement in &statements { + // PHASE 0: MATERIALIZE DERIVED TABLES. + // + // `FROM (SELECT ...) t` has no representation in the compiler, which + // only knows how to open a cursor on a named table. Rather than teach + // every FROM site about subqueries, each derived table is executed + // first and registered under its alias, and the statement is rewritten + // to reference that name. The compiler then sees an ordinary table. + let mut statement = statement.clone(); + let derived = materialize_derived_tables(&mut statement, database, verbose)?; + + // PHASE 1: SQL -> BYTECODE, against the CURRENT database state. + let program = { + let mut compiler = compiler::SqlCompiler::new(database, verbose); + compiler.compile_statement_program(&statement)? + }; + + if verbose { + println!("Generated bytecode:"); + println!("{}", program); + } - // PHASE 2: BYTECODE → EXECUTION & RESULTS + // PHASE 2: EXECUTE + let mut vm = engine::VmEngine::new_mut(database, verbose); + vm.init(program); + vm.execute()?; + + let result_table = vm.create_result_table()?; + let modifications = vm.take_modifications(); + drop(vm); + + // PHASE 3: APPLY MODIFICATIONS before the next statement compiles. + let applied = apply_modifications(database, modifications, verbose)?; + modified_tables.extend(applied.modified_tables); + + // A DML statement reports its own count, INCLUDING zero. + // + // Guarding on `applied.affected_rows > 0` conflated two different + // things: a statement that was not DML at all (a SELECT contributes no + // modifications), and a DML statement that matched no rows. The latter + // then inherited the previous statement's count, so + // `UPDATE ... WHERE ; UPDATE ... WHERE ` + // reported 3 rows affected instead of 0. + // + // Which statements count is the same set SQL's `changes()` uses. + if is_row_counting_dml(&statement) { + affected_rows = applied.affected_rows; + } - // Initialize the VM engine with the bytecode program - let mut vm = engine::VmEngine::new_mut(database, verbose); - vm.init(program); + // Derived tables live only for the statement that declared them. + for name in derived { + database.remove_table(&name); + } - if verbose { - println!("Phase 2 starting - Executing bytecode in VM"); + if let Some(t) = result_table { + tables.push(t); + } } - // Execute all instructions in the program - vm.execute()?; + Ok(VmExecutionResult { + tables, + modified_tables, + affected_rows, + }) +} - if verbose { - println!("Phase 2 complete - Execution finished"); +/// Execute every derived table in `statement`, register each under its alias, +/// and rewrite the statement to reference the registered name. +/// +/// Returns the names registered, so the caller can drop them once the +/// statement finishes -- a derived table is scoped to the statement that +/// declares it. +fn materialize_derived_tables( + statement: &mut sqlparser::ast::Statement, + database: &mut Database, + verbose: bool, +) -> SqawkResult> { + let mut registered = Vec::new(); + if let sqlparser::ast::Statement::Query(query) = statement { + materialize_in_query(query, database, verbose, &mut registered)?; + } + Ok(registered) +} + +fn materialize_in_query( + query: &mut sqlparser::ast::Query, + database: &mut Database, + verbose: bool, + registered: &mut Vec, +) -> SqawkResult<()> { + if let sqlparser::ast::SetExpr::Select(select) = &mut *query.body { + for twj in &mut select.from { + materialize_in_factor(&mut twj.relation, database, verbose, registered)?; + for join in &mut twj.joins { + materialize_in_factor(&mut join.relation, database, verbose, registered)?; + } + } + } + Ok(()) +} + +fn materialize_in_factor( + factor: &mut sqlparser::ast::TableFactor, + database: &mut Database, + verbose: bool, + registered: &mut Vec, +) -> SqawkResult<()> { + let (subquery, alias) = match factor { + sqlparser::ast::TableFactor::Derived { + subquery, alias, .. + } => (subquery.clone(), alias.clone()), + _ => return Ok(()), + }; + + // SQL requires a derived table to be named; without one there is nothing + // for the outer query to refer to. + let alias = alias.ok_or_else(|| { + SqawkError::InvalidSqlQuery("Subquery in FROM must have an alias".to_string()) + })?; + let name = alias.name.value.to_ascii_lowercase(); + + if database.has_table(&name) { + return Err(SqawkError::InvalidSqlQuery(format!( + "Derived table alias '{}' shadows an existing table", + name + ))); } - // Get results and modifications before dropping VM (to release database borrow) - let result_table = vm.create_result_table(); - let modifications = vm.take_modifications(); + // Nested derived tables are materialized innermost-first. + let mut inner = *subquery; + materialize_in_query(&mut inner, database, verbose, registered)?; + + let inner_sql = inner.to_string(); + let result = execute_vm(&inner_sql, database, verbose)?; + let mut table = result.tables.into_iter().next_back().ok_or_else(|| { + SqawkError::InvalidSqlQuery("Subquery in FROM produced no result".to_string()) + })?; + table.set_name(name.clone()); + database.add_table(name.clone(), table)?; + registered.push(name.clone()); + + *factor = sqlparser::ast::TableFactor::Table { + name: sqlparser::ast::ObjectName(vec![sqlparser::ast::ObjectNamePart::Identifier( + sqlparser::ast::Ident::new(name), + )]), + alias: None, + args: None, + with_hints: Vec::new(), + version: None, + partitions: Vec::new(), + with_ordinality: false, + json_path: None, + sample: None, + index_hints: Vec::new(), + }; + Ok(()) +} - // Drop VM to release database borrow - drop(vm); +/// Whether a statement's affected-row count is meaningful to report. +/// +/// The INSERT/UPDATE/DELETE family, matching what SQL's `changes()` covers. A +/// SELECT or DDL statement leaves the previous count alone rather than +/// resetting it to zero. +fn is_row_counting_dml(statement: &sqlparser::ast::Statement) -> bool { + use sqlparser::ast::Statement; + matches!( + statement, + Statement::Insert(_) | Statement::Update(_) | Statement::Delete(_) + ) +} +/// Tables touched and rows affected by one statement's modifications. +struct AppliedModifications { + modified_tables: HashSet, + affected_rows: usize, +} + +/// Apply a statement's pending modifications to the database. +fn apply_modifications( + database: &mut Database, + modifications: Vec, + verbose: bool, +) -> SqawkResult { // PHASE 3: APPLY MODIFICATIONS TO DATABASE // Track which tables were modified @@ -126,6 +311,10 @@ pub fn execute_vm( let mut insert_counts_by_table: std::collections::HashMap = std::collections::HashMap::new(); + // Rows replaced in place by UPDATE, per table. + let mut replace_counts_by_table: std::collections::HashMap = + std::collections::HashMap::new(); + // First pass: collect deletions and apply non-delete modifications for modification in modifications { match modification { @@ -137,6 +326,19 @@ pub fn execute_vm( .or_insert(0) += 1; modified_tables.insert(table_name); } + engine::TableModification::Replace { + table_name, + row_index, + row, + } => { + let table = database.get_table_mut(&table_name)?; + table.replace_row(row_index, row)?; + // Tallied per table and reported once, not once per row. + *replace_counts_by_table + .entry(table_name.clone()) + .or_insert(0) += 1; + modified_tables.insert(table_name); + } engine::TableModification::Delete { table_name, row_index, @@ -273,23 +475,14 @@ pub fn execute_vm( } } - let result = result_table?; - - if verbose { - if let Some(ref table) = result { - println!( - "Result table created with {} rows and {} columns", - table.row_count(), - table.column_count() - ); - } else { - println!("Query executed successfully with no result table"); + for (_table, count) in replace_counts_by_table { + affected_rows += count; + if verbose { + eprintln!("Updated {} rows", count); } } - // Return both the result table, modified table names, and affected row count - Ok(VmExecutionResult { - table: result, + Ok(AppliedModifications { modified_tables, affected_rows, }) diff --git a/src/vm/tests.rs b/src/vm/tests.rs index 326cbde..9b45049 100644 --- a/src/vm/tests.rs +++ b/src/vm/tests.rs @@ -1559,3 +1559,149 @@ mod comparison_tests { } } } + +/// Tests for the Not / Compare / Jump opcodes. +/// +/// These exist to support the compiler unification: `Not` makes `NOT ` +/// expressible and replaces several hand-rolled inversions, while +/// `Compare`/`Jump` make multi-column key comparison two instructions +/// regardless of key width -- which is what GROUP BY needs in order to detect +/// a group change across ALL key columns instead of only the first. +mod compare_jump_not_tests { + use super::bytecode_tests::{create_instruction, execute_bytecode_program}; + use super::*; + + /// Run a program and return the single result row. + fn run_row(instructions: Vec) -> Vec { + let database = Database::new(); + let table = execute_bytecode_program(instructions, &database) + .expect("program failed") + .expect("expected a result table"); + assert_eq!(table.row_count(), 1, "expected exactly one result row"); + table.rows()[0].clone() + } + + fn init() -> Instruction { + create_instruction(OpCode::Init, 0, 1, 0, None, None) + } + + fn halt() -> Instruction { + create_instruction(OpCode::Halt, 0, 0, 0, None, None) + } + + #[test] + fn not_inverts_truthy_and_falsy() { + // r1 = 5 -> NOT r1 = 0 ; r3 = 0 -> NOT r3 = 1 + let row = run_row(vec![ + init(), + create_instruction(OpCode::Integer, 5, 1, 0, None, None), + create_instruction(OpCode::Not, 1, 2, 0, None, None), + create_instruction(OpCode::Integer, 0, 3, 0, None, None), + create_instruction(OpCode::Not, 3, 4, 0, None, None), + // Emit r2, r4 as a two-column row (registers must be contiguous). + create_instruction(OpCode::Copy, 2, 10, 0, None, None), + create_instruction(OpCode::Copy, 4, 11, 0, None, None), + create_instruction(OpCode::ResultRow, 10, 2, 0, None, None), + halt(), + ]); + assert_eq!(row[0], Value::Integer(0), "NOT 5 should be 0"); + assert_eq!(row[1], Value::Integer(1), "NOT 0 should be 1"); + } + + #[test] + fn not_propagates_null() { + // NOT NULL must be NULL (SQL UNKNOWN), not true. + let row = run_row(vec![ + init(), + create_instruction(OpCode::Null, 0, 1, 0, None, None), + create_instruction(OpCode::Not, 1, 2, 0, None, None), + create_instruction(OpCode::ResultRow, 2, 1, 0, None, None), + halt(), + ]); + assert_eq!( + row[0], + Value::Null, + "NOT NULL must stay NULL, not become true" + ); + } + + /// Build a program that compares two 2-register vectors and reports which + /// branch Jump takes: 1 = Less, 2 = Equal, 3 = Greater. + fn compare_branch(lhs: [i64; 2], rhs: [i64; 2]) -> Value { + let row = run_row(vec![ + init(), + // lhs in r1,r2 ; rhs in r3,r4 + create_instruction(OpCode::Integer, lhs[0], 1, 0, None, None), + create_instruction(OpCode::Integer, lhs[1], 2, 0, None, None), + create_instruction(OpCode::Integer, rhs[0], 3, 0, None, None), + create_instruction(OpCode::Integer, rhs[1], 4, 0, None, None), + // Compare r1..r2 against r3..r4 (2 columns) + create_instruction(OpCode::Compare, 1, 3, 2, None, None), + // Jump: less -> 7, equal -> 9, greater -> 11 + create_instruction(OpCode::Jump, 7, 9, 11, None, None), + // 7: less + create_instruction(OpCode::Integer, 1, 5, 0, None, None), + create_instruction(OpCode::Goto, 0, 13, 0, None, None), + // 9: equal + create_instruction(OpCode::Integer, 2, 5, 0, None, None), + create_instruction(OpCode::Goto, 0, 13, 0, None, None), + // 11: greater + create_instruction(OpCode::Integer, 3, 5, 0, None, None), + create_instruction(OpCode::Goto, 0, 13, 0, None, None), + // 13: emit + create_instruction(OpCode::ResultRow, 5, 1, 0, None, None), + halt(), + ]); + row[0].clone() + } + + #[test] + fn compare_jump_branches_three_ways() { + assert_eq!(compare_branch([1, 1], [1, 1]), Value::Integer(2), "equal"); + assert_eq!(compare_branch([1, 1], [1, 2]), Value::Integer(1), "less"); + assert_eq!(compare_branch([1, 2], [1, 1]), Value::Integer(3), "greater"); + } + + #[test] + fn compare_uses_all_key_columns() { + // The whole point: a difference in the SECOND column must be detected. + // The old single-Ne group-change check compared only the first, which + // is why multi-column GROUP BY collapsed to one key. + assert_ne!( + compare_branch([7, 1], [7, 2]), + Value::Integer(2), + "vectors differing only in the second column must not compare equal" + ); + } + + #[test] + fn compare_treats_nulls_as_equal() { + // Internal ordering, not SQL `=`: two NULL group keys must group + // together. SQL equality must yield UNKNOWN for NULL, which is exactly + // why Compare is a separate opcode from Eq. + let row = run_row(vec![ + init(), + create_instruction(OpCode::Null, 0, 1, 0, None, None), + create_instruction(OpCode::Null, 0, 2, 0, None, None), + create_instruction(OpCode::Compare, 1, 2, 1, None, None), + create_instruction(OpCode::Jump, 5, 7, 9, None, None), + // 5: less + create_instruction(OpCode::Integer, 1, 3, 0, None, None), + create_instruction(OpCode::Goto, 0, 11, 0, None, None), + // 7: equal + create_instruction(OpCode::Integer, 2, 3, 0, None, None), + create_instruction(OpCode::Goto, 0, 11, 0, None, None), + // 9: greater + create_instruction(OpCode::Integer, 3, 3, 0, None, None), + create_instruction(OpCode::Goto, 0, 11, 0, None, None), + // 11: emit + create_instruction(OpCode::ResultRow, 3, 1, 0, None, None), + halt(), + ]); + assert_eq!( + row[0], + Value::Integer(2), + "NULL should compare equal to NULL" + ); + } +} diff --git a/tests/data/customers.csv b/tests/data/customers.csv new file mode 100644 index 0000000..2babfbf --- /dev/null +++ b/tests/data/customers.csv @@ -0,0 +1,4 @@ +id,name +1,Ann +2,Ben +3,Cara diff --git a/tests/data/nullable.csv b/tests/data/nullable.csv new file mode 100644 index 0000000..6bb9f7d --- /dev/null +++ b/tests/data/nullable.csv @@ -0,0 +1,6 @@ +id,name,score,grade +1,Alice,90,A +2,Bob,,B +3,Charlie,70, +4,,50,D +5,Eve,90,A diff --git a/tests/data/purchases.csv b/tests/data/purchases.csv new file mode 100644 index 0000000..5f8c5a5 --- /dev/null +++ b/tests/data/purchases.csv @@ -0,0 +1,5 @@ +id,customer_id,item +10,1,Book +11,1,Pen +12,2,Desk +13,99,Orphan diff --git a/tests/data/repeats.csv b/tests/data/repeats.csv new file mode 100644 index 0000000..cffb03f --- /dev/null +++ b/tests/data/repeats.csv @@ -0,0 +1,6 @@ +id,v +1,10 +2,10 +3,20 +4,30 +5,30 diff --git a/tests/defects/mod.rs b/tests/defects/mod.rs new file mode 100644 index 0000000..0345a97 --- /dev/null +++ b/tests/defects/mod.rs @@ -0,0 +1,538 @@ +//! Red-list: the correctness defects found in the 2026-08 audit. +//! +//! Every test in this module encodes **correct** SQL behaviour for a defect +//! found in that audit. Each was `#[ignore]`d when written, because sqawk got +//! it wrong, and un-ignored as it was fixed. +//! +//! As of this commit the red list is EMPTY: every test here passes, and they +//! now serve as regression cover for the defects rather than as a to-do list. +//! Add a new `#[ignore]`d test here when a defect is found, and remove the +//! attribute when it is fixed. +//! +//! Run the red list with: +//! cargo test --test mod defects -- --ignored +//! +//! These use the exact-output helpers (`assert_query`), not the substring +//! `contains` assertions used elsewhere in the suite. Substring matching is +//! precisely why every defect below survived a 302-test green suite: a test +//! asserting `"Engineering,3"` passes even when three extra wrong rows are +//! emitted, and passes regardless of row order. + +use crate::helpers::{ + assert_after_write, assert_query, departments_csv, employees_csv, nullable_csv, orders_csv, + users_csv, +}; + +// --------------------------------------------------------------------------- +// Group 1: WHERE / GROUP BY / HAVING -- silent wrong answers +// --------------------------------------------------------------------------- + +/// Defect 1: `compile_select_with_group_by` never reads `select.selection`, +/// so WHERE is silently ignored on *every* GROUP BY query. +/// Currently returns all four departments with unfiltered counts. +#[test] +fn where_is_applied_to_group_by() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT department, COUNT(*) FROM employees WHERE salary > 60000 GROUP BY department", + "department,COUNT\n\ + Engineering,3\n\ + Sales,2", + ) +} + +/// Defect 2: group-change detection compares only the first key column +/// (single `Ne` at compiler_aggregate.rs:396), so multi-column GROUP BY +/// collapses to the first column. Currently returns 4 rows instead of 6, +/// with an arbitrary `role` value. +#[test] +fn group_by_uses_all_key_columns() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT department, role, COUNT(*) FROM employees GROUP BY department, role", + "department,role,COUNT\n\ + Engineering,Developer,2\n\ + Engineering,Manager,1\n\ + HR,Intern,1\n\ + Marketing,Analyst,1\n\ + Marketing,Specialist,1\n\ + Sales,Director,2", + ) +} + +/// Defect 3a: `query` is never passed into the GROUP BY compiler +/// (compiler.rs:760-766), so ORDER BY is silently dropped. +#[test] +fn order_by_applies_to_group_by() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT department, COUNT(*) FROM employees GROUP BY department ORDER BY department DESC", + "department,COUNT\n\ + Sales,2\n\ + Marketing,2\n\ + HR,1\n\ + Engineering,3", + ) +} + +/// Defect 3b: same root cause -- LIMIT is silently dropped on GROUP BY. +#[test] +fn limit_applies_to_group_by() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT department, COUNT(*) FROM employees GROUP BY department LIMIT 1", + "department,COUNT\n\ + Engineering,3", + ) +} + +/// Defect 7: `compile_having_operand` matches aggregates by function *type*, +/// ignoring the argument column, so `HAVING SUM(age)` binds to `SUM(salary)`. +/// Needs two same-type aggregates to surface -- MIN vs MAX bind correctly. +#[test] +fn having_binds_to_the_named_aggregate() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT department, SUM(salary), SUM(age) FROM employees \ + GROUP BY department HAVING SUM(age) > 60", + "department,SUM,SUM\n\ + Engineering,210000,97\n\ + Sales,170000,85", + ) +} + +/// Defect 6: `func.distinct` is never read, so COUNT(DISTINCT x) counts rows. +#[test] +fn count_distinct_deduplicates() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT COUNT(DISTINCT department) FROM employees", + "COUNT\n4", + ) +} + +/// Defect A: the GROUP BY result block is laid out positionally as +/// [group keys.., aggregates..] while the schema is built in projection order, +/// so headers and values disagree. Currently prints `COUNT,department` over +/// `Engineering,3`. +#[test] +fn group_by_respects_projection_order() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT COUNT(*), department FROM employees GROUP BY department", + "COUNT,department\n\ + 3,Engineering\n\ + 1,HR\n\ + 2,Marketing\n\ + 2,Sales", + ) +} + +/// Defect A (second symptom): a GROUP BY key that is not in the projection +/// still emits a phantom output column, currently named `col1`. +#[test] +fn group_by_emits_only_projected_columns() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT COUNT(*) FROM employees GROUP BY department", + "COUNT\n3\n1\n2\n2", + ) +} + +// --------------------------------------------------------------------------- +// Group 2: JOIN projection and ordering +// --------------------------------------------------------------------------- + +/// Defect B: JOIN always emits left-table columns then right-table columns, +/// while the schema is built in SELECT order. Currently prints header +/// `orders.id,users.name` over values `John,101` -- swapped. +#[test] +fn join_respects_projection_order() -> Result<(), Box> { + assert_query( + &[users_csv(), orders_csv()], + "SELECT orders.id, users.name FROM users JOIN orders ON users.id = orders.user_id", + "orders.id,users.name\n\ + 101,John\n\ + 103,John\n\ + 105,John\n\ + 102,Jane\n\ + 104,Jane", + ) +} + +/// Defect 4: ORDER BY is deliberately skipped on explicit JOINs +/// (compiler.rs:729-737 -- "isn't fully implemented. For now [...] skip"). +#[test] +fn order_by_applies_to_join() -> Result<(), Box> { + assert_query( + &[users_csv(), orders_csv()], + "SELECT users.name, orders.id FROM users JOIN orders ON users.id = orders.user_id \ + ORDER BY orders.id DESC", + "users.name,orders.id\n\ + John,105\n\ + Jane,104\n\ + John,103\n\ + Jane,102\n\ + John,101", + ) +} + +// --------------------------------------------------------------------------- +// Group 3: projection expressions +// --------------------------------------------------------------------------- + +/// Defect 5: the `_` arm of `compile_projection_expr` (compiler.rs:2393-2405) +/// emits **no instruction** on failure, leaving the register at its default +/// NULL. The same query without LIMIT returns correct values, because it takes +/// a different compilation path. +#[test] +fn arithmetic_projection_survives_limit() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name, salary*2 FROM employees LIMIT 3", + "name,expr\n\ + Alice,140000\n\ + Bob,110000\n\ + Charlie,130000", + ) +} + +/// `CASE` is routed in WHERE but hits `resolve_column_expr` in a projection, +/// which errors with "Only column references supported in SELECT". +#[test] +fn case_works_in_projection() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name, CASE WHEN age > 30 THEN 'old' ELSE 'young' END FROM employees LIMIT 3", + "name,expr\n\ + Alice,young\n\ + Bob,young\n\ + Charlie,old", + ) +} + +/// `||` parses fine in sqlparser 0.36 -- the compiler simply has no arm for +/// `BinaryOperator::StringConcat` (compiler.rs:1922). +#[test] +fn string_concat_operator_works() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name || '!' FROM employees LIMIT 2", + "expr\n\ + Alice!\n\ + Bob!", + ) +} + +/// A comparison used as a projected *value* rather than a filter. +#[test] +fn comparison_works_in_projection() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name, salary > 60000 FROM employees LIMIT 3", + "name,expr\n\ + Alice,1\n\ + Bob,0\n\ + Charlie,1", + ) +} + +/// Aggregate over an expression -- `resolve_column_expr` rejects anything that +/// is not a bare column reference. +#[test] +fn aggregate_accepts_expression_argument() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT SUM(salary + age) FROM employees", + "SUM\n540257", + ) +} + +/// An aggregate nested inside an expression is not even *detected* as an +/// aggregate query (`is_aggregate_expr` only matches a top-level Function), +/// so it falls into the plain table-scan path. +#[test] +fn aggregate_nested_in_expression_works() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT SUM(salary) + 1 FROM employees", + "expr\n540001", + ) +} + +// --------------------------------------------------------------------------- +// Group 4: WHERE clause expression gaps +// --------------------------------------------------------------------------- + +/// `NOT` has no arm in the WHERE compiler at all. +#[test] +fn not_works_in_where() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name FROM employees WHERE NOT (age > 40)", + "name\n\ + Alice\nBob\nCharlie\nDavid\nEve\nFrank\nHenry", + ) +} + +/// Table-alias-qualified columns are rejected in a single-table WHERE: +/// `compile_where_operand` has no `CompoundIdentifier` arm. +#[test] +fn alias_qualified_column_works_in_where() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name FROM employees e WHERE e.age > 30", + "name\n\ + Charlie\nDavid\nFrank\nGrace", + ) +} + +/// Column resolution is case-insensitive in a projection but case-*sensitive* +/// in WHERE: `SELECT NAME` works while `WHERE NAME = 'Alice'` errors with +/// "Column 'NAME' not found" (find_column_index, compiler.rs:6010). +#[test] +fn column_resolution_is_case_insensitive() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name FROM employees WHERE NAME = 'Alice'", + "name\nAlice", + ) +} + +// --------------------------------------------------------------------------- +// Group 5: NULL semantics and type coercion +// --------------------------------------------------------------------------- + +/// The worst defect for a CSV tool: comparing against a NULL raises a runtime +/// error instead of evaluating to UNKNOWN and filtering the row out. +/// `nullable.csv` row 2 has an empty `score`. +#[test] +fn null_comparison_yields_unknown() -> Result<(), Box> { + assert_query( + &[nullable_csv()], + "SELECT name FROM nullable WHERE score > 60", + "name\n\ + Alice\nCharlie\nEve", + ) +} + +/// A NULL must not satisfy an equality test either -- but it must not error. +/// +/// NOT ignored: this already passes, because equality is routed through +/// `compare_registers_eq`, which returns `false` for mismatched types instead +/// of erroring like the ordering comparisons do. It is kept here as a +/// regression guard: the three-valued-logic change must not break it. +#[test] +fn null_never_equals_a_value() -> Result<(), Box> { + assert_query( + &[nullable_csv()], + "SELECT name FROM nullable WHERE score = 90", + "name\n\ + Alice\nEve", + ) +} + +/// awk-like coercion: a string that parses as a number compares numerically. +/// Matches what `arithmetic_op` already does, so `x + '1'` and `x > '1'` agree. +#[test] +fn string_number_comparison_coerces() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name FROM employees WHERE salary > '60000'", + "name\n\ + Alice\nCharlie\nDavid\nFrank\nGrace", + ) +} + +// --------------------------------------------------------------------------- +// Group 6: ORDER BY expressiveness +// --------------------------------------------------------------------------- + +/// `build_sort_spec` maps the ORDER BY key into the *projected* column list +/// and errors if it is absent, so you cannot sort by a column you don't select. +#[test] +fn order_by_column_not_in_select_list() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name FROM employees ORDER BY salary DESC", + "name\n\ + Grace\nDavid\nFrank\nAlice\nCharlie\nEve\nBob\nHenry", + ) +} + +/// ORDER BY an arbitrary expression. +#[test] +fn order_by_expression() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name FROM employees ORDER BY salary * -1 LIMIT 3", + "name\n\ + Grace\nDavid\nFrank", + ) +} + +// --------------------------------------------------------------------------- +// Group 7: DML and FROM expressiveness +// --------------------------------------------------------------------------- + +/// `compile_expr` had no `Identifier` arm, so a column could not appear on the +/// right-hand side of a SET. Verified through writeback rather than a trailing +/// SELECT, because `UPDATE ...; SELECT ...` in one invocation does not emit the +/// SELECT's rows -- see `update_then_select_emits_the_select`. +#[test] +fn update_set_from_column() -> Result<(), Box> { + assert_after_write( + employees_csv(), + "employees", + "UPDATE employees SET salary = salary + 1", + "id,name,age,salary,department,role\n\ + 1,Alice,30,70001,Engineering,Developer\n\ + 2,Bob,25,55001,Marketing,Specialist\n\ + 3,Charlie,35,65001,Engineering,Manager\n\ + 4,David,40,80001,Sales,Director\n\ + 5,Eve,28,60001,Marketing,Analyst\n\ + 6,Frank,32,75001,Engineering,Developer\n\ + 7,Grace,45,90001,Sales,Director\n\ + 8,Henry,22,45001,HR,Intern", + ) +} + +/// An expression RHS combined with WHERE. +#[test] +fn update_set_expression_with_where() -> Result<(), Box> { + assert_after_write( + employees_csv(), + "employees", + "UPDATE employees SET salary = salary * 2 WHERE department = 'HR'", + "id,name,age,salary,department,role\n\ + 1,Alice,30,70000,Engineering,Developer\n\ + 2,Bob,25,55000,Marketing,Specialist\n\ + 3,Charlie,35,65000,Engineering,Manager\n\ + 4,David,40,80000,Sales,Director\n\ + 5,Eve,28,60000,Marketing,Analyst\n\ + 6,Frank,32,75000,Engineering,Developer\n\ + 7,Grace,45,90000,Sales,Director\n\ + 8,Henry,22,90000,HR,Intern", + ) +} + +/// A DML statement followed by a SELECT must run the SELECT against the +/// updated table and emit its rows. Statements used to share one bytecode +/// program and one result buffer, and modifications were applied only after +/// the whole program finished, so the SELECT saw the pre-update table. +#[test] +fn update_then_select_emits_the_select() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "UPDATE employees SET salary = 1 WHERE id = 1; \ + SELECT name, salary FROM employees WHERE id = 1", + "name,salary\nAlice,1", + ) +} + +/// UPDATE must leave rows where it found them. +/// +/// It is implemented as delete-plus-insert, and the insert appends, so any row +/// it touches jumps to the end of the table. This is not merely cosmetic: with +/// --write the reordering is persisted to the user's file. +#[test] +fn update_preserves_row_order() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "UPDATE employees SET salary = 1 WHERE id = 1; SELECT id, salary FROM employees LIMIT 3", + "id,salary\n1,1\n2,55000\n3,65000", + ) +} + +/// Derived tables parse fine in sqlparser 0.36; `TableFactor::Derived` simply +/// has no handler anywhere in the compiler. +#[test] +fn derived_table_in_from() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name FROM (SELECT name FROM employees) t LIMIT 2", + "name\n\ + Alice\nBob", + ) +} + +// --------------------------------------------------------------------------- +// Group 8: window functions +// --------------------------------------------------------------------------- + +/// Defect 8: an aggregate OVER (PARTITION BY x) with no ORDER BY must return +/// the *partition total* on every row, not a running total. +#[test] +fn partition_aggregate_returns_partition_total() -> Result<(), Box> { + assert_query( + &[employees_csv()], + "SELECT name, SUM(salary) OVER (PARTITION BY department) FROM employees", + "name,SUM\n\ + Alice,210000\n\ + Charlie,210000\n\ + Frank,210000\n\ + Henry,45000\n\ + Bob,115000\n\ + Eve,115000\n\ + David,170000\n\ + Grace,170000", + ) +} + +// --------------------------------------------------------------------------- +// Group 9: aggregate + GROUP BY interaction on the departments fixture +// --------------------------------------------------------------------------- + +/// The suite's existing `test_group_by_with_order_by` asserts this ordering and +/// **passes by coincidence**: alphabetical department order happens to equal +/// avg_salary-descending order in this fixture. Pinned here as an exact-output +/// test so the coincidence cannot hide a regression again. +#[test] +fn group_by_order_by_avg_is_not_coincidental() -> Result<(), Box> { + assert_query( + &[departments_csv()], + "SELECT department, COUNT(*) FROM departments GROUP BY department \ + ORDER BY COUNT(*) ASC, department DESC", + "department,COUNT\n\ + Sales,2\n\ + Marketing,2\n\ + Engineering,3", + ) +} + +// --------------------------------------------------------------------------- +// Group 10: multi-statement output +// --------------------------------------------------------------------------- + +/// Each statement in a `stmt; stmt` script must print its own result set with +/// its own header. Currently the rows of every statement are concatenated and +/// printed under the LAST statement's schema, so +/// `SELECT name ...; SELECT COUNT(*) ...` emits the header `COUNT` above +/// `Alice`, `Charlie`, `3`. +#[test] +fn multi_statement_output_has_per_statement_headers() -> Result<(), Box> { + assert_query( + &["tests/data/people.csv"], + "SELECT name FROM people WHERE age > 30; SELECT COUNT(*) FROM people", + "name\n\ + Alice\n\ + Charlie\n\ + COUNT\n\ + 3", + ) +} + +// --------------------------------------------------------------------------- +// Group 11: aggregates over joins +// --------------------------------------------------------------------------- + +/// An aggregate over a join is rejected: the join projection resolver accepts +/// only column references, so `COUNT(*)` never reaches the aggregate path. +#[test] +fn aggregate_over_join() -> Result<(), Box> { + assert_query( + &[users_csv(), orders_csv()], + "SELECT COUNT(*) FROM users JOIN orders ON users.id = orders.user_id", + "COUNT\n5", + ) +} diff --git a/tests/golden/mod.rs b/tests/golden/mod.rs new file mode 100644 index 0000000..966f757 --- /dev/null +++ b/tests/golden/mod.rs @@ -0,0 +1,2291 @@ +//! Characterization ("golden") tests: exact-output pins on behaviour that +//! sqawk currently gets RIGHT. +//! +//! Generated from real output, then reviewed. These exist to be a semantic +//! oracle for refactors -- most immediately the direct sqlparser 0.36 -> 0.62 +//! jump, which has no bisection handle and ~130 mechanical edit sites. +//! +//! Unlike the rest of the suite, these assert the COMPLETE stdout: every row, +//! in order. If a refactor drops a row, reorders output, renames a header, or +//! changes a value, exactly one of these fails and names the query. +//! +//! Known-wrong behaviour is NOT pinned here -- it lives in `tests/defects/`. + +use crate::helpers::assert_query; + +#[test] +fn select_star() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT * FROM employees", + concat!( + "id,name,age,salary,department,role", + "\n1,Alice,30,70000,Engineering,Developer", + "\n2,Bob,25,55000,Marketing,Specialist", + "\n3,Charlie,35,65000,Engineering,Manager", + "\n4,David,40,80000,Sales,Director", + "\n5,Eve,28,60000,Marketing,Analyst", + "\n6,Frank,32,75000,Engineering,Developer", + "\n7,Grace,45,90000,Sales,Director", + "\n8,Henry,22,45000,HR,Intern", + ), + ) +} + +#[test] +fn select_columns() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, salary FROM employees", + concat!( + "name,salary", + "\nAlice,70000", + "\nBob,55000", + "\nCharlie,65000", + "\nDavid,80000", + "\nEve,60000", + "\nFrank,75000", + "\nGrace,90000", + "\nHenry,45000", + ), + ) +} + +#[test] +fn select_literal() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT 1", + concat!("col0", "\n1",), + ) +} + +#[test] +fn where_eq() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department = 'Engineering'", + concat!("name", "\nAlice", "\nCharlie", "\nFrank",), + ) +} + +#[test] +fn where_gt() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE salary > 70000", + concat!("name", "\nDavid", "\nFrank", "\nGrace",), + ) +} + +#[test] +fn where_and() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE salary > 60000 AND department = 'Engineering'", + concat!("name", "\nAlice", "\nCharlie", "\nFrank",), + ) +} + +#[test] +fn where_or() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department = 'HR' OR department = 'Sales'", + concat!("name", "\nDavid", "\nGrace", "\nHenry",), + ) +} + +#[test] +fn where_ne() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department <> 'Engineering'", + concat!("name", "\nBob", "\nDavid", "\nEve", "\nGrace", "\nHenry",), + ) +} + +#[test] +fn where_like() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE name LIKE 'A%'", + concat!("name", "\nAlice",), + ) +} + +#[test] +fn where_not_like() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE name NOT LIKE 'A%'", + concat!( + "name", + "\nBob", + "\nCharlie", + "\nDavid", + "\nEve", + "\nFrank", + "\nGrace", + "\nHenry", + ), + ) +} + +#[test] +fn where_ilike() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE name ILIKE 'a%'", + concat!("name", "\nAlice",), + ) +} + +#[test] +fn where_between() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE age BETWEEN 30 AND 40", + concat!("name", "\nAlice", "\nCharlie", "\nDavid", "\nFrank",), + ) +} + +#[test] +fn where_not_between() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE age NOT BETWEEN 30 AND 40", + concat!("name", "\nBob", "\nEve", "\nGrace", "\nHenry",), + ) +} + +#[test] +fn where_in_list() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department IN ('HR', 'Sales')", + concat!("name", "\nDavid", "\nGrace", "\nHenry",), + ) +} + +#[test] +fn where_not_in_list() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department NOT IN ('HR', 'Sales')", + concat!("name", "\nAlice", "\nBob", "\nCharlie", "\nEve", "\nFrank",), + ) +} + +#[test] +fn where_is_null() -> Result<(), Box> { + assert_query( + &["tests/data/nullable.csv"], + "SELECT id FROM nullable WHERE score IS NULL", + concat!("id", "\n2",), + ) +} + +#[test] +fn where_is_not_null() -> Result<(), Box> { + assert_query( + &["tests/data/nullable.csv"], + "SELECT id FROM nullable WHERE score IS NOT NULL", + concat!("id", "\n1", "\n3", "\n4", "\n5",), + ) +} + +#[test] +fn order_by_asc() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, salary FROM employees ORDER BY salary ASC", + concat!( + "name,salary", + "\nHenry,45000", + "\nBob,55000", + "\nEve,60000", + "\nCharlie,65000", + "\nAlice,70000", + "\nFrank,75000", + "\nDavid,80000", + "\nGrace,90000", + ), + ) +} + +#[test] +fn order_by_desc() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, salary FROM employees ORDER BY salary DESC", + concat!( + "name,salary", + "\nGrace,90000", + "\nDavid,80000", + "\nFrank,75000", + "\nAlice,70000", + "\nCharlie,65000", + "\nEve,60000", + "\nBob,55000", + "\nHenry,45000", + ), + ) +} + +#[test] +fn order_by_multi() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department, name FROM employees ORDER BY department ASC, name DESC", + concat!( + "department,name", + "\nEngineering,Frank", + "\nEngineering,Charlie", + "\nEngineering,Alice", + "\nHR,Henry", + "\nMarketing,Eve", + "\nMarketing,Bob", + "\nSales,Grace", + "\nSales,David", + ), + ) +} + +#[test] +fn limit() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees LIMIT 3", + concat!("name", "\nAlice", "\nBob", "\nCharlie",), + ) +} + +#[test] +fn limit_offset() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees LIMIT 3 OFFSET 2", + concat!("name", "\nCharlie", "\nDavid", "\nEve",), + ) +} + +#[test] +fn order_by_limit() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, salary FROM employees ORDER BY salary DESC LIMIT 3", + concat!( + "name,salary", + "\nGrace,90000", + "\nDavid,80000", + "\nFrank,75000", + ), + ) +} + +#[test] +fn distinct() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT DISTINCT department FROM employees", + concat!( + "department", + "\nEngineering", + "\nMarketing", + "\nSales", + "\nHR", + ), + ) +} + +#[test] +fn distinct_star() -> Result<(), Box> { + assert_query( + &["tests/data/duplicates.csv"], + "SELECT DISTINCT name, department FROM duplicates", + concat!( + "name,department", + "\nAlice,Engineering", + "\nBob,Marketing", + "\nCharlie,Engineering", + "\nDave,Finance", + "\nEve,HR", + "\nFrank,Sales", + ), + ) +} + +#[test] +fn agg_count() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT COUNT(*) FROM employees", + concat!("COUNT", "\n8",), + ) +} + +#[test] +fn agg_sum() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT SUM(salary) FROM employees", + concat!("SUM", "\n540000",), + ) +} + +#[test] +fn agg_avg() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT AVG(salary) FROM employees", + concat!("AVG", "\n67500",), + ) +} + +#[test] +fn agg_min_max() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT MIN(salary), MAX(salary) FROM employees", + concat!("MIN,MAX", "\n45000,90000",), + ) +} + +#[test] +fn agg_with_where() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT COUNT(*) FROM employees WHERE department = 'Engineering'", + concat!("COUNT", "\n3",), + ) +} + +#[test] +fn agg_alias() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT COUNT(*) AS n, SUM(salary) AS total FROM employees", + concat!("n,total", "\n8,540000",), + ) +} + +#[test] +fn group_by_basic() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*) FROM employees GROUP BY department", + concat!( + "department,COUNT", + "\nEngineering,3", + "\nHR,1", + "\nMarketing,2", + "\nSales,2", + ), + ) +} + +#[test] +fn group_by_sum() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department, SUM(salary) FROM employees GROUP BY department", + concat!( + "department,SUM", + "\nEngineering,210000", + "\nHR,45000", + "\nMarketing,115000", + "\nSales,170000", + ), + ) +} + +#[test] +fn group_by_having() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*) FROM employees GROUP BY department HAVING COUNT(*) > 1", + concat!( + "department,COUNT", + "\nEngineering,3", + "\nMarketing,2", + "\nSales,2", + ), + ) +} + +#[test] +fn join_on() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users JOIN orders ON users.id = orders.user_id", + concat!( + "users.name,orders.id", + "\nJohn,101", + "\nJohn,103", + "\nJohn,105", + "\nJane,102", + "\nJane,104", + ), + ) +} + +#[test] +fn join_inner() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.date FROM users INNER JOIN orders ON users.id = orders.user_id", + concat!( + "users.name,orders.date", + "\nJohn,2023-01-15", + "\nJohn,2023-02-10", + "\nJohn,2023-03-05", + "\nJane,2023-01-20", + "\nJane,2023-02-25", + ), + ) +} + +#[test] +fn join_left() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users LEFT JOIN orders ON users.id = orders.user_id", + concat!( + "users.name,orders.id", + "\nJohn,101", + "\nJohn,103", + "\nJohn,105", + "\nJane,102", + "\nJane,104", + ), + ) +} + +#[test] +fn join_right() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users RIGHT JOIN orders ON users.id = orders.user_id", + concat!( + "users.name,orders.id", + "\nJohn,101", + "\nJane,102", + "\nJohn,103", + "\nJane,104", + "\nJohn,105", + ), + ) +} + +#[test] +fn join_full() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users FULL JOIN orders ON users.id = orders.user_id", + concat!( + "users.name,orders.id", + "\nJohn,101", + "\nJohn,103", + "\nJohn,105", + "\nJane,102", + "\nJane,104", + ), + ) +} + +#[test] +fn join_cross() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users CROSS JOIN orders", + concat!( + "users.name,orders.id", + "\nJohn,101", + "\nJohn,102", + "\nJohn,103", + "\nJohn,104", + "\nJohn,105", + "\nJane,101", + "\nJane,102", + "\nJane,103", + "\nJane,104", + "\nJane,105", + ), + ) +} + +#[test] +fn join_implicit() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users, orders WHERE users.id = orders.user_id", + concat!( + "users.name,orders.id", + "\nJohn,101", + "\nJohn,103", + "\nJohn,105", + "\nJane,102", + "\nJane,104", + ), + ) +} + +#[test] +fn join_three() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv", "tests/data/products.csv"], + "SELECT users.name, products.name FROM users, orders, products WHERE users.id = orders.user_id AND orders.product_id = products.product_id", + concat!( + "users.name,products.name", + "\nJohn,Laptop", + "\nJohn,Headphones", + "\nJohn,Keyboard", + "\nJane,Phone", + "\nJane,Monitor", + ), + ) +} + +#[test] +fn union() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department FROM employees WHERE age < 26 UNION SELECT department FROM employees WHERE age > 44", + concat!( + "department", + "\nMarketing", + "\nHR", + "\nSales", + ), + ) +} + +#[test] +fn union_all() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department FROM employees WHERE age < 26 UNION ALL SELECT department FROM employees WHERE age > 44", + concat!( + "department", + "\nMarketing", + "\nHR", + "\nSales", + ), + ) +} + +#[test] +fn intersect() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department FROM employees WHERE age < 31 INTERSECT SELECT department FROM employees WHERE age > 31", + concat!( + "department", + "\nEngineering", + ), + ) +} + +#[test] +fn except() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department FROM employees WHERE age < 31 EXCEPT SELECT department FROM employees WHERE age > 31", + concat!( + "department", + "\nMarketing", + "\nHR", + ), + ) +} + +#[test] +fn subquery_scalar() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE salary = (SELECT MAX(salary) FROM employees)", + concat!("name", "\nGrace",), + ) +} + +#[test] +fn subquery_in() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT name FROM users WHERE id IN (SELECT user_id FROM orders)", + concat!("name", "\nJohn", "\nJane",), + ) +} + +#[test] +fn subquery_exists() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT name FROM users WHERE EXISTS (SELECT 1 FROM orders WHERE orders.user_id = users.id)", + concat!( + "name", + "\nJohn", + "\nJane", + ), + ) +} + +#[test] +fn str_upper() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT UPPER(name) FROM employees", + concat!( + "UPPER(name)", + "\nALICE", + "\nBOB", + "\nCHARLIE", + "\nDAVID", + "\nEVE", + "\nFRANK", + "\nGRACE", + "\nHENRY", + ), + ) +} + +#[test] +fn str_lower() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT LOWER(name) FROM employees", + concat!( + "LOWER(name)", + "\nalice", + "\nbob", + "\ncharlie", + "\ndavid", + "\neve", + "\nfrank", + "\ngrace", + "\nhenry", + ), + ) +} + +#[test] +fn str_length() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT LENGTH(name) FROM employees", + concat!( + "LENGTH(name)", + "\n5", + "\n3", + "\n7", + "\n5", + "\n3", + "\n5", + "\n5", + "\n5", + ), + ) +} + +#[test] +fn str_substr() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT SUBSTR(name, 1, 3) FROM employees", + concat!( + "SUBSTR(name)", + "\nAli", + "\nBob", + "\nCha", + "\nDav", + "\nEve", + "\nFra", + "\nGra", + "\nHen", + ), + ) +} + +#[test] +fn str_replace() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT REPLACE(department, 'Engineering', 'Eng') FROM employees", + concat!( + "REPLACE(department)", + "\nEng", + "\nMarketing", + "\nEng", + "\nSales", + "\nMarketing", + "\nEng", + "\nSales", + "\nHR", + ), + ) +} + +#[test] +fn str_trim() -> Result<(), Box> { + assert_query( + &["tests/data/strings.csv"], + "SELECT TRIM(padded_text) FROM strings", + concat!( + "expr", + "\ntrimme", + "\nneeds space", + "\nwhitespace", + "\nextra", + "\npadding", + ), + ) +} + +#[test] +fn str_concat() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT CONCAT(name, department) FROM employees", + concat!( + "CONCAT(name)", + "\nAliceEngineering", + "\nBobMarketing", + "\nCharlieEngineering", + "\nDavidSales", + "\nEveMarketing", + "\nFrankEngineering", + "\nGraceSales", + "\nHenryHR", + ), + ) +} + +#[test] +fn str_left_right() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT LEFT(name, 2), RIGHT(name, 2) FROM employees", + concat!( + "LEFT(name),RIGHT(name)", + "\nAl,ce", + "\nBo,ob", + "\nCh,ie", + "\nDa,id", + "\nEv,ve", + "\nFr,nk", + "\nGr,ce", + "\nHe,ry", + ), + ) +} + +#[test] +fn math_abs() -> Result<(), Box> { + assert_query( + &["tests/data/boundaries.csv"], + // NOTE: ABS(i64::MIN) has no representable result. sqawk saturates to + // i64::MAX rather than erroring or wrapping. Pinned so any change to + // that choice is deliberate, not so that it is endorsed. + "SELECT ABS(value) FROM boundaries", + concat!( + "ABS(value)", + "\n0", + "\n1", + "\n1", + "\n9223372036854775807", + "\n9223372036854775807", + ), + ) +} + +#[test] +fn math_round() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT ROUND(salary) FROM employees", + concat!( + "ROUND(salary)", + "\n70000", + "\n55000", + "\n65000", + "\n80000", + "\n60000", + "\n75000", + "\n90000", + "\n45000", + ), + ) +} + +#[test] +fn arith_add() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, salary + 1000 FROM employees", + concat!( + "name,expr", + "\nAlice,71000", + "\nBob,56000", + "\nCharlie,66000", + "\nDavid,81000", + "\nEve,61000", + "\nFrank,76000", + "\nGrace,91000", + "\nHenry,46000", + ), + ) +} + +#[test] +fn arith_mul() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, salary * 2 FROM employees", + concat!( + "name,expr", + "\nAlice,140000", + "\nBob,110000", + "\nCharlie,130000", + "\nDavid,160000", + "\nEve,120000", + "\nFrank,150000", + "\nGrace,180000", + "\nHenry,90000", + ), + ) +} + +#[test] +fn arith_div() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, salary / 1000 FROM employees", + concat!( + "name,expr", + "\nAlice,70", + "\nBob,55", + "\nCharlie,65", + "\nDavid,80", + "\nEve,60", + "\nFrank,75", + "\nGrace,90", + "\nHenry,45", + ), + ) +} + +#[test] +fn arith_mod() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, age % 7 FROM employees", + concat!( + "name,expr", + "\nAlice,2", + "\nBob,4", + "\nCharlie,0", + "\nDavid,5", + "\nEve,0", + "\nFrank,4", + "\nGrace,3", + "\nHenry,1", + ), + ) +} + +#[test] +fn arith_unary_minus() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, -salary FROM employees", + concat!( + "name,expr", + "\nAlice,-70000", + "\nBob,-55000", + "\nCharlie,-65000", + "\nDavid,-80000", + "\nEve,-60000", + "\nFrank,-75000", + "\nGrace,-90000", + "\nHenry,-45000", + ), + ) +} + +#[test] +fn coalesce() -> Result<(), Box> { + assert_query( + &["tests/data/nullable.csv"], + "SELECT id, COALESCE(score, 0) FROM nullable", + concat!( + "id,COALESCE(score)", + "\n1,90", + "\n2,0", + "\n3,70", + "\n4,50", + "\n5,90", + ), + ) +} + +#[test] +fn nullif() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, NULLIF(department, 'HR') FROM employees", + concat!( + "name,NULLIF(department)", + "\nAlice,Engineering", + "\nBob,Marketing", + "\nCharlie,Engineering", + "\nDavid,Sales", + "\nEve,Marketing", + "\nFrank,Engineering", + "\nGrace,Sales", + "\nHenry,NULL", + ), + ) +} + +#[test] +fn win_row_number() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, ROW_NUMBER() OVER (ORDER BY salary DESC) FROM employees", + concat!( + "name,ROW_NUMBER", + "\nGrace,1", + "\nDavid,2", + "\nFrank,3", + "\nAlice,4", + "\nCharlie,5", + "\nEve,6", + "\nBob,7", + "\nHenry,8", + ), + ) +} + +#[test] +fn win_rank() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, RANK() OVER (ORDER BY department) FROM employees", + concat!( + "name,RANK", + "\nAlice,1", + "\nCharlie,1", + "\nFrank,1", + "\nHenry,4", + "\nBob,5", + "\nEve,5", + "\nDavid,7", + "\nGrace,7", + ), + ) +} + +#[test] +fn win_dense_rank() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, DENSE_RANK() OVER (ORDER BY department) FROM employees", + concat!( + "name,DENSE_RANK", + "\nAlice,1", + "\nCharlie,1", + "\nFrank,1", + "\nHenry,2", + "\nBob,3", + "\nEve,3", + "\nDavid,4", + "\nGrace,4", + ), + ) +} + +#[test] +fn win_lag() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, LAG(salary) OVER (ORDER BY salary) FROM employees", + concat!( + "name,LAG", + "\nHenry,NULL", + "\nBob,45000", + "\nEve,55000", + "\nCharlie,60000", + "\nAlice,65000", + "\nFrank,70000", + "\nDavid,75000", + "\nGrace,80000", + ), + ) +} + +#[test] +fn win_lead() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, LEAD(salary) OVER (ORDER BY salary) FROM employees", + concat!( + "name,LEAD", + "\nHenry,55000", + "\nBob,60000", + "\nEve,65000", + "\nCharlie,70000", + "\nAlice,75000", + "\nFrank,80000", + "\nDavid,90000", + "\nGrace,NULL", + ), + ) +} + +#[test] +fn case_in_where() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE CASE WHEN age > 30 THEN 1 ELSE 0 END = 1", + concat!("name", "\nCharlie", "\nDavid", "\nFrank", "\nGrace",), + ) +} + +#[test] +fn cast_in_where() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE CAST(salary AS INTEGER) > 70000", + concat!("name", "\nDavid", "\nFrank", "\nGrace",), + ) +} + +#[test] +fn alias_column() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name AS who, salary AS pay FROM employees", + concat!( + "who,pay", + "\nAlice,70000", + "\nBob,55000", + "\nCharlie,65000", + "\nDavid,80000", + "\nEve,60000", + "\nFrank,75000", + "\nGrace,90000", + "\nHenry,45000", + ), + ) +} + +// --------------------------------------------------------------------------- +// Outer joins WITH unmatched rows on both sides. +// +// The users/orders fixtures cannot exercise NULL-fill: every user has an order +// and every order has a user, so LEFT/RIGHT/FULL all degenerate to INNER there. +// customers/purchases has an unmatched row on each side (Cara has no purchase; +// purchase 13 references customer 99), so these pin the actual outer-join +// semantics. +// --------------------------------------------------------------------------- + +#[test] +fn outer_join_inner_baseline() -> Result<(), Box> { + assert_query( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT customers.name, purchases.item FROM customers INNER JOIN purchases ON customers.id = purchases.customer_id", + concat!( + "customers.name,purchases.item", + "\nAnn,Book", + "\nAnn,Pen", + "\nBen,Desk", + ), + ) +} + +#[test] +fn outer_join_left_nullfill() -> Result<(), Box> { + assert_query( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT customers.name, purchases.item FROM customers LEFT JOIN purchases ON customers.id = purchases.customer_id", + concat!( + "customers.name,purchases.item", + "\nAnn,Book", + "\nAnn,Pen", + "\nBen,Desk", + "\nCara,NULL", + ), + ) +} + +#[test] +fn outer_join_right_nullfill() -> Result<(), Box> { + assert_query( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT customers.name, purchases.item FROM customers RIGHT JOIN purchases ON customers.id = purchases.customer_id", + concat!( + "customers.name,purchases.item", + "\nAnn,Book", + "\nAnn,Pen", + "\nBen,Desk", + "\nNULL,Orphan", + ), + ) +} + +#[test] +fn outer_join_full_nullfill() -> Result<(), Box> { + assert_query( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT customers.name, purchases.item FROM customers FULL JOIN purchases ON customers.id = purchases.customer_id", + concat!( + "customers.name,purchases.item", + "\nAnn,Book", + "\nAnn,Pen", + "\nBen,Desk", + "\nCara,NULL", + "\nNULL,Orphan", + ), + ) +} + +// --------------------------------------------------------------------------- +// Error paths. +// +// Only two tests in the pre-existing suite assert a failure at all, so nothing +// pinned which inputs must be REJECTED. These do -- and they matter especially +// after an AST upgrade, where an unhandled node can quietly start being +// accepted (or a supported one start being refused). +// --------------------------------------------------------------------------- + +use crate::helpers::assert_after_write; +use crate::helpers::assert_query_fails; + +#[test] +fn err_unknown_column() -> Result<(), Box> { + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT nosuchcolumn FROM employees", + "not found", + ) +} + +#[test] +fn err_unknown_table() -> Result<(), Box> { + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT name FROM nosuchtable", + "not found", + ) +} + +#[test] +fn err_syntax() -> Result<(), Box> { + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT FROM WHERE", + "Failed to execute SQL", + ) +} + +#[test] +fn err_group_by_all_rejected() -> Result<(), Box> { + // GROUP BY ALL parses but carries no key list. It must be rejected rather + // than silently treated as "no GROUP BY". + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*) FROM employees GROUP BY ALL", + "GROUP BY ALL is not supported", + ) +} + +#[test] +fn err_order_by_all_rejected() -> Result<(), Box> { + // Likewise ORDER BY ALL must not be silently ignored. Under the dialect + // sqawk parses with, `ALL` here comes through as an ordinary identifier + // rather than as OrderByKind::All, so this is rejected as an unknown + // column instead of by the explicit guard in compile_query. Either route + // is acceptable; silently returning unsorted rows is not, which is what + // this test pins. + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT name FROM employees ORDER BY ALL", + "not found", + ) +} + +#[test] +fn err_semi_join_rejected() -> Result<(), Box> { + // Join kinds sqawk cannot execute must produce a clear error, not a + // silently wrong join. This is the arm that a sqlparser variant rename + // would otherwise slip past. + assert_query_fails( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name FROM users LEFT SEMI JOIN orders ON users.id = orders.user_id", + "not supported", + ) +} + +// --------------------------------------------------------------------------- +// WHERE combined with LIMIT/OFFSET. +// +// The suite tested WHERE and tested LIMIT, but never the two together -- and +// the interaction was broken: the WHERE-false jump was computed as a fixed +// offset from the test, which landed on the LIMIT counter's DecrJumpZero. Every +// row that FAILED the filter still decremented the limit, so LIMIT counted rows +// scanned rather than rows returned, silently truncating results. +// --------------------------------------------------------------------------- + +#[test] +fn where_with_limit_not_reached() -> Result<(), Box> { + // Limit is larger than the match count: every match must be returned. + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department = 'Engineering' LIMIT 5", + "name\nAlice\nCharlie\nFrank", + ) +} + +#[test] +fn where_with_limit_late_matches() -> Result<(), Box> { + // Both matches come after three non-matching rows, so a limit that counts + // scanned rows would drop Grace. + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department = 'Sales' LIMIT 5", + "name\nDavid\nGrace", + ) +} + +#[test] +fn where_with_limit_reached() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department = 'Engineering' LIMIT 2", + "name\nAlice\nCharlie", + ) +} + +#[test] +fn where_with_limit_offset() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department = 'Engineering' LIMIT 5 OFFSET 1", + "name\nCharlie\nFrank", + ) +} + +#[test] +fn where_with_offset_only() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE age > 25 OFFSET 2", + "name\nDavid\nEve\nFrank\nGrace", + ) +} + +/// DISTINCT rides on the aggregate function type as a flag bit, so it composes +/// with every aggregate rather than only COUNT. repeats.csv holds 10,10,20,30,30. +#[test] +fn aggregate_distinct_variants() -> Result<(), Box> { + assert_query( + &["tests/data/repeats.csv"], + "SELECT SUM(v), SUM(DISTINCT v), COUNT(v), COUNT(DISTINCT v), \ + MIN(DISTINCT v), MAX(DISTINCT v), AVG(DISTINCT v) FROM repeats", + "SUM,SUM,COUNT,COUNT,MIN,MAX,AVG\n100,60,5,3,10,30,20", + ) +} + +// --------------------------------------------------------------------------- +// Expression position symmetry. +// +// SQL draws no distinction between an expression used as a filter and the same +// expression used as a projected value, but sqawk had three parallel +// expression compilers and each supported a different subset. These pin both +// positions for the same node kinds so the two cannot drift apart again. +// --------------------------------------------------------------------------- + +#[test] +fn case_as_projected_value() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, CASE WHEN age > 30 THEN 'old' ELSE 'young' END FROM employees LIMIT 3", + "name,expr\nAlice,young\nBob,young\nCharlie,old", + ) +} + +#[test] +fn cast_as_projected_value() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT CAST(salary AS TEXT) FROM employees LIMIT 2", + "expr\n70000\n55000", + ) +} + +#[test] +fn is_null_as_projected_value() -> Result<(), Box> { + assert_query( + &["tests/data/nullable.csv"], + "SELECT id, score IS NULL FROM nullable", + "id,expr\n1,0\n2,1\n3,0\n4,0\n5,0", + ) +} + +#[test] +fn like_as_projected_value() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, name LIKE 'A%' FROM employees LIMIT 2", + "name,expr\nAlice,1\nBob,0", + ) +} + +#[test] +fn between_as_projected_value() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, age BETWEEN 30 AND 40 FROM employees LIMIT 3", + "name,expr\nAlice,1\nBob,0\nCharlie,1", + ) +} + +#[test] +fn trim_as_filter() -> Result<(), Box> { + // TRIM/CEIL/FLOOR were handled only in the projection path, so they worked + // in a SELECT list and failed in WHERE -- the mirror image of SUBSTR. + assert_query( + &["tests/data/strings.csv"], + "SELECT id FROM strings WHERE TRIM(padded_text) = 'trimme'", + "id\n1", + ) +} + +#[test] +fn ceil_as_filter() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE CEIL(salary) > 85000", + "name\nGrace", + ) +} + +#[test] +fn concat_operator_as_filter() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE name || '!' = 'Alice!'", + "name\nAlice", + ) +} + +// --------------------------------------------------------------------------- +// GROUP BY interactions that were previously unrepresentable. +// --------------------------------------------------------------------------- + +#[test] +fn group_by_with_where_and_having() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*) FROM employees WHERE age > 25 GROUP BY department \ + HAVING COUNT(*) > 1", + // age > 25 drops Bob(25) and Henry(22), leaving Marketing with one + // row, which HAVING then removes. + "department,COUNT\nEngineering,3\nSales,2", + ) +} + +#[test] +fn group_by_three_columns() -> Result<(), Box> { + // Every key column must participate; a difference in the last one still + // starts a new group. + assert_query( + &["tests/data/employees.csv"], + "SELECT department, role, COUNT(*) FROM employees GROUP BY department, role", + "department,role,COUNT\n\ + Engineering,Developer,2\n\ + Engineering,Manager,1\n\ + HR,Intern,1\n\ + Marketing,Analyst,1\n\ + Marketing,Specialist,1\n\ + Sales,Director,2", + ) +} + +#[test] +fn group_by_with_where_excluding_a_whole_group() -> Result<(), Box> { + // HR's only row is filtered out, so the group must disappear entirely + // rather than appear with a zero or unfiltered count. + assert_query( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*) FROM employees WHERE salary > 60000 GROUP BY department", + "department,COUNT\nEngineering,3\nSales,2", + ) +} + +#[test] +fn having_on_a_group_key() -> Result<(), Box> { + // The HAVING operand compiler handled only numeric literals and silently + // emitted nothing for a string, so this matched no group at all. + assert_query( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*) FROM employees GROUP BY department \ + HAVING department = 'Sales'", + "department,COUNT\nSales,2", + ) +} + +#[test] +fn having_selects_between_same_type_aggregates() -> Result<(), Box> { + // Two SUMs: HAVING must bind to the one it names, not the first in the + // projection. SUM(age) values are 97/22/53/85 by department. + assert_query( + &["tests/data/employees.csv"], + "SELECT department, SUM(salary), SUM(age) FROM employees GROUP BY department \ + HAVING SUM(age) > 90", + "department,SUM,SUM\nEngineering,210000,97", + ) +} + +// --------------------------------------------------------------------------- +// ORDER BY / LIMIT over paths that emit rows directly rather than through the +// cursor-based sorter: GROUP BY, plain aggregates, and joins. +// --------------------------------------------------------------------------- + +#[test] +fn group_by_order_by_aggregate() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*) FROM employees GROUP BY department \ + ORDER BY COUNT(*) DESC, department ASC", + "department,COUNT\nEngineering,3\nMarketing,2\nSales,2\nHR,1", + ) +} + +#[test] +fn group_by_order_by_limit_together() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department, SUM(salary) FROM employees GROUP BY department \ + ORDER BY SUM(salary) DESC LIMIT 2", + "department,SUM\nEngineering,210000\nSales,170000", + ) +} + +#[test] +fn group_by_order_by_offset() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*) FROM employees GROUP BY department \ + ORDER BY department ASC LIMIT 2 OFFSET 1", + "department,COUNT\nHR,1\nMarketing,2", + ) +} + +#[test] +fn join_order_by_descending() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users JOIN orders ON users.id = orders.user_id \ + ORDER BY orders.id DESC", + "users.name,orders.id\nJohn,105\nJane,104\nJohn,103\nJane,102\nJohn,101", + ) +} + +#[test] +fn join_order_by_with_limit() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users JOIN orders ON users.id = orders.user_id \ + ORDER BY orders.id ASC LIMIT 2", + "users.name,orders.id\nJohn,101\nJane,102", + ) +} + +#[test] +fn order_by_key_absent_from_select_is_rejected() -> Result<(), Box> { + // After aggregation only the projected columns still exist, so ordering by + // something else must be an error, not a silently ignored clause. + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*) FROM employees GROUP BY department ORDER BY salary", + "must appear in the SELECT list", + ) +} + +// --------------------------------------------------------------------------- +// NULL semantics. +// +// The most consequential area for a CSV tool: empty fields are ubiquitous, and +// a comparison against one used to raise "Cannot compare incompatible types: +// Null and Integer(60)" and abort the query. +// --------------------------------------------------------------------------- + +#[test] +fn null_comparison_filters_rather_than_erroring() -> Result<(), Box> { + assert_query( + &["tests/data/nullable.csv"], + "SELECT name FROM nullable WHERE score > 60", + "name\nAlice\nCharlie\nEve", + ) +} + +#[test] +fn null_is_not_equal_to_null() -> Result<(), Box> { + // SQL equality over NULL is UNKNOWN, so no row qualifies -- even the rows + // where both sides are NULL. + assert_query( + &["tests/data/nullable.csv"], + "SELECT id FROM nullable WHERE score = grade", + "id", + ) +} + +#[test] +fn null_excluded_by_negated_comparison_too() -> Result<(), Box> { + // A NULL row satisfies neither `> 60` nor `<= 60`: UNKNOWN is not false. + assert_query( + &["tests/data/nullable.csv"], + "SELECT name FROM nullable WHERE score <= 60", + "name\nNULL", + ) +} + +#[test] +fn aggregates_skip_nulls() -> Result<(), Box> { + // COUNT(*) counts rows; COUNT(col) counts non-NULL values. + assert_query( + &["tests/data/nullable.csv"], + "SELECT COUNT(*), COUNT(score), SUM(score), AVG(score) FROM nullable", + "COUNT,COUNT,SUM,AVG\n5,4,300,75", + ) +} + +#[test] +fn nulls_sort_first() -> Result<(), Box> { + assert_query( + &["tests/data/nullable.csv"], + "SELECT id, score FROM nullable ORDER BY score", + "id,score\n2,NULL\n4,50\n3,70\n1,90\n5,90", + ) +} + +#[test] +fn string_number_coercion_in_comparison() -> Result<(), Box> { + // awk-shaped: a string that parses as a number compares numerically, so + // this agrees with arithmetic rather than silently matching nothing. + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE salary > '60000'", + "name\nAlice\nCharlie\nDavid\nFrank\nGrace", + ) +} + +#[test] +fn group_by_aggregate_before_key() -> Result<(), Box> { + // Values follow the SELECT list, not the internal [keys.., aggs..] layout. + assert_query( + &["tests/data/employees.csv"], + "SELECT COUNT(*), SUM(salary), department FROM employees GROUP BY department", + "COUNT,SUM,department\n\ + 3,210000,Engineering\n\ + 1,45000,HR\n\ + 2,115000,Marketing\n\ + 2,170000,Sales", + ) +} + +#[test] +fn group_by_key_repeated_in_projection() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT department, COUNT(*), department FROM employees GROUP BY department LIMIT 2", + "department,COUNT,department\nEngineering,3,Engineering\nHR,1,HR", + ) +} + +#[test] +fn group_by_non_key_column_is_rejected() -> Result<(), Box> { + // A column that is neither a group key nor inside an aggregate has no + // single value per group; it must be an error rather than an arbitrary pick. + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT department, name FROM employees GROUP BY department", + "must appear in GROUP BY", + ) +} + +// --------------------------------------------------------------------------- +// Join projection order. +// +// Join registers are laid out left-table columns then right-table columns. The +// SELECT list may name them in any order, so these pin the reversed and +// interleaved cases -- including on outer joins, where the NULL-filled row +// must follow the same order. +// --------------------------------------------------------------------------- + +#[test] +fn join_projection_reversed() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT orders.id, users.name FROM users JOIN orders ON users.id = orders.user_id", + "orders.id,users.name\n101,John\n103,John\n105,John\n102,Jane\n104,Jane", + ) +} + +#[test] +fn left_join_reversed_projection_nullfill() -> Result<(), Box> { + assert_query( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT purchases.item, customers.name FROM customers LEFT JOIN purchases \ + ON customers.id = purchases.customer_id", + "purchases.item,customers.name\nBook,Ann\nPen,Ann\nDesk,Ben\nNULL,Cara", + ) +} + +#[test] +fn right_join_reversed_projection_nullfill() -> Result<(), Box> { + assert_query( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT purchases.item, customers.name FROM customers RIGHT JOIN purchases \ + ON customers.id = purchases.customer_id", + "purchases.item,customers.name\nBook,Ann\nPen,Ann\nDesk,Ben\nOrphan,NULL", + ) +} + +#[test] +fn full_join_reversed_projection_nullfill() -> Result<(), Box> { + assert_query( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT purchases.item, customers.name FROM customers FULL JOIN purchases \ + ON customers.id = purchases.customer_id", + "purchases.item,customers.name\nBook,Ann\nPen,Ann\nDesk,Ben\nNULL,Cara\nOrphan,NULL", + ) +} + +// --------------------------------------------------------------------------- +// ORDER BY keys independent of the projection. +// +// The sorter payload is [sort keys.., output columns..], so a key is just a +// value computed per row whether or not it is projected. It previously held +// only the output columns, which forced every key to be a selected column. +// --------------------------------------------------------------------------- + +#[test] +fn order_by_unselected_column() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees ORDER BY salary DESC", + "name\nGrace\nDavid\nFrank\nAlice\nCharlie\nEve\nBob\nHenry", + ) +} + +#[test] +fn order_by_arithmetic_expression() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees ORDER BY salary * -1 LIMIT 3", + "name\nGrace\nDavid\nFrank", + ) +} + +#[test] +fn order_by_function_then_column() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees ORDER BY LENGTH(name) DESC, name ASC LIMIT 3", + "name\nCharlie\nAlice\nDavid", + ) +} + +#[test] +fn order_by_unselected_column_with_where() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees WHERE department = 'Engineering' ORDER BY age DESC", + "name\nCharlie\nFrank\nAlice", + ) +} + +// --------------------------------------------------------------------------- +// Aggregates composed with expressions. +// +// An aggregate argument is an ordinary expression, and after aggregation an +// aggregate result is an ordinary value. Both directions previously failed: +// the argument went through a resolver that yields a column INDEX, and +// detection looked only at the top level so an aggregate inside arithmetic was +// not recognised as an aggregate query at all. +// --------------------------------------------------------------------------- + +#[test] +fn aggregate_over_arithmetic() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT SUM(salary + age) FROM employees", + "SUM\n540257", + ) +} + +#[test] +fn aggregate_over_function() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT MAX(LENGTH(name)), MIN(LENGTH(name)) FROM employees", + "MAX,MIN\n7,3", + ) +} + +#[test] +fn arithmetic_over_aggregate() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT SUM(salary) + 1 FROM employees", + "expr\n540001", + ) +} + +#[test] +fn arithmetic_between_two_aggregates() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT COUNT(*) * 2, MAX(salary) - MIN(salary) FROM employees", + "expr,expr\n16,45000", + ) +} + +#[test] +fn aggregate_mixed_with_plain_projection() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT COUNT(*), SUM(salary) / COUNT(*) FROM employees", + "COUNT,expr\n8,67500", + ) +} + +// --------------------------------------------------------------------------- +// Multi-statement scripts. +// +// Each statement is compiled and executed on its own, so it gets its own +// result set AND observes the database as the previous statement left it. +// --------------------------------------------------------------------------- + +#[test] +fn two_selects_keep_separate_headers() -> Result<(), Box> { + assert_query( + &["tests/data/people.csv"], + "SELECT name FROM people WHERE age > 30; SELECT COUNT(*) FROM people", + "name\nAlice\nCharlie\nCOUNT\n3", + ) +} + +#[test] +fn select_sees_preceding_delete() -> Result<(), Box> { + assert_query( + &["tests/data/people.csv"], + "DELETE FROM people WHERE age < 30; SELECT name FROM people", + "name\nAlice\nCharlie", + ) +} + +#[test] +fn insert_then_select_sees_the_row() -> Result<(), Box> { + assert_query( + &["tests/data/people.csv"], + "INSERT INTO people VALUES (4, 'Dana', 41); SELECT name FROM people WHERE age > 40", + "name\nDana", + ) +} + +#[test] +fn create_table_then_insert_then_select() -> Result<(), Box> { + // The third statement can only compile if the first has already taken + // effect, which is what per-statement execution provides. + assert_query( + &["tests/data/people.csv"], + "CREATE TABLE t (a INTEGER, b TEXT); INSERT INTO t VALUES (1, 'x'); SELECT * FROM t", + "a,b\n1,x", + ) +} + +#[test] +fn update_writeback_preserves_row_order() -> Result<(), Box> { + // UPDATE replaces rows in place. Expressed as delete-plus-insert it + // appended, so a partial update rewrote the user's file in a different + // order. Updating the FIRST row is the case that exposes it. + assert_after_write( + "tests/data/employees.csv", + "employees", + "UPDATE employees SET salary = 1 WHERE id = 1", + "id,name,age,salary,department,role\n\ + 1,Alice,30,1,Engineering,Developer\n\ + 2,Bob,25,55000,Marketing,Specialist\n\ + 3,Charlie,35,65000,Engineering,Manager\n\ + 4,David,40,80000,Sales,Director\n\ + 5,Eve,28,60000,Marketing,Analyst\n\ + 6,Frank,32,75000,Engineering,Developer\n\ + 7,Grace,45,90000,Sales,Director\n\ + 8,Henry,22,45000,HR,Intern", + ) +} + +// --------------------------------------------------------------------------- +// Window frames. +// +// An aggregate window with PARTITION BY and no ORDER BY has the whole +// partition as its frame, so every row sees the partition total. With an +// ORDER BY the frame grows row by row and a running total is correct. The +// streaming implementation produced a running total in both cases. +// --------------------------------------------------------------------------- + +#[test] +fn unordered_window_gives_partition_totals() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, SUM(salary) OVER (PARTITION BY department) FROM employees", + "name,SUM\n\ + Alice,210000\nCharlie,210000\nFrank,210000\n\ + Henry,45000\n\ + Bob,115000\nEve,115000\n\ + David,170000\nGrace,170000", + ) +} + +#[test] +fn ordered_window_keeps_running_totals() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name, SUM(salary) OVER (PARTITION BY department ORDER BY salary) FROM employees", + "name,SUM\n\ + Charlie,65000\nAlice,135000\nFrank,210000\n\ + Henry,45000\n\ + Bob,55000\nEve,115000\n\ + David,80000\nGrace,170000", + ) +} + +#[test] +fn multiple_unordered_window_aggregates() -> Result<(), Box> { + // Every aggregate window column is finalized, not just the first. + assert_query( + &["tests/data/employees.csv"], + "SELECT name, COUNT(salary) OVER (PARTITION BY department), \ + MIN(salary) OVER (PARTITION BY department) FROM employees", + "name,COUNT,MIN\n\ + Alice,3,65000\nCharlie,3,65000\nFrank,3,65000\n\ + Henry,1,45000\n\ + Bob,2,55000\nEve,2,55000\n\ + David,2,80000\nGrace,2,80000", + ) +} + +// --------------------------------------------------------------------------- +// Aggregates over joins. +// +// `A INNER JOIN B ON c` and `FROM A, B WHERE c` mean the same thing, and the +// comma form already supported aggregates, so the explicit form is rewritten +// to it. The rewrite is deliberately NOT applied to outer joins: moving the ON +// condition into WHERE would discard exactly the NULL-extended rows an outer +// join exists to produce. +// --------------------------------------------------------------------------- + +#[test] +fn count_over_inner_join() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT COUNT(*) FROM users JOIN orders ON users.id = orders.user_id", + "COUNT\n5", + ) +} + +#[test] +fn group_by_over_inner_join() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, COUNT(*) FROM users JOIN orders ON users.id = orders.user_id \ + GROUP BY users.name", + "users.name,COUNT\nJane,2\nJohn,3", + ) +} + +#[test] +fn inner_join_and_comma_join_agree() -> Result<(), Box> { + // The rewrite's premise, pinned directly. + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT COUNT(*) FROM users, orders WHERE users.id = orders.user_id", + "COUNT\n5", + ) +} + +#[test] +fn outer_join_aggregate_is_not_silently_rewritten() -> Result<(), Box> { + // Must not answer as though it were an inner join: Cara has no purchase, + // so a LEFT JOIN has 4 rows where the rewritten inner form has 3. Erroring + // is acceptable; a wrong number is not. + assert_query_fails( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT COUNT(*) FROM customers LEFT JOIN purchases ON customers.id = purchases.customer_id", + "Unsupported", + ) +} + +// --------------------------------------------------------------------------- +// Derived tables. +// +// The compiler only knows how to open a cursor on a named table, so a subquery +// in FROM is executed first and registered under its alias; the outer query +// then sees an ordinary table. The registration lasts only for the statement. +// --------------------------------------------------------------------------- + +#[test] +fn derived_table_filters() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM (SELECT name, salary FROM employees WHERE salary > 70000) t", + "name\nDavid\nFrank\nGrace", + ) +} + +#[test] +fn aggregate_over_derived_table() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT COUNT(*) FROM (SELECT name FROM employees WHERE department = 'Engineering') t", + "COUNT\n3", + ) +} + +#[test] +fn derived_table_over_group_by() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT * FROM (SELECT department, COUNT(*) FROM employees GROUP BY department) t", + "department,COUNT\nEngineering,3\nHR,1\nMarketing,2\nSales,2", + ) +} + +#[test] +fn derived_table_exposes_inner_aliases() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT n FROM (SELECT name AS n FROM employees LIMIT 2) t", + "n\nAlice\nBob", + ) +} + +#[test] +fn nested_derived_tables() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT x FROM (SELECT name AS x FROM (SELECT name FROM employees LIMIT 3) inner1) t", + "x\nAlice\nBob\nCharlie", + ) +} + +#[test] +fn derived_table_alias_may_not_shadow_a_real_table() -> Result<(), Box> { + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT name FROM (SELECT name FROM employees) employees", + "shadows an existing table", + ) +} + +#[test] +fn derived_table_does_not_outlive_its_statement() -> Result<(), Box> { + // The alias is registered only for the statement that declares it, so a + // later statement must not see it. + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT name FROM (SELECT name FROM employees) t; SELECT name FROM t", + "not found", + ) +} + +// --------------------------------------------------------------------------- +// Compound JOIN conditions. +// +// The ON clause used to have its own small expression compiler that understood +// a single top-level comparison, so `ON a = b AND c` was rejected even though +// the same expression works in a WHERE. Join conditions now go through the +// shared expression compiler with a multi-source name context. +// --------------------------------------------------------------------------- + +#[test] +fn join_on_with_and() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users JOIN orders \ + ON users.id = orders.user_id AND orders.id > 102", + "users.name,orders.id\nJohn,103\nJohn,105\nJane,104", + ) +} + +#[test] +fn join_on_with_or() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users JOIN orders \ + ON users.id = orders.user_id OR orders.id = 999", + "users.name,orders.id\nJohn,101\nJohn,103\nJohn,105\nJane,102\nJane,104", + ) +} + +#[test] +fn left_join_with_compound_on_keeps_nullfill() -> Result<(), Box> { + // The extra ON predicate narrows the match without turning the outer join + // into an inner one: Cara still appears with NULLs. + assert_query( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT customers.name, purchases.item FROM customers LEFT JOIN purchases \ + ON customers.id = purchases.customer_id AND purchases.id > 10", + "customers.name,purchases.item\nAnn,Pen\nBen,Desk\nCara,NULL", + ) +} + +// --------------------------------------------------------------------------- +// Multi-table WHERE expressiveness. +// +// The comma-join WHERE had its own condition compiler that understood only +// AND, OR and the six comparisons over qualified columns and literals. It now +// shares the expression compiler, with one name-resolution source per table. +// --------------------------------------------------------------------------- + +#[test] +fn function_in_multi_table_where() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users, orders \ + WHERE users.id = orders.user_id AND UPPER(users.name) = 'JOHN'", + "users.name,orders.id\nJohn,101\nJohn,103\nJohn,105", + ) +} + +#[test] +fn arithmetic_in_multi_table_where() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users, orders \ + WHERE users.id = orders.user_id AND orders.id - 100 > 3", + "users.name,orders.id\nJohn,105\nJane,104", + ) +} + +#[test] +fn not_in_multi_table_where() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users, orders \ + WHERE users.id = orders.user_id AND NOT (orders.id = 101)", + "users.name,orders.id\nJohn,103\nJohn,105\nJane,102\nJane,104", + ) +} + +#[test] +fn unqualified_column_in_multi_table_where() -> Result<(), Box> { + // `date` exists only in orders, so it resolves without a qualifier. + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users, orders \ + WHERE users.id = orders.user_id AND date = '2023-01-15'", + "users.name,orders.id\nJohn,101", + ) +} + +#[test] +fn function_in_join_on_clause() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT users.name, orders.id FROM users JOIN orders \ + ON users.id = orders.user_id AND UPPER(users.name) = 'JANE'", + "users.name,orders.id\nJane,102\nJane,104", + ) +} + +#[test] +fn three_table_join_with_compound_on() -> Result<(), Box> { + // Multi-table joins load every column into registers before evaluating + // join conditions, so a column reference there is a register offset rather + // than a cursor read. Both addressing modes now go through one expression + // compiler, which is what lets this ON clause carry more than a single + // comparison. + assert_query( + &[ + "tests/data/users.csv", + "tests/data/orders.csv", + "tests/data/products.csv", + ], + "SELECT users.name, products.name FROM users \ + JOIN orders ON users.id = orders.user_id \ + JOIN products ON orders.product_id = products.product_id AND products.price > 200", + "users.name,products.name\nJohn,Laptop\nJane,Phone\nJane,Monitor", + ) +} + +#[test] +fn three_table_join_unfiltered() -> Result<(), Box> { + assert_query( + &[ + "tests/data/users.csv", + "tests/data/orders.csv", + "tests/data/products.csv", + ], + "SELECT users.name, products.name, products.price FROM users \ + JOIN orders ON users.id = orders.user_id \ + JOIN products ON orders.product_id = products.product_id", + "users.name,products.name,products.price\n\ + John,Laptop,1200\nJohn,Headphones,150\nJohn,Keyboard,80\n\ + Jane,Phone,800\nJane,Monitor,350", + ) +} + +// --------------------------------------------------------------------------- +// Correlated subquery scoping. +// +// A name binds in the subquery's scope first and only then in the enclosing +// query's. Function arguments must follow the same rule: resolving them in a +// rebuilt inner-only scope makes an outer reference silently bind to the inner +// table, which turns a comparison between the two into a comparison of a value +// with itself -- true for every row. +// --------------------------------------------------------------------------- + +#[test] +fn correlated_subquery_scalar() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees e \ + WHERE salary > (SELECT AVG(salary) FROM employees WHERE department = e.department)", + "name\nEve\nFrank\nGrace", + ) +} + +#[test] +fn correlated_subquery_exists() -> Result<(), Box> { + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees e WHERE EXISTS \ + (SELECT 1 FROM employees x WHERE x.department = e.department AND x.age > 40)", + "name\nDavid\nGrace", + ) +} + +#[test] +fn correlated_subquery_with_function_on_outer_column() -> Result<(), Box> { + // Must agree with the unwrapped form above. Wrapping both sides in a + // function may not change which rows qualify. + assert_query( + &["tests/data/employees.csv"], + "SELECT name FROM employees e WHERE EXISTS \ + (SELECT 1 FROM employees x \ + WHERE UPPER(x.department) = UPPER(e.department) AND x.age > 40)", + "name\nDavid\nGrace", + ) +} + +// --------------------------------------------------------------------------- +// Table aliases in joins. +// +// Aliases worked for a single table and for comma joins, but explicit joins +// resolved column qualifiers against the table name alone, so every aliased +// explicit join failed with "Table 'u' not found" -- including all three join +// examples in the user guide. No test covered an aliased join at all, which is +// also how a later refactor came to pass aliases where table names were +// expected without anything failing. +// --------------------------------------------------------------------------- + +#[test] +fn alias_in_inner_join() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT u.name, o.id FROM users u JOIN orders o ON u.id = o.user_id", + "u.name,o.id\nJohn,101\nJohn,103\nJohn,105\nJane,102\nJane,104", + ) +} + +#[test] +fn alias_in_left_join() -> Result<(), Box> { + assert_query( + &["tests/data/customers.csv", "tests/data/purchases.csv"], + "SELECT c.name, p.item FROM customers c LEFT JOIN purchases p ON c.id = p.customer_id", + "c.name,p.item\nAnn,Book\nAnn,Pen\nBen,Desk\nCara,NULL", + ) +} + +#[test] +fn alias_in_three_table_join() -> Result<(), Box> { + assert_query( + &[ + "tests/data/users.csv", + "tests/data/orders.csv", + "tests/data/products.csv", + ], + "SELECT u.name, p.name FROM users u \ + JOIN orders o ON u.id = o.user_id \ + JOIN products p ON o.product_id = p.product_id", + "u.name,p.name\nJohn,Laptop\nJohn,Headphones\nJohn,Keyboard\nJane,Phone\nJane,Monitor", + ) +} + +#[test] +fn alias_in_implicit_join() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT u.name, o.id FROM users u, orders o WHERE u.id = o.user_id", + "u.name,o.id\nJohn,101\nJohn,103\nJohn,105\nJane,102\nJane,104", + ) +} + +#[test] +fn explicit_window_frame_is_rejected() -> Result<(), Box> { + // The frame clause was parsed and then ignored, so this returned the + // default frame's answer -- a running total over the whole partition + // rather than a two-row sliding window. Rejecting it is correct until + // frames are implemented: a clear error beats a plausible wrong number. + assert_query_fails( + &["tests/data/employees.csv"], + "SELECT name, SUM(salary) OVER (ORDER BY salary ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) \ + FROM employees", + "Explicit window frames", + ) +} + +// --------------------------------------------------------------------------- +// ORDER BY / LIMIT over multi-table queries. +// +// Both were silently dropped on grouped multi-table queries, and ORDER BY on +// ungrouped comma joins. The scan there applies LIMIT as it goes, which is +// only correct without ORDER BY -- otherwise it keeps the first N rows and +// sorts those, rather than the top N. +// --------------------------------------------------------------------------- + +#[test] +fn grouped_join_order_by_and_limit() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT u.name, COUNT(*) AS n FROM users u JOIN orders o ON u.id = o.user_id \ + GROUP BY u.name ORDER BY n DESC LIMIT 1", + "u.name,n\nJohn,3", + ) +} + +#[test] +fn implicit_join_order_by_takes_top_rows() -> Result<(), Box> { + // The two highest order ids are 105 and 104. A LIMIT applied before the + // sort would return 101 and 102 instead. + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT u.name, o.id FROM users u, orders o WHERE u.id = o.user_id \ + ORDER BY o.id DESC LIMIT 2", + "u.name,o.id\nJohn,105\nJane,104", + ) +} + +#[test] +fn implicit_join_order_by_without_limit() -> Result<(), Box> { + assert_query( + &["tests/data/users.csv", "tests/data/orders.csv"], + "SELECT u.name, o.id FROM users u, orders o WHERE u.id = o.user_id ORDER BY o.id DESC", + "u.name,o.id\nJohn,105\nJane,104\nJohn,103\nJane,102\nJohn,101", + ) +} diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index 26f96ec..940abd1 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -139,6 +139,169 @@ fn run_test_case_with_file( Ok(()) } +// --------------------------------------------------------------------------- +// Exact-output helpers +// +// The `contains`-based assertions above are substring matches: a test expecting +// "Engineering,3" still passes when three extra wrong rows are emitted, and +// passes regardless of row order. Every defect in the step-0 red list survived +// the suite for exactly that reason. +// +// The helpers below assert the *complete* stdout -- every row, in order. Use +// these for new tests. +// --------------------------------------------------------------------------- + +/// Run `sql` against `files` and return (stdout, stderr, success) verbatim. +/// +/// Each entry in `files` is passed to sqawk as-is, so both `path.csv` and +/// `name=path.csv` forms work. +pub fn run_sql( + sql: &str, + files: &[&str], + extra_args: &[&str], +) -> Result<(String, String, bool), Box> { + let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); + cmd.arg("-s").arg(sql); + for arg in extra_args { + cmd.arg(arg); + } + for file in files { + cmd.arg(file); + } + + let output = cmd.output()?; + Ok(( + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + output.status.success(), + )) +} + +/// Assert that `sql` over `files` produces exactly `expected` on stdout. +/// +/// Both sides are trimmed of leading/trailing blank lines and of trailing +/// whitespace on each line, so tests can use indented raw strings. Everything +/// else -- row content, column order, row order, row count -- must match. +pub fn assert_query( + files: &[&str], + sql: &str, + expected: &str, +) -> Result<(), Box> { + let (stdout, stderr, success) = run_sql(sql, files, &[])?; + assert!( + success, + "query failed unexpectedly\n sql: {sql}\n stderr: {stderr}" + ); + + let actual = normalize_output(&stdout); + let want = normalize_output(expected); + assert_eq!( + actual, want, + "\n=== query ===\n{sql}\n=== expected ===\n{want}\n=== actual ===\n{actual}\n" + ); + Ok(()) +} + +/// Assert that `sql` over `files` fails, with `expected_err` appearing in stderr. +pub fn assert_query_fails( + files: &[&str], + sql: &str, + expected_err: &str, +) -> Result<(), Box> { + let (stdout, stderr, success) = run_sql(sql, files, &[])?; + assert!( + !success, + "query unexpectedly succeeded\n sql: {sql}\n stdout: {stdout}" + ); + assert!( + stderr.contains(expected_err), + "\n=== query ===\n{sql}\n=== expected stderr to contain ===\n{expected_err}\n=== actual stderr ===\n{stderr}\n" + ); + Ok(()) +} + +/// Trim trailing whitespace per line and drop leading/trailing blank lines. +fn normalize_output(s: &str) -> String { + s.lines() + .map(|l| l.trim_end()) + .collect::>() + .join("\n") + .trim_matches('\n') + .to_string() +} + +/// Run `sql` with `--write` against a private copy of `fixture`, and return the +/// resulting file contents. +/// +/// Needed because modifications are in-memory only without `--write`, and +/// `UPDATE ...; SELECT ...` in a single invocation does not currently emit the +/// SELECT's rows -- so writeback is the way to observe what a DML statement +/// actually did. +pub fn run_with_write( + fixture: &str, + table: &str, + sql: &str, +) -> Result> { + let dir = create_temp_dir()?; + let src = PathBuf::from(fixture); + let dest = dir.path().join(src.file_name().unwrap()); + fs::copy(&src, &dest)?; + + let spec = format!("{}={}", table, dest.to_str().unwrap()); + let (_stdout, stderr, success) = run_sql(sql, &[&spec], &["--write"])?; + assert!( + success, + "write query failed\n sql: {sql}\n stderr: {stderr}" + ); + + Ok(normalize_output(&fs::read_to_string(&dest)?)) +} + +/// Assert that `sql` run with `--write` leaves `fixture` with exactly +/// `expected` contents. +pub fn assert_after_write( + fixture: &str, + table: &str, + sql: &str, + expected: &str, +) -> Result<(), Box> { + let actual = run_with_write(fixture, table, sql)?; + let want = normalize_output(expected); + assert_eq!( + actual, want, + "\n=== sql ===\n{sql}\n=== expected file ===\n{want}\n=== actual file ===\n{actual}\n" + ); + Ok(()) +} + +/// Path to the nullable.csv fixture (contains empty fields -> NULL values). +/// +/// No other fixture in tests/data/ has an empty field, so nothing in the suite +/// currently exercises a NULL flowing into a comparison. +pub fn nullable_csv() -> &'static str { + "tests/data/nullable.csv" +} + +/// Path to employees.csv as a plain &str, for the exact-output helpers. +pub fn employees_csv() -> &'static str { + "tests/data/employees.csv" +} + +/// Path to departments.csv as a plain &str, for the exact-output helpers. +pub fn departments_csv() -> &'static str { + "tests/data/departments.csv" +} + +/// Path to users.csv as a plain &str, for the exact-output helpers. +pub fn users_csv() -> &'static str { + "tests/data/users.csv" +} + +/// Path to orders.csv as a plain &str, for the exact-output helpers. +pub fn orders_csv() -> &'static str { + "tests/data/orders.csv" +} + /// Helper function to create a temp directory for tests, respecting CARGO_TARGET_TMPDIR if set pub fn create_temp_dir() -> Result> { if let Ok(cargo_target_tmpdir) = env::var("CARGO_TARGET_TMPDIR") { diff --git a/tests/mod.rs b/tests/mod.rs index 30f518c..3052f88 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -64,6 +64,11 @@ mod output; // Tests for stdout and file output (atomic writes) mod repl; // Tests for interactive REPL functionality with pre-generated input mod string_functions; // Tests for string functions (UPPER, LOWER, TRIM, SUBSTR, REPLACE) +// Audit (2026-08): correctness work +mod defects; // Red list -- #[ignore]d tests encoding CORRECT behaviour for known defects +mod golden; // Characterization tests +mod tsq; // The tsq test-data generator binary + // Support modules mod helpers; // Test helpers and utilities diff --git a/tests/repl/commands.rs b/tests/repl/commands.rs index 36752c6..65f4a97 100644 --- a/tests/repl/commands.rs +++ b/tests/repl/commands.rs @@ -120,3 +120,89 @@ fn test_repl_stats_command() { let status = process.wait().expect("Failed to wait for sqawk process"); assert!(status.success(), "Process did not exit successfully"); } + +/// `.version` must report the crate's actual version. +/// +/// It was hardcoded to "0.1.1" while the crate was at 0.8.0, and nothing +/// noticed because nothing asserted on it. Reading it from the manifest makes +/// drift impossible; this test guards the reporting path itself. +#[test] +fn test_repl_version_matches_crate() { + use std::io::Write; + use std::process::{Command, Stdio}; + + let mut child = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .arg("-i") + .arg("tests/data/employees.csv") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to start sqawk"); + + child + .stdin + .as_mut() + .expect("stdin") + .write_all(b".version\n.exit\n") + .expect("write"); + + let out = child.wait_with_output().expect("wait"); + let stdout = String::from_utf8_lossy(&out.stdout); + let expected = format!("Sqawk version {}", env!("CARGO_PKG_VERSION")); + assert!( + stdout.contains(&expected), + "expected `{expected}` in .version output, got:\n{stdout}" + ); +} + +/// A DML statement that matches nothing must report 0, not inherit the +/// previous statement's count. +/// +/// The affected-row count was only overwritten when the new count was non-zero, +/// which conflated "this statement was not DML" with "this DML matched no +/// rows". So `UPDATE ; UPDATE ` reported 3. +#[test] +fn test_repl_zero_row_update_reports_zero() { + use std::io::Write; + use std::process::{Command, Stdio}; + + let dir = crate::helpers::create_temp_dir().expect("temp dir"); + let src = std::path::PathBuf::from("tests/data/employees.csv"); + let dest = dir.path().join("employees.csv"); + std::fs::copy(&src, &dest).expect("copy fixture"); + + let mut child = Command::new(env!("CARGO_BIN_EXE_sqawk")) + .arg("-i") + .arg(format!("employees={}", dest.to_str().unwrap())) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to start sqawk"); + + child + .stdin + .as_mut() + .expect("stdin") + .write_all( + b".changes on\n\ + UPDATE employees SET salary = 1 WHERE department = 'Engineering';\n\ + UPDATE employees SET salary = 2 WHERE department = 'NoSuchDept';\n\ + .exit\n", + ) + .expect("write"); + + let out = child.wait_with_output().expect("wait"); + let stdout = String::from_utf8_lossy(&out.stdout); + let counts: Vec<&str> = stdout + .lines() + .filter(|l| l.ends_with("rows affected")) + .collect(); + + assert_eq!( + counts, + ["3 rows affected", "0 rows affected"], + "second UPDATE matched nothing and must report 0, got:\n{stdout}" + ); +} diff --git a/tests/repl/test_repl_from_file.rs b/tests/repl/test_repl_from_file.rs index 92a2e14..f83883d 100644 --- a/tests/repl/test_repl_from_file.rs +++ b/tests/repl/test_repl_from_file.rs @@ -2,12 +2,12 @@ fn test_repl_from_file() { // Removed unused temporary directory - // Run sqawk with input from repl_commands.txt in the tmp directory + // Run sqawk with a canned REPL session from tests/data/ let mut cmd = assert_cmd::cargo::cargo_bin_cmd!("sqawk"); let assert = cmd .arg("--interactive") .arg("tests/data/sample.csv") - .pipe_stdin("tests/tmp/test_repl_commands.txt") + .pipe_stdin("tests/data/repl_commands.txt") .unwrap() .assert() .success(); diff --git a/tests/tmp/README.md b/tests/tmp/README.md deleted file mode 100644 index e03314c..0000000 --- a/tests/tmp/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Temporary Test Files Directory - -This directory contains temporary files used for testing purposes. Any test-generated data or test input files should be placed here rather than in the root directory of the codebase. - -The current files in this directory: - -- `test_changes_command.txt` - Test data for the `.changes` REPL command -- `test_exit_code.txt` - Test data for the exit code feature of the `.exit` command -- `test_pattern_matching.txt` - Test data for pattern matching in the `.tables` command -- `test_repl_commands.txt` - Combined test data for multiple REPL commands - -When adding new test files, please follow the naming convention of prefixing with `test_` and adding a descriptive name. \ No newline at end of file diff --git a/tests/tmp/create_table.sql b/tests/tmp/create_table.sql deleted file mode 100644 index 028d531..0000000 --- a/tests/tmp/create_table.sql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE TABLE test_table (id INT, name TEXT) LOCATION './test_output.csv' STORED AS TEXTFILE WITH (DELIMITER=','); -.tables diff --git a/tests/tmp/debug_create_table.rs b/tests/tmp/debug_create_table.rs deleted file mode 100644 index 7631fec..0000000 --- a/tests/tmp/debug_create_table.rs +++ /dev/null @@ -1,33 +0,0 @@ -use sqlparser::dialect::GenericDialect; -use sqlparser::parser::Parser; - -fn main() { - let sql = "CREATE TABLE test_table (id INT, name TEXT) LOCATION './test_output.csv' STORED AS TEXTFILE WITH (DELIMITER=',');"; - println!("Testing SQL: {}", sql); - - let dialect = GenericDialect {}; // First try with generic dialect - - match Parser::parse_sql(&dialect, sql) { - Ok(ast) => { - println!("Successfully parsed with GenericDialect!"); - println!("AST: {:#?}", ast); - }, - Err(e) => { - println!("Failed to parse with GenericDialect: {}", e); - } - } - - // Also try with HiveDialect which is known to support LOCATION - use sqlparser::dialect::HiveDialect; - let hive_dialect = HiveDialect {}; - - match Parser::parse_sql(&hive_dialect, sql) { - Ok(ast) => { - println!("Successfully parsed with HiveDialect!"); - println!("AST: {:#?}", ast); - }, - Err(e) => { - println!("Failed to parse with HiveDialect: {}", e); - } - } -} diff --git a/tests/tmp/debug_sql.rs b/tests/tmp/debug_sql.rs deleted file mode 100644 index 941a1bd..0000000 --- a/tests/tmp/debug_sql.rs +++ /dev/null @@ -1,17 +0,0 @@ -use sqlparser::dialect::GenericDialect; -use sqlparser::parser::Parser; - -fn main() { - let sql = "CREATE TABLE test_table (id INT, name TEXT) LOCATION './test_output.csv' STORED AS TEXTFILE WITH (DELIMITER=',');"; - println!("Testing SQL: {}", sql); - - let dialect = GenericDialect {}; - match Parser::parse_sql(&dialect, sql) { - Ok(ast) => { - println!("Parsed AST: {:#?}", ast); - }, - Err(e) => { - println!("Error parsing SQL: {}", e); - } - } -} diff --git a/tests/tmp/dialect_test.rs b/tests/tmp/dialect_test.rs deleted file mode 100644 index 75c95d2..0000000 --- a/tests/tmp/dialect_test.rs +++ /dev/null @@ -1,4 +0,0 @@ -fn main() { - // Will be filled by running the actual sqawk binary with debug output - println!("Testing SQL parser dialects via sqawk..."); -} diff --git a/tests/tmp/test-dialects.rs b/tests/tmp/test-dialects.rs deleted file mode 100644 index 61fb6ff..0000000 --- a/tests/tmp/test-dialects.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Test different SQL dialects for CREATE TABLE with LOCATION -extern crate sqlparser; - -use sqlparser::dialect::{GenericDialect, HiveDialect, SnowflakeDialect}; -use sqlparser::parser::Parser; - -fn main() { - let sql = "CREATE TABLE test_table (id INT, name TEXT) LOCATION './test_output.csv' STORED AS TEXTFILE WITH (DELIMITER=',');"; - println!("Testing CREATE TABLE with LOCATION:"); - println!("{}\n", sql); - - // Try different dialects - let dialects = [ - ("Generic", GenericDialect {}), - ("Hive", HiveDialect {}), - ("Snowflake", SnowflakeDialect {}) - ]; - - for (name, dialect) in dialects.iter() { - println!("=== {} Dialect ===", name); - match Parser::parse_sql(dialect, sql) { - Ok(statements) => { - println!(" Parsed successfully!"); - if let Some(stmt) = statements.first() { - println!(" Statement type: {:?}", std::mem::discriminant(stmt)); - - use sqlparser::ast::Statement; - if let Statement::CreateTable { location, .. } = stmt { - println!(" LOCATION clause: {:?}", location); - } else { - println!(" Not a CREATE TABLE statement!"); - } - } - }, - Err(e) => println!(" Parse error: {}", e) - } - println!(); - } -} diff --git a/tests/tmp/test_changes_command.txt b/tests/tmp/test_changes_command.txt deleted file mode 100644 index cc2b85a..0000000 --- a/tests/tmp/test_changes_command.txt +++ /dev/null @@ -1,7 +0,0 @@ -.changes on -INSERT INTO sample VALUES (4, 'David', 40); -UPDATE sample SET age = 33 WHERE name = 'Alice'; -DELETE FROM sample WHERE name = 'Bob'; -.changes off -INSERT INTO sample VALUES (5, 'Eve', 28); -.exit \ No newline at end of file diff --git a/tests/tmp/test_data.csv b/tests/tmp/test_data.csv deleted file mode 100644 index d1e9a87..0000000 --- a/tests/tmp/test_data.csv +++ /dev/null @@ -1,4 +0,0 @@ -id,name,age,value -1,John,30,10.5 -2,Jane,25,20.75 -3,Bob,40,30.25 diff --git a/tests/tmp/test_exit_code.txt b/tests/tmp/test_exit_code.txt deleted file mode 100644 index d50e0ba..0000000 --- a/tests/tmp/test_exit_code.txt +++ /dev/null @@ -1,4 +0,0 @@ -.changes on -INSERT INTO sample VALUES (4, 'David', 40); -UPDATE sample SET age = 33 WHERE name = 'Alice'; -.exit 42 \ No newline at end of file diff --git a/tests/tmp/test_input.txt b/tests/tmp/test_input.txt deleted file mode 100644 index d5a6f71..0000000 --- a/tests/tmp/test_input.txt +++ /dev/null @@ -1,3 +0,0 @@ -CREATE TABLE test_types (id INTEGER, name TEXT, price FLOAT, active BOOLEAN) LOCATION './test_types.csv' STORED AS TEXTFILE WITH (DELIMITER=','); -.schema test_types -.exit diff --git a/tests/tmp/test_output.csv b/tests/tmp/test_output.csv deleted file mode 100644 index c4b4829..0000000 --- a/tests/tmp/test_output.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,name -1,Test -2,Example diff --git a/tests/tmp/test_pattern_matching.txt b/tests/tmp/test_pattern_matching.txt deleted file mode 100644 index 860a7b5..0000000 --- a/tests/tmp/test_pattern_matching.txt +++ /dev/null @@ -1,5 +0,0 @@ -.load tests/data/animals.csv -.tables -.tables sa -.tables an -.exit \ No newline at end of file diff --git a/tests/tmp/test_repl_commands.txt b/tests/tmp/test_repl_commands.txt deleted file mode 100644 index aa3e960..0000000 --- a/tests/tmp/test_repl_commands.txt +++ /dev/null @@ -1,8 +0,0 @@ -.tables -.tables s% -SELECT * FROM sample; -.changes on -INSERT INTO sample VALUES (4, 'David', 28); -.changes -.version -.exit \ No newline at end of file diff --git a/tests/tmp/test_repl_schema.sh b/tests/tmp/test_repl_schema.sh deleted file mode 100755 index dad512b..0000000 --- a/tests/tmp/test_repl_schema.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Create a file with input commands for the REPL -cat > test_input.txt << 'END' -CREATE TABLE test_types (id INTEGER, name TEXT, price FLOAT, active BOOLEAN) LOCATION './test_types.csv' STORED AS TEXTFILE WITH (DELIMITER=','); -.schema test_types -.exit -END - -# Run sqawk in interactive mode with these commands -cat test_input.txt | cargo run -- tests/data/sample.csv --interactive diff --git a/tests/tmp/test_schema_display.sql b/tests/tmp/test_schema_display.sql deleted file mode 100644 index 57210e4..0000000 --- a/tests/tmp/test_schema_display.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE TABLE typed_table ( - id INTEGER, - name TEXT, - price FLOAT, - active BOOLEAN -) LOCATION './typed_table.csv' STORED AS TEXTFILE WITH (DELIMITER=','); - -INSERT INTO typed_table VALUES - (1, 'Widget', 19.99, true), - (2, 'Gadget', 24.95, false), - (3, 'Doohickey', 14.50, true); diff --git a/tests/tmp/test_script.sh b/tests/tmp/test_script.sh deleted file mode 100755 index 57ee7cd..0000000 --- a/tests/tmp/test_script.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -# Start sqawk in interactive mode with a test file -echo "CREATE TABLE test_table (id INT, name TEXT, value FLOAT) LOCATION './test_output.csv' STORED AS TEXTFILE WITH (DELIMITER=',');" | cargo run -- tests/data/sample.csv --interactive diff --git a/tests/tsq/mod.rs b/tests/tsq/mod.rs new file mode 100644 index 0000000..c547ca5 --- /dev/null +++ b/tests/tsq/mod.rs @@ -0,0 +1,155 @@ +//! Tests for the `tsq` test-data generator. +//! +//! `tsq` is a second binary installed by `cargo install sqawk`, so it is part +//! of the shipped surface. It had no tests at all; these cover the properties +//! that make it useful -- determinism, referential integrity of the generated +//! tables, and that sqawk can actually read what it writes. + +use crate::helpers::create_temp_dir; +use std::fs; +use std::path::Path; + +/// Run tsq into a fresh directory and return that directory. +fn generate(seed: u64, rows: usize) -> Result> { + let dir = create_temp_dir()?; + let out = dir.path().join("out"); + let output = assert_cmd::cargo::cargo_bin_cmd!("tsq") + .arg("--seed") + .arg(seed.to_string()) + .arg("--rows") + .arg(rows.to_string()) + .arg("--output-dir") + .arg(out.to_str().unwrap()) + .output()?; + assert!( + output.status.success(), + "tsq failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(dir) +} + +fn read(dir: &Path, rel: &str) -> String { + fs::read_to_string(dir.join("out").join(rel)).unwrap_or_else(|e| panic!("missing {rel}: {e}")) +} + +#[test] +fn generates_the_expected_tree() -> Result<(), Box> { + let dir = generate(1, 20)?; + let base = dir.path().join("out"); + for rel in [ + "metadata.json", + "data/customers.csv", + "data/products.csv", + "data/orders.csv", + "data/order_items.csv", + "data/reviews.csv", + "verify/run_verification.sh", + ] { + assert!(base.join(rel).exists(), "expected {rel} to be generated"); + } + assert!( + base.join("queries").read_dir()?.count() > 0, + "expected generated queries" + ); + Ok(()) +} + +#[test] +fn same_seed_produces_identical_data() -> Result<(), Box> { + // The whole point of the seed: a run is reproducible. + let a = generate(1234, 25)?; + let b = generate(1234, 25)?; + for rel in ["data/customers.csv", "data/orders.csv", "data/products.csv"] { + assert_eq!( + read(a.path(), rel), + read(b.path(), rel), + "{rel} differed between two runs with the same seed" + ); + } + Ok(()) +} + +#[test] +fn different_seeds_produce_different_data() -> Result<(), Box> { + let a = generate(1, 25)?; + let b = generate(2, 25)?; + assert_ne!( + read(a.path(), "data/customers.csv"), + read(b.path(), "data/customers.csv"), + "different seeds should not produce identical data" + ); + Ok(()) +} + +#[test] +fn metadata_records_the_seed_and_row_counts() -> Result<(), Box> { + let dir = generate(99, 30)?; + let meta = read(dir.path(), "metadata.json"); + assert!(meta.contains("\"seed\": 99"), "seed not recorded: {meta}"); + assert!( + meta.contains("\"customers\": 30"), + "row count not recorded: {meta}" + ); + Ok(()) +} + +#[test] +fn sqawk_can_query_the_generated_data() -> Result<(), Box> { + // The data exists to be fed to sqawk, so the two must agree on format. + let dir = generate(7, 40)?; + let customers = dir.path().join("out/data/customers.csv"); + + let output = assert_cmd::cargo::cargo_bin_cmd!("sqawk") + .arg("-s") + .arg("SELECT COUNT(*) FROM customers") + .arg(customers.to_str().unwrap()) + .output()?; + assert!( + output.status.success(), + "sqawk could not read tsq output: {}", + String::from_utf8_lossy(&output.stderr) + ); + // Exact output, not a substring: `contains("40")` also passes on 140. + let stdout = String::from_utf8_lossy(&output.stdout); + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!( + lines, + ["COUNT", "40"], + "expected exactly 40 customers, got: {stdout}" + ); + Ok(()) +} + +#[test] +fn generated_orders_reference_real_customers() -> Result<(), Box> { + // Referential integrity is what makes the join queries meaningful. + let dir = generate(11, 50)?; + let base = dir.path().join("out/data"); + + let output = assert_cmd::cargo::cargo_bin_cmd!("sqawk") + .arg("-s") + // Rows are counted here rather than with COUNT(*), which is not yet + // supported over a join -- see defects::aggregate_over_join. + .arg( + "SELECT orders.order_id FROM orders JOIN customers \ + ON orders.customer_id = customers.customer_id", + ) + .arg(base.join("orders.csv").to_str().unwrap()) + .arg(base.join("customers.csv").to_str().unwrap()) + .output()?; + assert!( + output.status.success(), + "join over generated data failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + // Every order must match a customer, so the join count equals the row count. + let joined = String::from_utf8_lossy(&output.stdout).lines().count() - 1; + let orders = read(dir.path(), "data/orders.csv").lines().count() - 1; + assert_eq!( + joined, orders, + "every generated order should reference an existing customer" + ); + Ok(()) +}