From a06e7d7f0a28f86ac6c9aab85f60105226ac7042 Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh <18012955+ajitpratap0@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:53:30 +0530 Subject: [PATCH 1/4] perf(fingerprint,formatter,transform): tokenize into the pooled scratch buffer [W5-01] Normalize/Fingerprint, Formatter.Format and transform's parse helpers called the public TokenizeString, which copies every token into a fresh slice that the parser immediately copies again. They now use internal/tokscratch, the path gosqlx.Parse already uses: tokens live in the pooled tokenizer's buffer for the duration of the call. Each caller finishes with the tokens before the deferred PutTokenizer, the parser copies what the AST needs, and token strings are substrings of the input, so nothing borrowed outlives the call. Outputs are byte-identical. Tests: differential tests against the public tokenizer path over every statement in the real-world corpora; concurrency tests (8+ goroutines, -race) checked against a sequential run; pool-reuse tests that returned strings, ASTs, conditions and joins stay correct after later calls reuse the buffer. M4, GOGC=off -cpu 1, 10 interleaved rounds, the 11 compare queries: Normalize B/op -42% to -70%, Fingerprint -40% to -65%, TransformParse -49% to -78% (geomean -54% / -63%), allocs -5% to -29%. Time: no regression; the shared machine was too noisy for a time claim. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DnbRnTFErhoMJdAjVEeCg2 --- pkg/fingerprint/fingerprint.go | 3 +- pkg/fingerprint/fingerprint_bench_test.go | 83 +++++-- pkg/fingerprint/tokscratch_legacy_test.go | 55 +++++ pkg/fingerprint/tokscratch_test.go | 103 +++++++++ pkg/formatter/formatter.go | 3 +- pkg/formatter/scratch_corpus_test.go | 137 +++++++++++ pkg/formatter/scratch_legacy_test.go | 95 ++++++++ pkg/transform/tokscratch_bench_test.go | 92 ++++++++ pkg/transform/tokscratch_test.go | 267 ++++++++++++++++++++++ pkg/transform/transform.go | 7 +- 10 files changed, 822 insertions(+), 23 deletions(-) create mode 100644 pkg/fingerprint/tokscratch_legacy_test.go create mode 100644 pkg/fingerprint/tokscratch_test.go create mode 100644 pkg/formatter/scratch_corpus_test.go create mode 100644 pkg/formatter/scratch_legacy_test.go create mode 100644 pkg/transform/tokscratch_bench_test.go create mode 100644 pkg/transform/tokscratch_test.go diff --git a/pkg/fingerprint/fingerprint.go b/pkg/fingerprint/fingerprint.go index adc30a9eb..6b3f4e6d1 100644 --- a/pkg/fingerprint/fingerprint.go +++ b/pkg/fingerprint/fingerprint.go @@ -42,6 +42,7 @@ import ( "fmt" "strings" + "github.com/ajitpratap0/GoSQLX/internal/tokscratch" "github.com/ajitpratap0/GoSQLX/pkg/formatter" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" @@ -91,7 +92,7 @@ func Normalize(sql string) (string, error) { tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return "", fmt.Errorf("fingerprint: tokenization failed: %w", err) } diff --git a/pkg/fingerprint/fingerprint_bench_test.go b/pkg/fingerprint/fingerprint_bench_test.go index 0a4fea111..11ac71193 100644 --- a/pkg/fingerprint/fingerprint_bench_test.go +++ b/pkg/fingerprint/fingerprint_bench_test.go @@ -20,25 +20,72 @@ import ( "github.com/ajitpratap0/GoSQLX/pkg/fingerprint" ) -const benchJoinSQL = "SELECT c.id, c.name, SUM(o.total) AS spend FROM customers c " + - "JOIN orders o ON o.customer_id = c.id LEFT JOIN regions g ON g.code = c.region " + - "WHERE c.active = 1 AND o.created_at > '2024-01-01' GROUP BY c.id, c.name " + - "ORDER BY spend DESC, c.name LIMIT 50" - -func BenchmarkFingerprint(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - if _, err := fingerprint.Fingerprint(benchJoinSQL); err != nil { - b.Fatal(err) - } - } +var portableQueries = []struct{ ID, SQL string }{ + { + ID: "simple_select", + SQL: "SELECT id, name FROM users", + }, + { + ID: "select_where", + SQL: "SELECT id, name, email FROM users WHERE active = 1 AND created_at > '2024-01-01' ORDER BY name LIMIT 10", + }, + { + ID: "join", + SQL: "SELECT u.id, u.name, o.total FROM users u JOIN orders o ON u.id = o.user_id WHERE o.total > 100 ORDER BY o.total DESC LIMIT 50", + }, + { + ID: "aggregate", + SQL: "SELECT status, COUNT(*), SUM(total) FROM orders GROUP BY status HAVING COUNT(*) > 10", + }, + { + ID: "subquery", + SQL: "SELECT name FROM users WHERE id IN (SELECT user_id FROM orders WHERE total > 1000)", + }, + { + ID: "cte", + SQL: "WITH recent AS (SELECT user_id, total FROM orders WHERE total > 100) SELECT user_id, SUM(total) FROM recent GROUP BY user_id", + }, + { + ID: "window", + SQL: "SELECT name, salary, ROW_NUMBER() OVER (PARTITION BY dept ORDER BY salary DESC) FROM employees", + }, + { + ID: "complex", + SQL: "SELECT c.name, p.category, COUNT(o.id), SUM(oi.quantity * oi.price) " + + "FROM customers c " + + "JOIN orders o ON c.id = o.customer_id " + + "JOIN order_items oi ON o.id = oi.order_id " + + "LEFT JOIN products p ON oi.product_id = p.id " + + "WHERE o.created_at >= '2024-01-01' AND p.category IS NOT NULL " + + "GROUP BY c.name, p.category " + + "HAVING SUM(oi.quantity * oi.price) > 500 " + + "ORDER BY c.name, p.category LIMIT 100", + }, + { + ID: "insert", + SQL: "INSERT INTO users (id, name, email) VALUES (1, 'alice', 'alice@example.com')", + }, + { + ID: "update", + SQL: "UPDATE users SET name = 'bob', active = 0 WHERE id = 42", + }, + { + ID: "delete", + SQL: "DELETE FROM sessions WHERE expires_at < '2024-01-01'", + }, } -func BenchmarkNormalize(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - if _, err := fingerprint.Normalize(benchJoinSQL); err != nil { - b.Fatal(err) - } +func BenchmarkNormalize(b *testing.B) { benchmarkQueries(b, fingerprint.Normalize) } +func BenchmarkFingerprint(b *testing.B) { benchmarkQueries(b, fingerprint.Fingerprint) } +func benchmarkQueries(b *testing.B, fn func(string) (string, error)) { + for _, q := range portableQueries { + b.Run(q.ID, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, err := fn(q.SQL); err != nil { + b.Fatal(err) + } + } + }) } } diff --git a/pkg/fingerprint/tokscratch_legacy_test.go b/pkg/fingerprint/tokscratch_legacy_test.go new file mode 100644 index 000000000..b1d9dd6d3 --- /dev/null +++ b/pkg/fingerprint/tokscratch_legacy_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package fingerprint + +import ( + "fmt" + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +// LegacyNormalizeForScratchTest preserves the public-tokenizer pipeline as a compatibility oracle. +func LegacyNormalizeForScratchTest(sql string) (string, error) { + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + + tokens, err := tkz.TokenizeString(sql) + if err != nil { + return "", fmt.Errorf("fingerprint: tokenization failed: %w", err) + } + + p := parser.GetParser() + defer parser.PutParser(p) + + astObj, err := p.ParseFromModelTokens(tokens) + if err != nil { + return "", fmt.Errorf("fingerprint: parsing failed: %w", err) + } + defer ast.ReleaseAST(astObj) + + // Walk the AST and replace all non-placeholder literals with "?". + v := &literalNormalizer{} + for _, stmt := range astObj.Statements { + if err := ast.Walk(v, stmt); err != nil { + return "", fmt.Errorf("fingerprint: AST walk failed: %w", err) + } + } + + // Format the mutated AST back to SQL using compact (single-line) style. + opts := ast.CompactStyle() + var parts []string + for _, stmt := range astObj.Statements { + formatted, err := formatter.FormatStatementChecked(stmt, opts) + if err != nil { + return "", fmt.Errorf("fingerprint: formatting failed: %w", err) + } + parts = append(parts, formatted) + } + + return strings.Join(parts, "; "), nil +} diff --git a/pkg/fingerprint/tokscratch_test.go b/pkg/fingerprint/tokscratch_test.go new file mode 100644 index 000000000..5805ec3f4 --- /dev/null +++ b/pkg/fingerprint/tokscratch_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package fingerprint_test + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/fingerprint" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" +) + +type scratchResult struct{ normalized, fingerprint, err string } + +func scratchRun(sql string) scratchResult { + n, e := fingerprint.Normalize(sql) + f, fe := fingerprint.Fingerprint(sql) + if fmt.Sprint(e) != fmt.Sprint(fe) { + return scratchResult{n, f, "inconsistent errors"} + } + return scratchResult{n, f, fmt.Sprint(e)} +} +func scratchCorpus(t *testing.T) []string { + t.Helper() + var queries []string + for _, root := range []string{"../../testdata/realworld", "../../testdata/corpus"} { + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || filepath.Ext(path) != ".sql" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + queries = append(queries, string(data)) + queries = append(queries, gosqlx.SplitStatements(string(data), "")...) + return nil + }) + if err != nil { + t.Fatal(err) + } + } + if len(queries) == 0 { + t.Fatal("empty corpus") + } + return queries +} +func TestScratchDifferential(t *testing.T) { + for i, sql := range scratchCorpus(t) { + want, err := fingerprint.LegacyNormalizeForScratchTest(sql) + got := scratchRun(sql) + hash := "" + if err == nil { + hash = fmt.Sprintf("%x", sha256.Sum256([]byte(want))) + } + if got != (scratchResult{want, hash, fmt.Sprint(err)}) { + t.Fatalf("query %d mismatch: got %#v, want %q, %q, %v", i, got, want, hash, err) + } + } +} +func TestScratchConcurrentAndRetainedResults(t *testing.T) { + queries := scratchCorpus(t) + want := make([]scratchResult, len(queries)) + for i, q := range queries { + result := scratchRun(q) + want[i] = scratchResult{strings.Clone(result.normalized), strings.Clone(result.fingerprint), strings.Clone(result.err)} + } + var wg sync.WaitGroup + for worker := 0; worker < 8; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + retained := make([]scratchResult, len(queries)) + for i, q := range queries { + retained[i] = scratchRun(q) + if retained[i] != want[i] { + t.Errorf("concurrent query %d mismatch", i) + return + } + } + // Further calls recycle pooled token/Word storage while earlier strings stay live. + for _, q := range queries { + scratchRun(q) + } + for i, got := range retained { + if got != want[i] { + t.Errorf("retained query %d changed", i) + return + } + } + }() + } + wg.Wait() +} diff --git a/pkg/formatter/formatter.go b/pkg/formatter/formatter.go index 3b2f34320..ff5d27cce 100644 --- a/pkg/formatter/formatter.go +++ b/pkg/formatter/formatter.go @@ -93,6 +93,7 @@ import ( "fmt" "strings" + "github.com/ajitpratap0/GoSQLX/internal/tokscratch" "github.com/ajitpratap0/GoSQLX/pkg/models" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" @@ -139,7 +140,7 @@ func (f *Formatter) Format(sql string) (string, error) { tkz.SetDialect(dialect) } - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return "", fmt.Errorf("tokenization failed: %w", err) } diff --git a/pkg/formatter/scratch_corpus_test.go b/pkg/formatter/scratch_corpus_test.go new file mode 100644 index 000000000..93ccc7f58 --- /dev/null +++ b/pkg/formatter/scratch_corpus_test.go @@ -0,0 +1,137 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package formatter_test + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" +) + +type scratchFormatCase struct { + name, sql, dialect string +} + +type scratchFormatResult struct { + text, err string +} + +func scratchFormatResultOf(text string, err error) scratchFormatResult { + result := scratchFormatResult{text: text} + if err != nil { + result.err = err.Error() + } + return result +} + +func scratchFormatCorpus(t *testing.T) []scratchFormatCase { + t.Helper() + var cases []scratchFormatCase + for _, root := range []string{"realworld", "corpus"} { + dir := filepath.Join("..", "..", "testdata", root) + err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(path) != ".sql" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + dialect := "" + if root == "realworld" { + dialect = strings.Split(filepath.ToSlash(rel), "/")[0] + } + name := root + "/" + filepath.ToSlash(rel) + cases = append(cases, scratchFormatCase{name + "/whole", string(data), dialect}) + for i, stmt := range gosqlx.SplitStatements(string(data), dialect) { + if strings.TrimSpace(stmt) != "" { + cases = append(cases, scratchFormatCase{fmt.Sprintf("%s/statement_%d", name, i), stmt, dialect}) + } + } + return nil + }) + if err != nil { + t.Fatalf("load %s: %v", dir, err) + } + } + if len(cases) == 0 { + t.Fatal("empty SQL corpus") + } + return cases +} + +func TestScratchFormatDifferentialCorpus(t *testing.T) { + cases := scratchFormatCorpus(t) + successes := 0 + for _, tc := range cases { + variants := []formatter.Options{ + {Dialect: tc.dialect}, + {Dialect: tc.dialect, Compact: true, Uppercase: true, IndentSize: 4}, + {Dialect: "unknown-dialect", Uppercase: true}, + } + for i, opts := range variants { + want := scratchFormatResultOf(formatter.LegacyFormatForScratchTest(tc.sql, opts)) + got := scratchFormatResultOf(formatter.New(opts).Format(tc.sql)) + if got != want { + t.Errorf("%s/options_%d: scratch and public tokenizer differ\ngot: %#v\nwant: %#v", tc.name, i, got, want) + } + if got.err == "" && got.text != "" { + successes++ + } + } + } + if successes == 0 { + t.Fatal("no corpus statements formatted successfully") + } + t.Logf("compared %d corpus inputs with three formatting options; %d successful results", len(cases), successes) +} + +func TestScratchFormatConcurrentCorpusAndRetainedResults(t *testing.T) { + cases := scratchFormatCorpus(t) + baseline := make([]scratchFormatResult, len(cases)) + // Clone the reference so retaining it cannot hide corruption through a shared backing store. + for i, tc := range cases { + result := scratchFormatResultOf(formatter.New(formatter.Options{Dialect: tc.dialect, Uppercase: true}).Format(tc.sql)) + baseline[i] = scratchFormatResult{strings.Clone(result.text), strings.Clone(result.err)} + } + var wg sync.WaitGroup + for worker := 0; worker < 8; worker++ { + wg.Add(1) + go func(worker int) { + defer wg.Done() + retained := make([]scratchFormatResult, len(cases)) + for step := range cases { + i := (step + worker) % len(cases) + tc := cases[i] + // Each goroutine owns its Formatter, as required by its API contract. + got := scratchFormatResultOf(formatter.New(formatter.Options{Dialect: tc.dialect, Uppercase: true}).Format(tc.sql)) + if got != baseline[i] { + t.Errorf("worker %d %s: result differs from sequential run", worker, tc.name) + } + retained[i] = got + } + // All strings remain live while further calls return and reuse pooled buffers. + for i, got := range retained { + if got != baseline[i] { + t.Errorf("worker %d %s: retained output changed after pool reuse", worker, cases[i].name) + } + } + }(worker) + } + wg.Wait() +} diff --git a/pkg/formatter/scratch_legacy_test.go b/pkg/formatter/scratch_legacy_test.go new file mode 100644 index 000000000..e5175b6d4 --- /dev/null +++ b/pkg/formatter/scratch_legacy_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "fmt" + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/models" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +// LegacyFormatForScratchTest freezes the public-tokenizer pipeline as a differential oracle. +func LegacyFormatForScratchTest(sql string, opts Options) (string, error) { + f := New(opts) + if strings.TrimSpace(sql) == "" { + return "", nil + } + + // The dialect selects the tokenizer and parser rules (string escapes, + // quoting, dialect-only syntax) as well as the rendering. An unknown + // dialect name is ignored for tokenizing and parsing, as before. + dialect := keywords.SQLDialect(strings.ToLower(f.opts.Dialect)) + useDialect := dialect != "" && keywords.IsValidDialect(string(dialect)) + + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + if useDialect { + tkz.SetDialect(dialect) + } + + tokens, err := tkz.TokenizeString(sql) + if err != nil { + return "", fmt.Errorf("tokenization failed: %w", err) + } + + if len(tokens) == 0 { + return "", nil + } + + // Capture comments from tokenizer before parsing + comments := tkz.Comments + + p := parser.GetParser() + defer parser.PutParser(p) + if useDialect { + p.ApplyOptions(parser.WithDialect(string(dialect))) + } + var parsedAST *ast.AST + if useDialect && dialect == keywords.DialectOracle { + parsedAST, err = parser.ParseWithDialect(sql, dialect) + } else { + parsedAST, err = p.ParseFromModelTokens(tokens) + } + if err != nil { + return "", fmt.Errorf("parsing failed: %w", err) + } + defer ast.ReleaseAST(parsedAST) + + // Attach captured comments to AST + if len(comments) > 0 { + parsedAST.Comments = make([]models.Comment, len(comments)) + copy(parsedAST.Comments, comments) + } + + // Build format options and delegate to the visitor-based renderer. + style := ast.ReadableStyle() + if f.opts.Compact { + style = ast.CompactStyle() + } + if f.opts.IndentSize > 0 { + style.IndentWidth = f.opts.IndentSize + } + if f.opts.Uppercase { + style.KeywordCase = ast.KeywordUpper + } + style.Dialect = f.opts.Dialect + + return formatASTCheckedWithTokens(parsedAST, style, tokens) +} diff --git a/pkg/transform/tokscratch_bench_test.go b/pkg/transform/tokscratch_bench_test.go new file mode 100644 index 000000000..70d6ad2da --- /dev/null +++ b/pkg/transform/tokscratch_bench_test.go @@ -0,0 +1,92 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package transform + +import ( + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" +) + +// Queries copied from benchmarks/compare/corpus.go, which lives in a separate module. +var transformScratchBenchCorpus = []struct{ ID, Name, SQL string }{ + { + ID: "simple_select", + Name: "Simple SELECT", + SQL: "SELECT id, name FROM users", + }, + { + ID: "select_where", + Name: "SELECT with WHERE, ORDER BY, LIMIT", + SQL: "SELECT id, name, email FROM users WHERE active = 1 AND created_at > '2024-01-01' ORDER BY name LIMIT 10", + }, + { + ID: "join", + Name: "Two-table JOIN", + SQL: "SELECT u.id, u.name, o.total FROM users u JOIN orders o ON u.id = o.user_id WHERE o.total > 100 ORDER BY o.total DESC LIMIT 50", + }, + { + ID: "aggregate", + Name: "GROUP BY with HAVING", + SQL: "SELECT status, COUNT(*), SUM(total) FROM orders GROUP BY status HAVING COUNT(*) > 10", + }, + { + ID: "subquery", + Name: "IN subquery", + SQL: "SELECT name FROM users WHERE id IN (SELECT user_id FROM orders WHERE total > 1000)", + }, + { + ID: "cte", + Name: "Common table expression", + SQL: "WITH recent AS (SELECT user_id, total FROM orders WHERE total > 100) SELECT user_id, SUM(total) FROM recent GROUP BY user_id", + }, + { + ID: "window", + Name: "Window function", + SQL: "SELECT name, salary, ROW_NUMBER() OVER (PARTITION BY dept ORDER BY salary DESC) FROM employees", + }, + { + ID: "complex", + Name: "Multi-JOIN analytical query", + SQL: "SELECT c.name, p.category, COUNT(o.id), SUM(oi.quantity * oi.price) " + + "FROM customers c " + + "JOIN orders o ON c.id = o.customer_id " + + "JOIN order_items oi ON o.id = oi.order_id " + + "LEFT JOIN products p ON oi.product_id = p.id " + + "WHERE o.created_at >= '2024-01-01' AND p.category IS NOT NULL " + + "GROUP BY c.name, p.category " + + "HAVING SUM(oi.quantity * oi.price) > 500 " + + "ORDER BY c.name, p.category LIMIT 100", + }, + { + ID: "insert", + Name: "INSERT VALUES", + SQL: "INSERT INTO users (id, name, email) VALUES (1, 'alice', 'alice@example.com')", + }, + { + ID: "update", + Name: "UPDATE with WHERE", + SQL: "UPDATE users SET name = 'bob', active = 0 WHERE id = 42", + }, + { + ID: "delete", + Name: "DELETE with WHERE", + SQL: "DELETE FROM sessions WHERE expires_at < '2024-01-01'", + }, +} + +func BenchmarkTransformParse(b *testing.B) { + for _, q := range transformScratchBenchCorpus { + b.Run(q.ID, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + tree, err := ParseSQL(q.SQL) + if err != nil { + b.Fatal(err) + } + ast.ReleaseAST(tree) + } + }) + } +} diff --git a/pkg/transform/tokscratch_test.go b/pkg/transform/tokscratch_test.go new file mode 100644 index 000000000..630917147 --- /dev/null +++ b/pkg/transform/tokscratch_test.go @@ -0,0 +1,267 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package transform + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +// legacyScratchParse deliberately uses the public copying tokenizer API as an +// independent reference for all three scratch consumers in transform.go. +func legacyScratchParse(sql string, dialect keywords.SQLDialect, withDialect bool) (*ast.AST, error) { + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + if withDialect && dialect != "" { + tkz.SetDialect(dialect) + } + tokens, err := tkz.TokenizeString(sql) + if err != nil { + return nil, fmt.Errorf("tokenize: %w", err) + } + p := parser.GetParser() + defer parser.PutParser(p) + if withDialect { + p.ApplyOptions(parser.WithDialect(string(dialect))) + } + tree, err := p.ParseFromModelTokens(tokens) + if err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + return tree, nil +} + +type scratchCase struct { + name, sql string + dialect keywords.SQLDialect +} + +func scratchCorpus(t *testing.T) []scratchCase { + t.Helper() + var cases []scratchCase + for _, root := range []string{"../../testdata/realworld", "../../testdata/corpus"} { + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(path) != ".sql" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + dialect := keywords.DialectGeneric + for _, d := range keywords.AllDialects() { + if strings.Contains(filepath.ToSlash(path), "/"+string(d)+"/") { + dialect = d + break + } + } + for i, sql := range gosqlx.SplitStatements(string(data), string(dialect)) { + cases = append(cases, scratchCase{fmt.Sprintf("%s/%d", path, i), sql, dialect}) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + } + if len(cases) < 100 { + t.Fatalf("corpus unexpectedly small: %d", len(cases)) + } + for _, sql := range []string{"", "SELECT 'unterminated", "SELECT FROM", "SELECT first_name FROM first_table; SELECT second_name FROM second_table"} { + cases = append(cases, scratchCase{sql, sql, keywords.DialectGeneric}) + } + return cases +} + +type scratchResult struct { + output []string + err string +} + +func scratchRender(tree *ast.AST, err error) scratchResult { + if err != nil { + return scratchResult{err: err.Error()} + } + var result scratchResult + if tree != nil { + for _, stmt := range tree.Statements { + sql, err := FormatSQLChecked(stmt) + if err != nil { + result.output = append(result.output, "render error: "+err.Error()) + } else { + result.output = append(result.output, sql) + } + } + } + return result +} + +func scratchParseCase(c scratchCase, mode int, legacy bool) (*ast.AST, error) { + if legacy { + tree, err := legacyScratchParse(c.sql, c.dialect, mode == 1) + if mode != 2 || err != nil { + return tree, err + } + if tree == nil || len(tree.Statements) == 0 { + ast.ReleaseAST(tree) + return nil, fmt.Errorf("no statements parsed") + } + stmt := tree.Statements[0] + tree.Statements[0] = nil + ast.ReleaseAST(tree) + return &ast.AST{Statements: []ast.Statement{stmt}}, nil + } + switch mode { + case 0: + return ParseSQL(c.sql) + case 1: + return ParseSQLWithDialect(c.sql, c.dialect) + default: + stmt, err := parseSQL(c.sql) + if err != nil { + return nil, err + } + return &ast.AST{Statements: []ast.Statement{stmt}}, nil + } +} + +func TestTokenScratchCorpusDifferential(t *testing.T) { + for _, c := range scratchCorpus(t) { + for mode, name := range []string{"ParseSQL", "ParseSQLWithDialect", "parseSQL"} { + t.Run(c.name+"/"+name, func(t *testing.T) { + oldTree, oldErr := scratchParseCase(c, mode, true) + defer ast.ReleaseAST(oldTree) + newTree, newErr := scratchParseCase(c, mode, false) + defer ast.ReleaseAST(newTree) + want, got := scratchRender(oldTree, oldErr), scratchRender(newTree, newErr) + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v, want %#v", got, want) + } + }) + } + } +} + +func TestTokenScratchCorpusConcurrent(t *testing.T) { + cases := scratchCorpus(t) + sequential := make([][3]scratchResult, len(cases)) + for i, c := range cases { + for mode := range 3 { + tree, err := scratchParseCase(c, mode, false) + sequential[i][mode] = scratchRender(tree, err) + ast.ReleaseAST(tree) + } + } + var wg sync.WaitGroup + for worker := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for i, c := range cases { + for mode := range 3 { + tree, err := scratchParseCase(c, mode, false) + got := scratchRender(tree, err) + ast.ReleaseAST(tree) + if !reflect.DeepEqual(got, sequential[i][mode]) { + t.Errorf("worker %d %s mode %d: got %#v, want %#v", worker, c.name, mode, got, sequential[i][mode]) + return + } + } + } + }() + } + wg.Wait() +} + +func TestTokenScratchReturnedASTSurvivesReuse(t *testing.T) { + // Retain caller-owned nodes while later calls overwrite tokenizer scratch. + for mode, name := range []string{"ParseSQL", "ParseSQLWithDialect", "parseSQL"} { + t.Run(name, func(t *testing.T) { + c := scratchCase{sql: "SELECT retained_name, 'retained value' FROM retained_table WHERE retained_id = 123", dialect: keywords.DialectPostgreSQL} + tree, err := scratchParseCase(c, mode, false) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + want := scratchRender(tree, nil) + for i := range 64 { + other, err := scratchParseCase(scratchCase{sql: fmt.Sprintf("SELECT overwritten_%d, 'replacement' FROM another_table WHERE another_id = 999", i), dialect: keywords.DialectMySQL}, i%3, false) + if err != nil { + t.Fatal(err) + } + ast.ReleaseAST(other) + } + if got := scratchRender(tree, nil); !reflect.DeepEqual(got, want) { + t.Fatalf("after reuse: got %#v, want %#v", got, want) + } + }) + } +} + +func TestTokenScratchExtractedNodesConcurrent(t *testing.T) { + // These two helpers return subnodes detached from parseSQL's statement. + // Retain those subnodes across calls on every worker as well. + const condition = "retained_id = 123 AND retained_name = 'kept'" + const join = "JOIN retained_table r ON r.retained_id = _t.retained_id" + makeTree := func() (*ast.AST, error) { + expr, err := parseCondition(condition) + if err != nil { + return nil, err + } + clause, err := parseJoinClause(join) + if err != nil { + return nil, err + } + return &ast.AST{Statements: []ast.Statement{&ast.SelectStatement{ + Columns: []ast.Expression{&ast.Identifier{Name: "retained_column"}}, + Where: expr, Joins: []ast.JoinClause{*clause}, + }}}, nil + } + baseline, err := makeTree() + if err != nil { + t.Fatal(err) + } + want := scratchRender(baseline, nil) + ast.ReleaseAST(baseline) + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + retained, err := makeTree() + if err != nil { + t.Error(err) + return + } + defer ast.ReleaseAST(retained) + for range 32 { + other, err := makeTree() + if err != nil { + t.Error(err) + return + } + ast.ReleaseAST(other) + } + if got := scratchRender(retained, nil); !reflect.DeepEqual(got, want) { + t.Errorf("retained condition/join after reuse: got %#v, want %#v", got, want) + } + }() + } + wg.Wait() +} diff --git a/pkg/transform/transform.go b/pkg/transform/transform.go index 254a590b4..9935239b8 100644 --- a/pkg/transform/transform.go +++ b/pkg/transform/transform.go @@ -17,6 +17,7 @@ package transform import ( "fmt" + "github.com/ajitpratap0/GoSQLX/internal/tokscratch" "github.com/ajitpratap0/GoSQLX/pkg/formatter" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" @@ -115,7 +116,7 @@ func parseSQL(sql string) (ast.Statement, error) { tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return nil, fmt.Errorf("tokenize: %w", err) } @@ -210,7 +211,7 @@ func ParseSQL(sql string) (*ast.AST, error) { tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return nil, fmt.Errorf("tokenize: %w", err) } @@ -294,7 +295,7 @@ func ParseSQLWithDialect(sql string, dialect keywords.SQLDialect) (*ast.AST, err tkz.SetDialect(dialect) } - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return nil, fmt.Errorf("tokenize: %w", err) } From 89fa21140a48c855347ad78d8ddf6809aeab7cfc Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh <18012955+ajitpratap0@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:58:21 +0530 Subject: [PATCH 2/4] docs(changelog): W5-01 tokscratch consumers Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DnbRnTFErhoMJdAjVEeCg2 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84e85635b..c597e0ca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -274,6 +274,7 @@ upgrading. MCP Registry (`publish-mcp.yml`, `docs/MCP_REGISTRY.md`). (#859, #876) ### Changed +- **Performance (fingerprint, formatter, transform)**: `fingerprint.Normalize`/`Fingerprint`, the formatter parse path and `transform.ParseSQL` tokenize through the internal pooled token scratch instead of the public `TokenizeString`. `Normalize` allocates 42-70% fewer bytes/op and `transform.ParseSQL` 49-78% fewer, with 2-3 fewer allocs/op and no output change [W5-01]. - **Transpiling dialect-specific operators**: `^` is exponentiation in PostgreSQL and bitwise XOR in MySQL, MariaDB and SQL Server (recorded as `BinaryExpression.BitwiseXor`). Exponentiation is written `POWER(a, b)` From 02196959bad7abe52db6631bfa31112555016100 Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh <18012955+ajitpratap0@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:34:42 +0530 Subject: [PATCH 3/4] docs(changelog): move W5-01 entry under [Unreleased] Performance Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DnbRnTFErhoMJdAjVEeCg2 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e755a953..290c07c79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance - **Pools**: the tokenizer, parser and AST pools now count Get/Put through precreated, cache-line padded atomic counters (`internal/poolstats`) instead of an RWMutex-guarded map keyed by pool name, removing a lock, map lookup and string hash from every pool operation. `metrics.GetPoolStats` and `metrics.ResetPoolStats` behave exactly as before; named custom pools keep the map path. [W1-01] - **Parser**: token preprocessing now takes a type-gated fast path. Only identifier, keyword, aggregate and compound tokens take the normalisation path; every other token is appended with a single copy. Output is identical (guarded by an old-versus-new differential test); no API or behaviour change. +- **Fingerprint, formatter, transform**: `fingerprint.Normalize`/`Fingerprint`, the formatter parse path and `transform.ParseSQL` tokenize through the internal pooled token scratch instead of the public `TokenizeString`. `Normalize` allocates 42-70% fewer bytes/op and `transform.ParseSQL` 49-78% fewer, with 2-3 fewer allocs/op and no output change [W5-01]. ### Fixed - **AST**: `AlterTableStatement.SQL()` now separates multiple ALTER TABLE actions with commas, matching the compact formatter and preserving action boundaries when re-parsed. @@ -278,7 +279,6 @@ upgrading. MCP Registry (`publish-mcp.yml`, `docs/MCP_REGISTRY.md`). (#859, #876) ### Changed -- **Performance (fingerprint, formatter, transform)**: `fingerprint.Normalize`/`Fingerprint`, the formatter parse path and `transform.ParseSQL` tokenize through the internal pooled token scratch instead of the public `TokenizeString`. `Normalize` allocates 42-70% fewer bytes/op and `transform.ParseSQL` 49-78% fewer, with 2-3 fewer allocs/op and no output change [W5-01]. - **Transpiling dialect-specific operators**: `^` is exponentiation in PostgreSQL and bitwise XOR in MySQL, MariaDB and SQL Server (recorded as `BinaryExpression.BitwiseXor`). Exponentiation is written `POWER(a, b)` From d36d27d49d43bdf43eb5caf97f4e0008965ddfe0 Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh <18012955+ajitpratap0@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:45:02 +0530 Subject: [PATCH 4/4] test(perf): gate W5-01 targets and add cross-consumer tokenizer pool race test [W5-01] Add fingerprint.Normalize and transform ParseSQL cases to the perf gate suite, add a parallel gosqlx/formatter/fingerprint/transform race test on the shared tokenizer pool, and fix the CHANGELOG wording. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DnbRnTFErhoMJdAjVEeCg2 --- CHANGELOG.md | 2 +- pkg/fingerprint/crossconsumer_race_test.go | 81 ++++++++++++++++++++++ scripts/perf/gate-suite.txt | 2 + 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 pkg/fingerprint/crossconsumer_race_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 290c07c79..1a6a88d08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Performance - **Pools**: the tokenizer, parser and AST pools now count Get/Put through precreated, cache-line padded atomic counters (`internal/poolstats`) instead of an RWMutex-guarded map keyed by pool name, removing a lock, map lookup and string hash from every pool operation. `metrics.GetPoolStats` and `metrics.ResetPoolStats` behave exactly as before; named custom pools keep the map path. [W1-01] - **Parser**: token preprocessing now takes a type-gated fast path. Only identifier, keyword, aggregate and compound tokens take the normalisation path; every other token is appended with a single copy. Output is identical (guarded by an old-versus-new differential test); no API or behaviour change. -- **Fingerprint, formatter, transform**: `fingerprint.Normalize`/`Fingerprint`, the formatter parse path and `transform.ParseSQL` tokenize through the internal pooled token scratch instead of the public `TokenizeString`. `Normalize` allocates 42-70% fewer bytes/op and `transform.ParseSQL` 49-78% fewer, with 2-3 fewer allocs/op and no output change [W5-01]. +- **Fingerprint, formatter, transform**: `fingerprint.Normalize`/`Fingerprint`, `Formatter.Format` and `transform.ParseSQL`/`ParseSQLWithDialect` tokenize through the internal pooled token scratch instead of the public `TokenizeString`. `Normalize` allocates 42-70% fewer bytes/op and `transform.ParseSQL` 49-78% fewer, with 2-3 fewer allocs/op and no output change. [W5-01] ### Fixed - **AST**: `AlterTableStatement.SQL()` now separates multiple ALTER TABLE actions with commas, matching the compact formatter and preserving action boundaries when re-parsed. diff --git a/pkg/fingerprint/crossconsumer_race_test.go b/pkg/fingerprint/crossconsumer_race_test.go new file mode 100644 index 000000000..e0d82e74d --- /dev/null +++ b/pkg/fingerprint/crossconsumer_race_test.go @@ -0,0 +1,81 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package fingerprint_test + +import ( + "sync" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/fingerprint" + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/transform" +) + +// TestCrossConsumerSharedTokenizerPool runs every consumer of the shared +// tokenizer pool and its pooled token scratch (gosqlx.Parse, formatter.Format, +// fingerprint.Normalize, transform.ParseSQL) in parallel and checks that each +// result matches its single-goroutine baseline. Under -race this catches a +// consumer that reads scratch memory after the tokenizer went back to the pool. +func TestCrossConsumerSharedTokenizerPool(t *testing.T) { + queries := []string{ + "SELECT id, name FROM users WHERE id = 42 AND status IN ('a','b')", + "SELECT u.id, o.total FROM users u JOIN orders o ON o.user_id = u.id WHERE o.total > 100 ORDER BY o.total DESC LIMIT 10", + "WITH t AS (SELECT a, SUM(b) AS s FROM x GROUP BY a) SELECT a, s FROM t WHERE s > 5 -- trailing comment", + "INSERT INTO t (a, b) VALUES (1, 'x'), (2, 'y')", + "UPDATE t SET a = 1, b = 'z' WHERE id = 7", + } + f := formatter.New(formatter.Options{}) + + type base struct{ norm, formatted string } + want := make([]base, len(queries)) + for i, q := range queries { + n, err := fingerprint.Normalize(q) + if err != nil { + t.Fatalf("baseline Normalize(%q): %v", q, err) + } + s, err := f.Format(q) + if err != nil { + t.Fatalf("baseline Format(%q): %v", q, err) + } + want[i] = base{n, s} + } + + const goroutines, iters = 16, 60 + var wg sync.WaitGroup + errs := make(chan string, goroutines*iters) + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < iters; i++ { + idx := (g + i) % len(queries) + q := queries[idx] + switch (g + i) % 4 { + case 0: + if _, err := gosqlx.Parse(q); err != nil { + errs <- "gosqlx.Parse: " + err.Error() + } + case 1: + if got, err := f.Format(q); err != nil || got != want[idx].formatted { + errs <- "formatter.Format mismatch for " + q + } + case 2: + if got, err := fingerprint.Normalize(q); err != nil || got != want[idx].norm { + errs <- "fingerprint.Normalize mismatch for " + q + } + default: + if _, err := transform.ParseSQL(q); err != nil { + errs <- "transform.ParseSQL: " + err.Error() + } + } + } + }(g) + } + wg.Wait() + close(errs) + for e := range errs { + t.Error(e) + } +} diff --git a/scripts/perf/gate-suite.txt b/scripts/perf/gate-suite.txt index 21cfb33e7..f9abca288 100644 --- a/scripts/perf/gate-suite.txt +++ b/scripts/perf/gate-suite.txt @@ -13,3 +13,5 @@ ./pkg/gosqlx ^BenchmarkParseFresh$/^(simple_select|join|complex)$ ./pkg/gosqlx ^BenchmarkPoolGetPut$/^serial$ ./pkg/sql/tokenizer ^BenchmarkTokenize$/^scratch$/^(join|complex|insert)$ +./pkg/fingerprint ^BenchmarkNormalize$/^(simple_select|join|complex)$ +./pkg/transform ^BenchmarkTransformParse$/^(simple_select|join|complex)$