diff --git a/CHANGELOG.md b/CHANGELOG.md index 745a8859..b3801161 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Tokenizer**: keyword and compound-keyword-start lookup now uses a single fused-hash open-addressed table (one FNV-1a hash computed while upper-casing, one probe) instead of two Go map lookups per identifier. The table is filled in a fixed order (hot keywords first, then sorted) so its probe layout is deterministic across processes. No behaviour change; token output is identical [W2-01]. - **Pools**: the tokenizer, parser and AST pools now count Get/Put through precreated, cache-line padded atomic counters (`internal/poolstats`) instead of an RWMutex-guarded map keyed by pool name, removing a lock, map lookup and string hash from every pool operation. `metrics.GetPoolStats` and `metrics.ResetPoolStats` behave exactly as before; named custom pools keep the map path. [W1-01] - **Parser**: token preprocessing now takes a type-gated fast path. Only identifier, keyword, aggregate and compound tokens take the normalisation path; every other token is appended with a single copy. Output is identical (guarded by an old-versus-new differential test); no API or behaviour change. +- **Fingerprint, formatter, transform**: `fingerprint.Normalize`/`Fingerprint`, `Formatter.Format` and `transform.ParseSQL`/`ParseSQLWithDialect` tokenize through the internal pooled token scratch instead of the public `TokenizeString`. `Normalize` allocates 42-70% fewer bytes/op and `transform.ParseSQL` 49-78% fewer, with 2-3 fewer allocs/op and no output change. [W5-01] ### Fixed - **AST**: `AlterTableStatement.SQL()` now separates multiple ALTER TABLE actions with commas, matching the compact formatter and preserving action boundaries when re-parsed. diff --git a/pkg/fingerprint/crossconsumer_race_test.go b/pkg/fingerprint/crossconsumer_race_test.go new file mode 100644 index 00000000..e0d82e74 --- /dev/null +++ b/pkg/fingerprint/crossconsumer_race_test.go @@ -0,0 +1,81 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package fingerprint_test + +import ( + "sync" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/fingerprint" + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/transform" +) + +// TestCrossConsumerSharedTokenizerPool runs every consumer of the shared +// tokenizer pool and its pooled token scratch (gosqlx.Parse, formatter.Format, +// fingerprint.Normalize, transform.ParseSQL) in parallel and checks that each +// result matches its single-goroutine baseline. Under -race this catches a +// consumer that reads scratch memory after the tokenizer went back to the pool. +func TestCrossConsumerSharedTokenizerPool(t *testing.T) { + queries := []string{ + "SELECT id, name FROM users WHERE id = 42 AND status IN ('a','b')", + "SELECT u.id, o.total FROM users u JOIN orders o ON o.user_id = u.id WHERE o.total > 100 ORDER BY o.total DESC LIMIT 10", + "WITH t AS (SELECT a, SUM(b) AS s FROM x GROUP BY a) SELECT a, s FROM t WHERE s > 5 -- trailing comment", + "INSERT INTO t (a, b) VALUES (1, 'x'), (2, 'y')", + "UPDATE t SET a = 1, b = 'z' WHERE id = 7", + } + f := formatter.New(formatter.Options{}) + + type base struct{ norm, formatted string } + want := make([]base, len(queries)) + for i, q := range queries { + n, err := fingerprint.Normalize(q) + if err != nil { + t.Fatalf("baseline Normalize(%q): %v", q, err) + } + s, err := f.Format(q) + if err != nil { + t.Fatalf("baseline Format(%q): %v", q, err) + } + want[i] = base{n, s} + } + + const goroutines, iters = 16, 60 + var wg sync.WaitGroup + errs := make(chan string, goroutines*iters) + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < iters; i++ { + idx := (g + i) % len(queries) + q := queries[idx] + switch (g + i) % 4 { + case 0: + if _, err := gosqlx.Parse(q); err != nil { + errs <- "gosqlx.Parse: " + err.Error() + } + case 1: + if got, err := f.Format(q); err != nil || got != want[idx].formatted { + errs <- "formatter.Format mismatch for " + q + } + case 2: + if got, err := fingerprint.Normalize(q); err != nil || got != want[idx].norm { + errs <- "fingerprint.Normalize mismatch for " + q + } + default: + if _, err := transform.ParseSQL(q); err != nil { + errs <- "transform.ParseSQL: " + err.Error() + } + } + } + }(g) + } + wg.Wait() + close(errs) + for e := range errs { + t.Error(e) + } +} diff --git a/pkg/fingerprint/fingerprint.go b/pkg/fingerprint/fingerprint.go index adc30a9e..6b3f4e6d 100644 --- a/pkg/fingerprint/fingerprint.go +++ b/pkg/fingerprint/fingerprint.go @@ -42,6 +42,7 @@ import ( "fmt" "strings" + "github.com/ajitpratap0/GoSQLX/internal/tokscratch" "github.com/ajitpratap0/GoSQLX/pkg/formatter" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" @@ -91,7 +92,7 @@ func Normalize(sql string) (string, error) { tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return "", fmt.Errorf("fingerprint: tokenization failed: %w", err) } diff --git a/pkg/fingerprint/tokscratch_legacy_test.go b/pkg/fingerprint/tokscratch_legacy_test.go new file mode 100644 index 00000000..b1d9dd6d --- /dev/null +++ b/pkg/fingerprint/tokscratch_legacy_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package fingerprint + +import ( + "fmt" + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +// LegacyNormalizeForScratchTest preserves the public-tokenizer pipeline as a compatibility oracle. +func LegacyNormalizeForScratchTest(sql string) (string, error) { + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + + tokens, err := tkz.TokenizeString(sql) + if err != nil { + return "", fmt.Errorf("fingerprint: tokenization failed: %w", err) + } + + p := parser.GetParser() + defer parser.PutParser(p) + + astObj, err := p.ParseFromModelTokens(tokens) + if err != nil { + return "", fmt.Errorf("fingerprint: parsing failed: %w", err) + } + defer ast.ReleaseAST(astObj) + + // Walk the AST and replace all non-placeholder literals with "?". + v := &literalNormalizer{} + for _, stmt := range astObj.Statements { + if err := ast.Walk(v, stmt); err != nil { + return "", fmt.Errorf("fingerprint: AST walk failed: %w", err) + } + } + + // Format the mutated AST back to SQL using compact (single-line) style. + opts := ast.CompactStyle() + var parts []string + for _, stmt := range astObj.Statements { + formatted, err := formatter.FormatStatementChecked(stmt, opts) + if err != nil { + return "", fmt.Errorf("fingerprint: formatting failed: %w", err) + } + parts = append(parts, formatted) + } + + return strings.Join(parts, "; "), nil +} diff --git a/pkg/fingerprint/tokscratch_test.go b/pkg/fingerprint/tokscratch_test.go new file mode 100644 index 00000000..5805ec3f --- /dev/null +++ b/pkg/fingerprint/tokscratch_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package fingerprint_test + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/fingerprint" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" +) + +type scratchResult struct{ normalized, fingerprint, err string } + +func scratchRun(sql string) scratchResult { + n, e := fingerprint.Normalize(sql) + f, fe := fingerprint.Fingerprint(sql) + if fmt.Sprint(e) != fmt.Sprint(fe) { + return scratchResult{n, f, "inconsistent errors"} + } + return scratchResult{n, f, fmt.Sprint(e)} +} +func scratchCorpus(t *testing.T) []string { + t.Helper() + var queries []string + for _, root := range []string{"../../testdata/realworld", "../../testdata/corpus"} { + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || filepath.Ext(path) != ".sql" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + queries = append(queries, string(data)) + queries = append(queries, gosqlx.SplitStatements(string(data), "")...) + return nil + }) + if err != nil { + t.Fatal(err) + } + } + if len(queries) == 0 { + t.Fatal("empty corpus") + } + return queries +} +func TestScratchDifferential(t *testing.T) { + for i, sql := range scratchCorpus(t) { + want, err := fingerprint.LegacyNormalizeForScratchTest(sql) + got := scratchRun(sql) + hash := "" + if err == nil { + hash = fmt.Sprintf("%x", sha256.Sum256([]byte(want))) + } + if got != (scratchResult{want, hash, fmt.Sprint(err)}) { + t.Fatalf("query %d mismatch: got %#v, want %q, %q, %v", i, got, want, hash, err) + } + } +} +func TestScratchConcurrentAndRetainedResults(t *testing.T) { + queries := scratchCorpus(t) + want := make([]scratchResult, len(queries)) + for i, q := range queries { + result := scratchRun(q) + want[i] = scratchResult{strings.Clone(result.normalized), strings.Clone(result.fingerprint), strings.Clone(result.err)} + } + var wg sync.WaitGroup + for worker := 0; worker < 8; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + retained := make([]scratchResult, len(queries)) + for i, q := range queries { + retained[i] = scratchRun(q) + if retained[i] != want[i] { + t.Errorf("concurrent query %d mismatch", i) + return + } + } + // Further calls recycle pooled token/Word storage while earlier strings stay live. + for _, q := range queries { + scratchRun(q) + } + for i, got := range retained { + if got != want[i] { + t.Errorf("retained query %d changed", i) + return + } + } + }() + } + wg.Wait() +} diff --git a/pkg/formatter/formatter.go b/pkg/formatter/formatter.go index 3b2f3432..ff5d27cc 100644 --- a/pkg/formatter/formatter.go +++ b/pkg/formatter/formatter.go @@ -93,6 +93,7 @@ import ( "fmt" "strings" + "github.com/ajitpratap0/GoSQLX/internal/tokscratch" "github.com/ajitpratap0/GoSQLX/pkg/models" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" @@ -139,7 +140,7 @@ func (f *Formatter) Format(sql string) (string, error) { tkz.SetDialect(dialect) } - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return "", fmt.Errorf("tokenization failed: %w", err) } diff --git a/pkg/formatter/scratch_corpus_test.go b/pkg/formatter/scratch_corpus_test.go new file mode 100644 index 00000000..93ccc7f5 --- /dev/null +++ b/pkg/formatter/scratch_corpus_test.go @@ -0,0 +1,137 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package formatter_test + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/formatter" + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" +) + +type scratchFormatCase struct { + name, sql, dialect string +} + +type scratchFormatResult struct { + text, err string +} + +func scratchFormatResultOf(text string, err error) scratchFormatResult { + result := scratchFormatResult{text: text} + if err != nil { + result.err = err.Error() + } + return result +} + +func scratchFormatCorpus(t *testing.T) []scratchFormatCase { + t.Helper() + var cases []scratchFormatCase + for _, root := range []string{"realworld", "corpus"} { + dir := filepath.Join("..", "..", "testdata", root) + err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(path) != ".sql" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + dialect := "" + if root == "realworld" { + dialect = strings.Split(filepath.ToSlash(rel), "/")[0] + } + name := root + "/" + filepath.ToSlash(rel) + cases = append(cases, scratchFormatCase{name + "/whole", string(data), dialect}) + for i, stmt := range gosqlx.SplitStatements(string(data), dialect) { + if strings.TrimSpace(stmt) != "" { + cases = append(cases, scratchFormatCase{fmt.Sprintf("%s/statement_%d", name, i), stmt, dialect}) + } + } + return nil + }) + if err != nil { + t.Fatalf("load %s: %v", dir, err) + } + } + if len(cases) == 0 { + t.Fatal("empty SQL corpus") + } + return cases +} + +func TestScratchFormatDifferentialCorpus(t *testing.T) { + cases := scratchFormatCorpus(t) + successes := 0 + for _, tc := range cases { + variants := []formatter.Options{ + {Dialect: tc.dialect}, + {Dialect: tc.dialect, Compact: true, Uppercase: true, IndentSize: 4}, + {Dialect: "unknown-dialect", Uppercase: true}, + } + for i, opts := range variants { + want := scratchFormatResultOf(formatter.LegacyFormatForScratchTest(tc.sql, opts)) + got := scratchFormatResultOf(formatter.New(opts).Format(tc.sql)) + if got != want { + t.Errorf("%s/options_%d: scratch and public tokenizer differ\ngot: %#v\nwant: %#v", tc.name, i, got, want) + } + if got.err == "" && got.text != "" { + successes++ + } + } + } + if successes == 0 { + t.Fatal("no corpus statements formatted successfully") + } + t.Logf("compared %d corpus inputs with three formatting options; %d successful results", len(cases), successes) +} + +func TestScratchFormatConcurrentCorpusAndRetainedResults(t *testing.T) { + cases := scratchFormatCorpus(t) + baseline := make([]scratchFormatResult, len(cases)) + // Clone the reference so retaining it cannot hide corruption through a shared backing store. + for i, tc := range cases { + result := scratchFormatResultOf(formatter.New(formatter.Options{Dialect: tc.dialect, Uppercase: true}).Format(tc.sql)) + baseline[i] = scratchFormatResult{strings.Clone(result.text), strings.Clone(result.err)} + } + var wg sync.WaitGroup + for worker := 0; worker < 8; worker++ { + wg.Add(1) + go func(worker int) { + defer wg.Done() + retained := make([]scratchFormatResult, len(cases)) + for step := range cases { + i := (step + worker) % len(cases) + tc := cases[i] + // Each goroutine owns its Formatter, as required by its API contract. + got := scratchFormatResultOf(formatter.New(formatter.Options{Dialect: tc.dialect, Uppercase: true}).Format(tc.sql)) + if got != baseline[i] { + t.Errorf("worker %d %s: result differs from sequential run", worker, tc.name) + } + retained[i] = got + } + // All strings remain live while further calls return and reuse pooled buffers. + for i, got := range retained { + if got != baseline[i] { + t.Errorf("worker %d %s: retained output changed after pool reuse", worker, cases[i].name) + } + } + }(worker) + } + wg.Wait() +} diff --git a/pkg/formatter/scratch_legacy_test.go b/pkg/formatter/scratch_legacy_test.go new file mode 100644 index 00000000..e5175b6d --- /dev/null +++ b/pkg/formatter/scratch_legacy_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 GoSQLX Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package formatter + +import ( + "fmt" + "strings" + + "github.com/ajitpratap0/GoSQLX/pkg/models" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +// LegacyFormatForScratchTest freezes the public-tokenizer pipeline as a differential oracle. +func LegacyFormatForScratchTest(sql string, opts Options) (string, error) { + f := New(opts) + if strings.TrimSpace(sql) == "" { + return "", nil + } + + // The dialect selects the tokenizer and parser rules (string escapes, + // quoting, dialect-only syntax) as well as the rendering. An unknown + // dialect name is ignored for tokenizing and parsing, as before. + dialect := keywords.SQLDialect(strings.ToLower(f.opts.Dialect)) + useDialect := dialect != "" && keywords.IsValidDialect(string(dialect)) + + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + if useDialect { + tkz.SetDialect(dialect) + } + + tokens, err := tkz.TokenizeString(sql) + if err != nil { + return "", fmt.Errorf("tokenization failed: %w", err) + } + + if len(tokens) == 0 { + return "", nil + } + + // Capture comments from tokenizer before parsing + comments := tkz.Comments + + p := parser.GetParser() + defer parser.PutParser(p) + if useDialect { + p.ApplyOptions(parser.WithDialect(string(dialect))) + } + var parsedAST *ast.AST + if useDialect && dialect == keywords.DialectOracle { + parsedAST, err = parser.ParseWithDialect(sql, dialect) + } else { + parsedAST, err = p.ParseFromModelTokens(tokens) + } + if err != nil { + return "", fmt.Errorf("parsing failed: %w", err) + } + defer ast.ReleaseAST(parsedAST) + + // Attach captured comments to AST + if len(comments) > 0 { + parsedAST.Comments = make([]models.Comment, len(comments)) + copy(parsedAST.Comments, comments) + } + + // Build format options and delegate to the visitor-based renderer. + style := ast.ReadableStyle() + if f.opts.Compact { + style = ast.CompactStyle() + } + if f.opts.IndentSize > 0 { + style.IndentWidth = f.opts.IndentSize + } + if f.opts.Uppercase { + style.KeywordCase = ast.KeywordUpper + } + style.Dialect = f.opts.Dialect + + return formatASTCheckedWithTokens(parsedAST, style, tokens) +} diff --git a/pkg/transform/tokscratch_test.go b/pkg/transform/tokscratch_test.go new file mode 100644 index 00000000..63091714 --- /dev/null +++ b/pkg/transform/tokscratch_test.go @@ -0,0 +1,267 @@ +// Copyright 2026 GoSQLX Authors +// SPDX-License-Identifier: Apache-2.0 + +package transform + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + + "github.com/ajitpratap0/GoSQLX/pkg/gosqlx" + "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" + "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" + "github.com/ajitpratap0/GoSQLX/pkg/sql/parser" + "github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer" +) + +// legacyScratchParse deliberately uses the public copying tokenizer API as an +// independent reference for all three scratch consumers in transform.go. +func legacyScratchParse(sql string, dialect keywords.SQLDialect, withDialect bool) (*ast.AST, error) { + tkz := tokenizer.GetTokenizer() + defer tokenizer.PutTokenizer(tkz) + if withDialect && dialect != "" { + tkz.SetDialect(dialect) + } + tokens, err := tkz.TokenizeString(sql) + if err != nil { + return nil, fmt.Errorf("tokenize: %w", err) + } + p := parser.GetParser() + defer parser.PutParser(p) + if withDialect { + p.ApplyOptions(parser.WithDialect(string(dialect))) + } + tree, err := p.ParseFromModelTokens(tokens) + if err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + return tree, nil +} + +type scratchCase struct { + name, sql string + dialect keywords.SQLDialect +} + +func scratchCorpus(t *testing.T) []scratchCase { + t.Helper() + var cases []scratchCase + for _, root := range []string{"../../testdata/realworld", "../../testdata/corpus"} { + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(path) != ".sql" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + dialect := keywords.DialectGeneric + for _, d := range keywords.AllDialects() { + if strings.Contains(filepath.ToSlash(path), "/"+string(d)+"/") { + dialect = d + break + } + } + for i, sql := range gosqlx.SplitStatements(string(data), string(dialect)) { + cases = append(cases, scratchCase{fmt.Sprintf("%s/%d", path, i), sql, dialect}) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + } + if len(cases) < 100 { + t.Fatalf("corpus unexpectedly small: %d", len(cases)) + } + for _, sql := range []string{"", "SELECT 'unterminated", "SELECT FROM", "SELECT first_name FROM first_table; SELECT second_name FROM second_table"} { + cases = append(cases, scratchCase{sql, sql, keywords.DialectGeneric}) + } + return cases +} + +type scratchResult struct { + output []string + err string +} + +func scratchRender(tree *ast.AST, err error) scratchResult { + if err != nil { + return scratchResult{err: err.Error()} + } + var result scratchResult + if tree != nil { + for _, stmt := range tree.Statements { + sql, err := FormatSQLChecked(stmt) + if err != nil { + result.output = append(result.output, "render error: "+err.Error()) + } else { + result.output = append(result.output, sql) + } + } + } + return result +} + +func scratchParseCase(c scratchCase, mode int, legacy bool) (*ast.AST, error) { + if legacy { + tree, err := legacyScratchParse(c.sql, c.dialect, mode == 1) + if mode != 2 || err != nil { + return tree, err + } + if tree == nil || len(tree.Statements) == 0 { + ast.ReleaseAST(tree) + return nil, fmt.Errorf("no statements parsed") + } + stmt := tree.Statements[0] + tree.Statements[0] = nil + ast.ReleaseAST(tree) + return &ast.AST{Statements: []ast.Statement{stmt}}, nil + } + switch mode { + case 0: + return ParseSQL(c.sql) + case 1: + return ParseSQLWithDialect(c.sql, c.dialect) + default: + stmt, err := parseSQL(c.sql) + if err != nil { + return nil, err + } + return &ast.AST{Statements: []ast.Statement{stmt}}, nil + } +} + +func TestTokenScratchCorpusDifferential(t *testing.T) { + for _, c := range scratchCorpus(t) { + for mode, name := range []string{"ParseSQL", "ParseSQLWithDialect", "parseSQL"} { + t.Run(c.name+"/"+name, func(t *testing.T) { + oldTree, oldErr := scratchParseCase(c, mode, true) + defer ast.ReleaseAST(oldTree) + newTree, newErr := scratchParseCase(c, mode, false) + defer ast.ReleaseAST(newTree) + want, got := scratchRender(oldTree, oldErr), scratchRender(newTree, newErr) + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v, want %#v", got, want) + } + }) + } + } +} + +func TestTokenScratchCorpusConcurrent(t *testing.T) { + cases := scratchCorpus(t) + sequential := make([][3]scratchResult, len(cases)) + for i, c := range cases { + for mode := range 3 { + tree, err := scratchParseCase(c, mode, false) + sequential[i][mode] = scratchRender(tree, err) + ast.ReleaseAST(tree) + } + } + var wg sync.WaitGroup + for worker := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for i, c := range cases { + for mode := range 3 { + tree, err := scratchParseCase(c, mode, false) + got := scratchRender(tree, err) + ast.ReleaseAST(tree) + if !reflect.DeepEqual(got, sequential[i][mode]) { + t.Errorf("worker %d %s mode %d: got %#v, want %#v", worker, c.name, mode, got, sequential[i][mode]) + return + } + } + } + }() + } + wg.Wait() +} + +func TestTokenScratchReturnedASTSurvivesReuse(t *testing.T) { + // Retain caller-owned nodes while later calls overwrite tokenizer scratch. + for mode, name := range []string{"ParseSQL", "ParseSQLWithDialect", "parseSQL"} { + t.Run(name, func(t *testing.T) { + c := scratchCase{sql: "SELECT retained_name, 'retained value' FROM retained_table WHERE retained_id = 123", dialect: keywords.DialectPostgreSQL} + tree, err := scratchParseCase(c, mode, false) + if err != nil { + t.Fatal(err) + } + defer ast.ReleaseAST(tree) + want := scratchRender(tree, nil) + for i := range 64 { + other, err := scratchParseCase(scratchCase{sql: fmt.Sprintf("SELECT overwritten_%d, 'replacement' FROM another_table WHERE another_id = 999", i), dialect: keywords.DialectMySQL}, i%3, false) + if err != nil { + t.Fatal(err) + } + ast.ReleaseAST(other) + } + if got := scratchRender(tree, nil); !reflect.DeepEqual(got, want) { + t.Fatalf("after reuse: got %#v, want %#v", got, want) + } + }) + } +} + +func TestTokenScratchExtractedNodesConcurrent(t *testing.T) { + // These two helpers return subnodes detached from parseSQL's statement. + // Retain those subnodes across calls on every worker as well. + const condition = "retained_id = 123 AND retained_name = 'kept'" + const join = "JOIN retained_table r ON r.retained_id = _t.retained_id" + makeTree := func() (*ast.AST, error) { + expr, err := parseCondition(condition) + if err != nil { + return nil, err + } + clause, err := parseJoinClause(join) + if err != nil { + return nil, err + } + return &ast.AST{Statements: []ast.Statement{&ast.SelectStatement{ + Columns: []ast.Expression{&ast.Identifier{Name: "retained_column"}}, + Where: expr, Joins: []ast.JoinClause{*clause}, + }}}, nil + } + baseline, err := makeTree() + if err != nil { + t.Fatal(err) + } + want := scratchRender(baseline, nil) + ast.ReleaseAST(baseline) + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + retained, err := makeTree() + if err != nil { + t.Error(err) + return + } + defer ast.ReleaseAST(retained) + for range 32 { + other, err := makeTree() + if err != nil { + t.Error(err) + return + } + ast.ReleaseAST(other) + } + if got := scratchRender(retained, nil); !reflect.DeepEqual(got, want) { + t.Errorf("retained condition/join after reuse: got %#v, want %#v", got, want) + } + }() + } + wg.Wait() +} diff --git a/pkg/transform/transform.go b/pkg/transform/transform.go index 254a590b..9935239b 100644 --- a/pkg/transform/transform.go +++ b/pkg/transform/transform.go @@ -17,6 +17,7 @@ package transform import ( "fmt" + "github.com/ajitpratap0/GoSQLX/internal/tokscratch" "github.com/ajitpratap0/GoSQLX/pkg/formatter" "github.com/ajitpratap0/GoSQLX/pkg/sql/ast" "github.com/ajitpratap0/GoSQLX/pkg/sql/keywords" @@ -115,7 +116,7 @@ func parseSQL(sql string) (ast.Statement, error) { tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return nil, fmt.Errorf("tokenize: %w", err) } @@ -210,7 +211,7 @@ func ParseSQL(sql string) (*ast.AST, error) { tkz := tokenizer.GetTokenizer() defer tokenizer.PutTokenizer(tkz) - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return nil, fmt.Errorf("tokenize: %w", err) } @@ -294,7 +295,7 @@ func ParseSQLWithDialect(sql string, dialect keywords.SQLDialect) (*ast.AST, err tkz.SetDialect(dialect) } - tokens, err := tkz.TokenizeString(sql) + tokens, err := tokscratch.String(tkz, sql) if err != nil { return nil, fmt.Errorf("tokenize: %w", err) } diff --git a/scripts/perf/gate-suite.txt b/scripts/perf/gate-suite.txt index 21cfb33e..f9abca28 100644 --- a/scripts/perf/gate-suite.txt +++ b/scripts/perf/gate-suite.txt @@ -13,3 +13,5 @@ ./pkg/gosqlx ^BenchmarkParseFresh$/^(simple_select|join|complex)$ ./pkg/gosqlx ^BenchmarkPoolGetPut$/^serial$ ./pkg/sql/tokenizer ^BenchmarkTokenize$/^scratch$/^(join|complex|insert)$ +./pkg/fingerprint ^BenchmarkNormalize$/^(simple_select|join|complex)$ +./pkg/transform ^BenchmarkTransformParse$/^(simple_select|join|complex)$