perf(fingerprint,formatter,transform): tokenize into the pooled scratch buffer [W5-01]

CHANGELOG.md

@@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Tokenizer**: keyword and compound-keyword-start lookup now uses a single fused-hash open-addressed table (one FNV-1a hash computed while upper-casing, one probe) instead of two Go map lookups per identifier. The table is filled in a fixed order (hot keywords first, then sorted) so its probe layout is deterministic across processes. No behaviour change; token output is identical [W2-01].
 - **Pools**: the tokenizer, parser and AST pools now count Get/Put through precreated, cache-line padded atomic counters (`internal/poolstats`) instead of an RWMutex-guarded map keyed by pool name, removing a lock, map lookup and string hash from every pool operation. `metrics.GetPoolStats` and `metrics.ResetPoolStats` behave exactly as before; named custom pools keep the map path. [W1-01]
 - **Parser**: token preprocessing now takes a type-gated fast path. Only identifier, keyword, aggregate and compound tokens take the normalisation path; every other token is appended with a single copy. Output is identical (guarded by an old-versus-new differential test); no API or behaviour change.
+- **Fingerprint, formatter, transform**: `fingerprint.Normalize`/`Fingerprint`, `Formatter.Format` and `transform.ParseSQL`/`ParseSQLWithDialect` tokenize through the internal pooled token scratch instead of the public `TokenizeString`. `Normalize` allocates 42-70% fewer bytes/op and `transform.ParseSQL` 49-78% fewer, with 2-3 fewer allocs/op and no output change. [W5-01]
 
 ### Fixed
 - **AST**: `AlterTableStatement.SQL()` now separates multiple ALTER TABLE actions with commas, matching the compact formatter and preserving action boundaries when re-parsed.

pkg/fingerprint/crossconsumer_race_test.go

@@ -0,0 +1,81 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package fingerprint_test
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/fingerprint"
+	"github.com/ajitpratap0/GoSQLX/pkg/formatter"
+	"github.com/ajitpratap0/GoSQLX/pkg/gosqlx"
+	"github.com/ajitpratap0/GoSQLX/pkg/transform"
+)
+
+// TestCrossConsumerSharedTokenizerPool runs every consumer of the shared
+// tokenizer pool and its pooled token scratch (gosqlx.Parse, formatter.Format,
+// fingerprint.Normalize, transform.ParseSQL) in parallel and checks that each
+// result matches its single-goroutine baseline. Under -race this catches a
+// consumer that reads scratch memory after the tokenizer went back to the pool.
+func TestCrossConsumerSharedTokenizerPool(t *testing.T) {
+	queries := []string{
+		"SELECT id, name FROM users WHERE id = 42 AND status IN ('a','b')",
+		"SELECT u.id, o.total FROM users u JOIN orders o ON o.user_id = u.id WHERE o.total > 100 ORDER BY o.total DESC LIMIT 10",
+		"WITH t AS (SELECT a, SUM(b) AS s FROM x GROUP BY a) SELECT a, s FROM t WHERE s > 5 -- trailing comment",
+		"INSERT INTO t (a, b) VALUES (1, 'x'), (2, 'y')",
+		"UPDATE t SET a = 1, b = 'z' WHERE id = 7",
+	}
+	f := formatter.New(formatter.Options{})
+
+	type base struct{ norm, formatted string }
+	want := make([]base, len(queries))
+	for i, q := range queries {
+		n, err := fingerprint.Normalize(q)
+		if err != nil {
+			t.Fatalf("baseline Normalize(%q): %v", q, err)
+		}
+		s, err := f.Format(q)
+		if err != nil {
+			t.Fatalf("baseline Format(%q): %v", q, err)
+		}
+		want[i] = base{n, s}
+	}
+
+	const goroutines, iters = 16, 60
+	var wg sync.WaitGroup
+	errs := make(chan string, goroutines*iters)
+	for g := 0; g < goroutines; g++ {
+		wg.Add(1)
+		go func(g int) {
+			defer wg.Done()
+			for i := 0; i < iters; i++ {
+				idx := (g + i) % len(queries)
+				q := queries[idx]
+				switch (g + i) % 4 {
+				case 0:
+					if _, err := gosqlx.Parse(q); err != nil {
+						errs <- "gosqlx.Parse: " + err.Error()
+					}
+				case 1:
+					if got, err := f.Format(q); err != nil || got != want[idx].formatted {
+						errs <- "formatter.Format mismatch for " + q
+					}
+				case 2:
+					if got, err := fingerprint.Normalize(q); err != nil || got != want[idx].norm {
+						errs <- "fingerprint.Normalize mismatch for " + q
+					}
+				default:
+					if _, err := transform.ParseSQL(q); err != nil {
+						errs <- "transform.ParseSQL: " + err.Error()
+					}
+				}
+			}
+		}(g)
+	}
+	wg.Wait()
+	close(errs)
+	for e := range errs {
+		t.Error(e)
+	}
+}

pkg/fingerprint/fingerprint.go

@@ -42,6 +42,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/ajitpratap0/GoSQLX/internal/tokscratch"
 	"github.com/ajitpratap0/GoSQLX/pkg/formatter"
 	"github.com/ajitpratap0/GoSQLX/pkg/sql/ast"
 	"github.com/ajitpratap0/GoSQLX/pkg/sql/parser"
@@ -91,7 +92,7 @@ func Normalize(sql string) (string, error) {
 	tkz := tokenizer.GetTokenizer()
 	defer tokenizer.PutTokenizer(tkz)
 
-	tokens, err := tkz.TokenizeString(sql)
+	tokens, err := tokscratch.String(tkz, sql)
 	if err != nil {
 		return "", fmt.Errorf("fingerprint: tokenization failed: %w", err)
 	}

pkg/fingerprint/tokscratch_legacy_test.go

@@ -0,0 +1,55 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package fingerprint
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/formatter"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/ast"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/parser"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer"
+)
+
+// LegacyNormalizeForScratchTest preserves the public-tokenizer pipeline as a compatibility oracle.
+func LegacyNormalizeForScratchTest(sql string) (string, error) {
+	tkz := tokenizer.GetTokenizer()
+	defer tokenizer.PutTokenizer(tkz)
+
+	tokens, err := tkz.TokenizeString(sql)
+	if err != nil {
+		return "", fmt.Errorf("fingerprint: tokenization failed: %w", err)
+	}
+
+	p := parser.GetParser()
+	defer parser.PutParser(p)
+
+	astObj, err := p.ParseFromModelTokens(tokens)
+	if err != nil {
+		return "", fmt.Errorf("fingerprint: parsing failed: %w", err)
+	}
+	defer ast.ReleaseAST(astObj)
+
+	// Walk the AST and replace all non-placeholder literals with "?".
+	v := &literalNormalizer{}
+	for _, stmt := range astObj.Statements {
+		if err := ast.Walk(v, stmt); err != nil {
+			return "", fmt.Errorf("fingerprint: AST walk failed: %w", err)
+		}
+	}
+
+	// Format the mutated AST back to SQL using compact (single-line) style.
+	opts := ast.CompactStyle()
+	var parts []string
+	for _, stmt := range astObj.Statements {
+		formatted, err := formatter.FormatStatementChecked(stmt, opts)
+		if err != nil {
+			return "", fmt.Errorf("fingerprint: formatting failed: %w", err)
+		}
+		parts = append(parts, formatted)
+	}
+
+	return strings.Join(parts, "; "), nil
+}

pkg/fingerprint/tokscratch_test.go

@@ -0,0 +1,103 @@
+// Copyright 2026 GoSQLX Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package fingerprint_test
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/fingerprint"
+	"github.com/ajitpratap0/GoSQLX/pkg/gosqlx"
+)
+
+type scratchResult struct{ normalized, fingerprint, err string }
+
+func scratchRun(sql string) scratchResult {
+	n, e := fingerprint.Normalize(sql)
+	f, fe := fingerprint.Fingerprint(sql)
+	if fmt.Sprint(e) != fmt.Sprint(fe) {
+		return scratchResult{n, f, "inconsistent errors"}
+	}
+	return scratchResult{n, f, fmt.Sprint(e)}
+}
+func scratchCorpus(t *testing.T) []string {
+	t.Helper()
+	var queries []string
+	for _, root := range []string{"../../testdata/realworld", "../../testdata/corpus"} {
+		err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if d.IsDir() || filepath.Ext(path) != ".sql" {
+				return nil
+			}
+			data, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+			queries = append(queries, string(data))
+			queries = append(queries, gosqlx.SplitStatements(string(data), "")...)
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if len(queries) == 0 {
+		t.Fatal("empty corpus")
+	}
+	return queries
+}
+func TestScratchDifferential(t *testing.T) {
+	for i, sql := range scratchCorpus(t) {
+		want, err := fingerprint.LegacyNormalizeForScratchTest(sql)
+		got := scratchRun(sql)
+		hash := ""
+		if err == nil {
+			hash = fmt.Sprintf("%x", sha256.Sum256([]byte(want)))
+		}
+		if got != (scratchResult{want, hash, fmt.Sprint(err)}) {
+			t.Fatalf("query %d mismatch: got %#v, want %q, %q, %v", i, got, want, hash, err)
+		}
+	}
+}
+func TestScratchConcurrentAndRetainedResults(t *testing.T) {
+	queries := scratchCorpus(t)
+	want := make([]scratchResult, len(queries))
+	for i, q := range queries {
+		result := scratchRun(q)
+		want[i] = scratchResult{strings.Clone(result.normalized), strings.Clone(result.fingerprint), strings.Clone(result.err)}
+	}
+	var wg sync.WaitGroup
+	for worker := 0; worker < 8; worker++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			retained := make([]scratchResult, len(queries))
+			for i, q := range queries {
+				retained[i] = scratchRun(q)
+				if retained[i] != want[i] {
+					t.Errorf("concurrent query %d mismatch", i)
+					return
+				}
+			}
+			// Further calls recycle pooled token/Word storage while earlier strings stay live.
+			for _, q := range queries {
+				scratchRun(q)
+			}
+			for i, got := range retained {
+				if got != want[i] {
+					t.Errorf("retained query %d changed", i)
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}

pkg/formatter/formatter.go

@@ -93,6 +93,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/ajitpratap0/GoSQLX/internal/tokscratch"
 	"github.com/ajitpratap0/GoSQLX/pkg/models"
 	"github.com/ajitpratap0/GoSQLX/pkg/sql/ast"
 	"github.com/ajitpratap0/GoSQLX/pkg/sql/keywords"
@@ -139,7 +140,7 @@ func (f *Formatter) Format(sql string) (string, error) {
 		tkz.SetDialect(dialect)
 	}
 
-	tokens, err := tkz.TokenizeString(sql)
+	tokens, err := tokscratch.String(tkz, sql)
 	if err != nil {
 		return "", fmt.Errorf("tokenization failed: %w", err)
 	}