perf(parser): size joins slice from a JOIN-count hint [W4-03]

CHANGELOG.md

@@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     are removed.
 
 ### Performance
+- **Parser joins slice sized from a JOIN lookahead [W4-03]**: when the first JOIN of a FROM clause is appended, `parseJoinClauses` counts the JOIN tokens that follow at the same paren depth (within 256 tokens, stopping at `;`, UNION/EXCEPT/INTERSECT or the closing `)`) and allocates the slice with capacity 1 + that count (max 8) instead of a fixed 2. Join-free statements pay nothing. `ParseE2E/join` and `complex` B/op drop about 20%; no API or AST change. A full slice handed in by a parenthesized join group or a comma table is grown by the same hint. Known edge cost: shapes the lookahead cannot see (`(a JOIN b) JOIN c`, `a JOIN b, c`) incur one extra allocation per statement; the gated suite is unaffected.
 - **`ast.ReleaseAST` is O(1) for parser-built trees that hold no pooled nodes**: the parser counts the nodes it draws from the AST pools and tells
   `ReleaseAST` through an internal hook (`pkg/sql/internal/astrelease`). When
   the count is zero, which covers ordinary SELECT/INSERT/UPDATE/DELETE

pkg/sql/parser/join_hint_test.go

@@ -0,0 +1,302 @@
+// Copyright 2026 GoSQLX Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/ajitpratap0/GoSQLX/pkg/models"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/ast"
+	"github.com/ajitpratap0/GoSQLX/pkg/sql/tokenizer"
+)
+
+func joinHintTokens(t *testing.T, sql string) []models.TokenWithSpan {
+	t.Helper()
+	tkz := tokenizer.GetTokenizer()
+	defer tokenizer.PutTokenizer(tkz)
+	toks, err := tkz.Tokenize([]byte(sql))
+	if err != nil {
+		t.Fatalf("tokenize %q: %v", sql, err)
+	}
+	return preprocessTokens(toks)
+}
+
+func joinsOf(t *testing.T, sql string) []ast.JoinClause {
+	t.Helper()
+	p := &Parser{}
+	tree, err := p.parseTokens(joinHintTokens(t, sql))
+	if err != nil {
+		t.Fatalf("parse %q: %v", sql, err)
+	}
+	sel, ok := tree.Statements[0].(*ast.SelectStatement)
+	if !ok {
+		t.Fatalf("expected SelectStatement, got %T", tree.Statements[0])
+	}
+	return sel.Joins
+}
+
+func joinQuery(n int) string {
+	var b strings.Builder
+	b.WriteString("SELECT * FROM t0")
+	for i := 1; i <= n; i++ {
+		fmt.Fprintf(&b, " JOIN t%d ON t%d.id = t0.id", i, i)
+	}
+	return b.String()
+}
+
+// The initial joins capacity is 1 plus the JOINs ahead, capped at 8, and the
+// parsed joins are complete and ordered for every count, including above the cap.
+func TestJoinsSliceCapacityAndContent(t *testing.T) {
+	for _, n := range []int{1, 2, 3, 5, 8, 9, 12} {
+		joins := joinsOf(t, joinQuery(n))
+		if len(joins) != n {
+			t.Fatalf("n=%d: got %d joins", n, len(joins))
+		}
+		for i, j := range joins {
+			if want := fmt.Sprintf("t%d", i+1); j.Right.Name != want {
+				t.Errorf("n=%d join %d: right=%q want %q", n, i, j.Right.Name, want)
+			}
+		}
+		wantCap := min(n, 8)
+		if n <= 8 && cap(joins) != wantCap {
+			t.Errorf("n=%d: cap=%d want %d", n, cap(joins), wantCap)
+		}
+		if n > 8 && cap(joins) < n {
+			t.Errorf("n=%d: cap=%d below len", n, cap(joins))
+		}
+	}
+}
+
+func TestJoinsCapWindowAndCap(t *testing.T) {
+	p := &Parser{}
+	p.tokens = joinHintTokens(t, joinQuery(20))
+	p.currentPos = 0
+	if got := p.joinsCap(); got != 8 {
+		t.Errorf("cap=%d want 8", got)
+	}
+	// Past the end of the stream there is nothing to look at.
+	p.currentPos = len(p.tokens)
+	if got := p.joinsCap(); got != 1 {
+		t.Errorf("cap=%d want 1 at end of stream", got)
+	}
+}
+
+// A join-free statement among join statements must not be affected, and a
+// multi-statement script parses identically statement by statement.
+func TestJoinsCapMultiStatement(t *testing.T) {
+	script := "SELECT * FROM a JOIN b ON a.i=b.i; SELECT 1; SELECT * FROM c JOIN d ON c.i=d.i JOIN e ON e.i=d.i"
+	p := &Parser{}
+	tree, err := p.parseTokens(joinHintTokens(t, script))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []int{1, 0, 2}
+	if len(tree.Statements) != len(want) {
+		t.Fatalf("got %d statements", len(tree.Statements))
+	}
+	for i, st := range tree.Statements {
+		sel, ok := st.(*ast.SelectStatement)
+		if !ok {
+			t.Fatalf("stmt %d: %T", i, st)
+		}
+		if len(sel.Joins) != want[i] {
+			t.Errorf("stmt %d: %d joins want %d", i, len(sel.Joins), want[i])
+		}
+	}
+}
+
+func selectStatements(t *testing.T, sql string) []*ast.SelectStatement {
+	t.Helper()
+	p := &Parser{}
+	tree, err := p.parseTokens(joinHintTokens(t, sql))
+	if err != nil {
+		t.Fatalf("parse %q: %v", sql, err)
+	}
+	var out []*ast.SelectStatement
+	var walk func(st ast.Statement)
+	walk = func(st ast.Statement) {
+		switch s := st.(type) {
+		case *ast.SelectStatement:
+			out = append(out, s)
+			for _, f := range s.From {
+				if f.Subquery != nil {
+					walk(f.Subquery)
+				}
+			}
+		case *ast.SetOperation:
+			walk(s.Left)
+			walk(s.Right)
+		}
+	}
+	for _, st := range tree.Statements {
+		walk(st)
+	}
+	return out
+}
+
+// The hint must not look past the current statement, set-operation branch or
+// subquery: every one-join SELECT below must get capacity exactly 1.
+func TestJoinsCapStopsAtBoundaries(t *testing.T) {
+	cases := []struct {
+		name string
+		sql  string
+		want []int // cap of Joins for each SELECT with joins, in order
+	}{
+		{"script of single-join statements",
+			strings.Repeat("SELECT * FROM a JOIN b ON a.i=b.i; ", 5), []int{1, 1, 1, 1, 1}},
+		{"union branches",
+			"SELECT a.x FROM a JOIN b ON a.i=b.i UNION ALL SELECT c.x FROM c JOIN d ON c.i=d.i", []int{1, 1}},
+		{"except branches",
+			"SELECT a.x FROM a JOIN b ON a.i=b.i EXCEPT SELECT c.x FROM c JOIN d ON c.i=d.i JOIN e ON e.i=d.i", []int{1, 2}},
+		{"intersect branches",
+			"SELECT a.x FROM a JOIN b ON a.i=b.i INTERSECT SELECT c.x FROM c JOIN d ON c.i=d.i", []int{1, 1}},
+		{"mixed one and two join statements",
+			"SELECT * FROM a JOIN b ON a.i=b.i; SELECT * FROM c JOIN d ON c.i=d.i JOIN e ON e.i=d.i; SELECT * FROM f JOIN g ON f.i=g.i", []int{1, 2, 1}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			var got []int
+			for _, s := range selectStatements(t, tc.sql) {
+				if len(s.Joins) > 0 {
+					got = append(got, cap(s.Joins))
+				}
+			}
+			if fmt.Sprint(got) != fmt.Sprint(tc.want) {
+				t.Errorf("caps=%v want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// A subquery's join must not count the outer query's later joins, and the
+// outer query must not count joins nested in subqueries.
+func TestJoinsCapSubqueryDepth(t *testing.T) {
+	sql := "SELECT * FROM (SELECT * FROM a JOIN b ON a.i=b.i) s JOIN c ON c.i=s.i JOIN d ON d.i=c.i"
+	var inner, outer *ast.SelectStatement
+	for _, s := range selectStatements(t, sql) {
+		if len(s.Joins) == 2 {
+			outer = s
+		} else if len(s.Joins) == 1 {
+			inner = s
+		}
+	}
+	if inner == nil || outer == nil {
+		t.Fatalf("did not find inner and outer selects (inner=%v outer=%v)", inner != nil, outer != nil)
+	}
+	if cap(inner.Joins) != 1 {
+		t.Errorf("subquery joins cap=%d want 1", cap(inner.Joins))
+	}
+	if cap(outer.Joins) != 2 {
+		t.Errorf("outer joins cap=%d want 2", cap(outer.Joins))
+	}
+
+	// Joins nested in a subquery that sits after the first join (WHERE, or a
+	// derived table in a later join) must not count toward the outer cap.
+	for _, tc := range []struct {
+		sql  string
+		want int
+	}{
+		{"SELECT * FROM a JOIN b ON a.i=b.i WHERE a.x IN (SELECT x FROM c JOIN d ON c.i=d.i JOIN e ON e.i=d.i)", 1},
+		{"SELECT * FROM a JOIN b ON a.i=b.i JOIN (SELECT * FROM c JOIN d ON c.i=d.i JOIN e ON e.i=d.i) s ON s.i=a.i", 2},
+	} {
+		found := false
+		for _, s := range selectStatements(t, tc.sql) {
+			if s.Joins != nil && s.Joins[0].Left.Name == "a" {
+				found = true
+				if cap(s.Joins) != tc.want {
+					t.Errorf("%s: outer cap=%d want %d", tc.sql, cap(s.Joins), tc.want)
+				}
+			}
+		}
+		if !found {
+			t.Errorf("%s: outer select not found", tc.sql)
+		}
+	}
+}
+
+// A parenthesized join group hands its slice to the outer join loop; the outer
+// joins must grow it by the hint rather than reallocating, so the capacity is
+// reached in at most one extra growth and never exceeds the join count rounded
+// to the hint.
+func TestJoinsCapParenGroupThenOuter(t *testing.T) {
+	for _, tc := range []struct {
+		sql    string
+		joins  int
+		maxCap int
+	}{
+		{"SELECT * FROM (a JOIN b ON a.i=b.i) JOIN c ON c.i=a.i", 2, 3},
+		{"SELECT * FROM (a JOIN b ON a.i=b.i) JOIN c ON c.i=a.i JOIN d ON d.i=a.i", 3, 4},
+	} {
+		got := joinsOf(t, tc.sql)
+		if len(got) != tc.joins {
+			t.Fatalf("%s: joins=%d want %d", tc.sql, len(got), tc.joins)
+		}
+		if cap(got) > tc.maxCap {
+			t.Errorf("%s: cap=%d want <= %d", tc.sql, cap(got), tc.maxCap)
+		}
+	}
+}
+
+// A full slice handed in by a parenthesized group must be re-sized from the
+// hint of the joins ahead in one step. Without that, append doubles the cap
+// repeatedly (1 -> 2 -> 4 -> 8 for the four outer joins below) and the final
+// capacity overshoots the join count. Guards the len(joins) == cap(joins)
+// condition in parseJoinClauses against regressing to cap(joins) == 0.
+func TestJoinsFullSliceRegrownFromHint(t *testing.T) {
+	sql := "SELECT * FROM (a JOIN b ON a.i=b.i) JOIN c ON c.i=a.i JOIN d ON d.i=a.i JOIN e ON e.i=a.i JOIN f ON f.i=a.i"
+	got := joinsOf(t, sql)
+	if len(got) != 5 {
+		t.Fatalf("joins=%d want 5", len(got))
+	}
+	// Grow(1 slot, hint 4) reserves 5 slots; allocator rounding may add a few
+	// but never reaches the 8 produced by repeated doubling.
+	if cap(got) >= 8 {
+		t.Errorf("cap=%d: full slice was doubled by append instead of grown from the hint", cap(got))
+	}
+}
+
+// Known trade-off: a comma table after a single JOIN appends to the cap-1
+// slice, costing one growth. Content must still be correct.
+func TestJoinsCommaAfterJoin(t *testing.T) {
+	got := joinsOf(t, "SELECT * FROM a JOIN b ON a.i=b.i, c")
+	if len(got) != 2 || !got[1].Comma || got[1].Type != "CROSS" {
+		t.Fatalf("unexpected joins: %+v", got)
+	}
+}
+
+// BenchmarkParseManyShortJoinStatements keeps the multi-statement trade-off
+// visible: each statement has one JOIN, so each joins slice must stay cap 1.
+func BenchmarkParseManyShortJoinStatements(b *testing.B) {
+	sql := strings.Repeat("SELECT * FROM a JOIN b ON a.i=b.i; ", 50)
+	tkz := tokenizer.GetTokenizer()
+	toks, err := tkz.Tokenize([]byte(sql))
+	if err != nil {
+		b.Fatal(err)
+	}
+	pt := preprocessTokens(toks)
+	tokenizer.PutTokenizer(tkz)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		p := &Parser{}
+		tree, err := p.parseTokens(pt)
+		if err != nil {
+			b.Fatal(err)
+		}
+		ast.ReleaseAST(tree)
+	}
+}

pkg/sql/parser/select_clauses.go

@@ -20,6 +20,7 @@ package parser
 
 import (
 	"fmt"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -409,8 +410,11 @@ func (p *Parser) parseJoinClauses(firstRef ast.TableReference, joins []ast.JoinC
 			}
 		}
 
-		if cap(joins) == 0 {
-			joins = make([]ast.JoinClause, 0, 2)
+		if len(joins) == cap(joins) {
+			// Empty slice: size it from the JOINs ahead. Full slice handed in
+			// by a caller (parenthesized group, comma join): grow by the hint
+			// once instead of letting append double repeatedly.
+			joins = slices.Grow(joins, p.joinsCap())
 		}
 		jc.Type = joinType
 		jc.Left = leftTable
@@ -1255,3 +1259,44 @@ func (p *Parser) isOrderByAll() bool {
 	}
 	return equalsUpper(p.peekToken().Token.Value, "NULLS")
 }
+
+// joinHintWindow bounds how far joinsCap looks ahead for JOIN tokens, so the
+// scan stays O(1) per statement on huge scripts.
+const joinHintWindow = 256
+
+// joinsCap returns the initial capacity for a statement's joins slice: 1 plus
+// the number of JOIN tokens that follow at the same parenthesis depth, capped
+// at 8. The scan stops at a depth-0 ';', UNION, EXCEPT or INTERSECT, or at the
+// ')' closing the enclosing level, so neighbouring statements, set-operation
+// branches and nested subqueries never inflate the hint. It also looks at most
+// joinHintWindow tokens ahead. It runs only when a JOIN is about to be
+// appended to an empty or full slice (a full one is handed in by a
+// parenthesized join group or a comma join), so join-free statements pay
+// nothing. The result
+// is a capacity hint only; append still grows past it. Oracle MINUS is
+// tokenized as an identifier, so it is not a boundary and a MINUS chain may
+// over-count later branches' JOINs (bounded by the cap and window).
+func (p *Parser) joinsCap() int {
+	end := min(p.currentPos+joinHintWindow, len(p.tokens))
+	n, depth := 1, 0 // n counts the join being appended; its JOIN token is already consumed
+	for i := p.currentPos; i < end && n < 8; i++ {
+		switch p.tokens[i].Token.Type {
+		case models.TokenTypeJoin:
+			if depth == 0 {
+				n++
+			}
+		case models.TokenTypeLParen:
+			depth++
+		case models.TokenTypeRParen:
+			if depth == 0 {
+				return n
+			}
+			depth--
+		case models.TokenTypeSemicolon, models.TokenTypeUnion, models.TokenTypeExcept, models.TokenTypeIntersect:
+			if depth == 0 {
+				return n
+			}
+		}
+	}
+	return n
+}