Keep bad type rendering balanced in type error recovery

entry_recovery_test.go

@@ -302,3 +302,27 @@ func requireSafeEntryNodes(t *testing.T, nodes []ast.Node) {
 		}()
 	}
 }
+
+func TestParseTypeRecoveryKeepsBalancedBrackets(t *testing.T) {
+	tests := []struct {
+		input string
+		want  string
+	}{
+		// ">>" is split between a bad inner type and its enclosing type; the
+		// first ">" must stay in the bad type so its rendering is balanced.
+		{input: "ARRAY<STRUCT<x !, y INT64>>", want: "ARRAY<STRUCT<x !, y INT64>>"},
+		{input: "STRUCT<x STRUCT<y !>>", want: "STRUCT<x STRUCT<y !>>"},
+		// Parentheses inside a bad type are tracked as nesting instead of
+		// unconditionally ending it.
+		{input: "ARRAY<STRING(!)>", want: "ARRAY<STRING(!)>"},
+	}
+	for _, test := range tests {
+		typ, err := memefish.ParseType("", test.input)
+		if err == nil {
+			t.Errorf("ParseType(%q) error = nil, want an error", test.input)
+		}
+		if got := typ.SQL(); got != test.want {
+			t.Errorf("ParseType(%q).SQL() = %q, want %q", test.input, got, test.want)
+		}
+	}
+}

parser.go

@@ -7044,11 +7044,11 @@ func (p *Parser) handleParseTypeError(r any, l *Lexer) *ast.BadType {
 skip:
 	for p.Token.Kind != token.TokenEOF {
 		switch p.Token.Kind {
-		case ";", ")":
+		case ";":
 			break skip
-		case "<":
+		case "<", "(":
 			nesting += 1
-		case ">":
+		case ">", ")":
 			if nesting == 0 {
 				break skip
 			}
@@ -7058,7 +7058,17 @@ skip:
 				break skip
 			}
 			if nesting == 1 {
+				// The first ">" of ">>" closes this bad type, so consume it and
+				// leave the second ">" for the enclosing type.
+				gt := p.Token.Clone()
+				gt.Kind = ">"
+				gt.Raw = ">"
+				gt.End = gt.Pos + 1
+				tokens = append(tokens, gt)
+				end = gt.End
 				p.Token.Kind = ">"
+				p.Token.Raw = ">"
+				p.Token.Space = ""
 				p.Token.Pos += 1
 				break skip
 			}

parser_test.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 
@@ -20,6 +21,33 @@ import (
 
 var update = flag.Bool("update", false, "update result files")
 
+func errorMessages(err error) []string {
+	list, ok := err.(memefish.MultiError)
+	if !ok {
+		return nil
+	}
+	msgs := make([]string, 0, len(list))
+	for _, e := range list {
+		msgs = append(msgs, e.Message)
+	}
+	return msgs
+}
+
+func errorMessagesWithin(err error, input string, end token.Pos) []string {
+	list, ok := err.(memefish.MultiError)
+	if !ok {
+		return nil
+	}
+	trailingJunk := strings.TrimSpace(input[min(int(end), len(input)):]) != ""
+	msgs := make([]string, 0, len(list))
+	for _, e := range list {
+		if e.Position.Pos <= end || !trailingJunk {
+			msgs = append(msgs, e.Message)
+		}
+	}
+	return msgs
+}
+
 type pathVisitor struct {
 	f    func(path string, node ast.Node) bool
 	path string
@@ -93,6 +121,7 @@ func testParser(t *testing.T, inputPath, resultPath string, parse func(p *memefi
 			}
 
 			node, err := parse(p)
+			parseErr := err
 			if !bad && node == nil {
 				t.Fatal("parser returned a nil AST without an expected parse error")
 			}
@@ -233,12 +262,20 @@ func testParser(t *testing.T, inputPath, resultPath string, parse func(p *memefi
 				},
 			}
 
-			node1, _ := parse(p1)
+			node1, err1 := parse(p1)
 
 			s2 := node1.SQL()
 			if s1 != s2 {
 				t.Errorf("%q != %q", s1, s2)
 			}
+
+			// The unparsed source can only witness errors located within the
+			// node; an error past its end (trailing tokens) is out of scope.
+			want := errorMessagesWithin(parseErr, string(b), node.End())
+			got := errorMessages(err1)
+			if !slices.Equal(want, got) {
+				t.Errorf("error mismatch on re-parsing the unparsed source\nsource: %q\nwant errors: %q\ngot errors: %q", s1, want, got)
+			}
 		})
 	}
 }

testdata/input/dml/!bad_trailing_token_dml.sql

@@ -0,0 +1 @@
+DELETE FROM t WHERE TRUE !

testdata/input/expr/!bad_trailing_token_expr.sql

@@ -0,0 +1 @@
+1 + 2 !

testdata/input/gql/!bad_trailing_token_gql.sql

@@ -0,0 +1 @@
+GRAPH g MATCH (n) RETURN n !

testdata/input/query/!bad_trailing_token_query.sql

@@ -0,0 +1 @@
+SELECT 1 !

testdata/result/dml/!bad_trailing_token_dml.sql.txt

@@ -0,0 +1,31 @@
+--- !bad_trailing_token_dml.sql
+DELETE FROM t WHERE TRUE !
+
+--- Error
+syntax error: testdata/input/dml/!bad_trailing_token_dml.sql:1:26: expected token: <eof>, but: !
+  1|  DELETE FROM t WHERE TRUE !
+   |                           ^
+
+
+--- AST
+&ast.Delete{
+  TableName: &ast.Path{
+    Idents: []*ast.Ident{
+      &ast.Ident{
+        NamePos: 12,
+        NameEnd: 13,
+        Name:    "t",
+      },
+    },
+  },
+  Where: &ast.Where{
+    Where: 14,
+    Expr:  &ast.BoolLiteral{
+      ValuePos: 20,
+      Value:    true,
+    },
+  },
+}
+
+--- SQL
+DELETE FROM t WHERE TRUE

testdata/result/expr/!bad_trailing_token_expr.sql.txt

@@ -0,0 +1,27 @@
+--- !bad_trailing_token_expr.sql
+1 + 2 !
+
+--- Error
+syntax error: testdata/input/expr/!bad_trailing_token_expr.sql:1:7: expected token: <eof>, but: !
+  1|  1 + 2 !
+   |        ^
+
+
+--- AST
+&ast.BinaryExpr{
+  Op:   "+",
+  Left: &ast.IntLiteral{
+    ValueEnd: 1,
+    Base:     10,
+    Value:    "1",
+  },
+  Right: &ast.IntLiteral{
+    ValuePos: 4,
+    ValueEnd: 5,
+    Base:     10,
+    Value:    "2",
+  },
+}
+
+--- SQL
+1 + 2

testdata/result/gql/!bad_trailing_token_gql.sql.txt

@@ -0,0 +1,75 @@
+--- !bad_trailing_token_gql.sql
+GRAPH g MATCH (n) RETURN n !
+
+--- Error
+syntax error: testdata/input/gql/!bad_trailing_token_gql.sql:1:28: expected token: <eof>, but: !
+  1|  GRAPH g MATCH (n) RETURN n !
+   |                             ^
+
+
+--- AST
+&ast.GQLGraphQuery{
+  GraphClause: &ast.GQLGraphClause{
+    PropertyGraphName: &ast.Path{
+      Idents: []*ast.Ident{
+        &ast.Ident{
+          NamePos: 6,
+          NameEnd: 7,
+          Name:    "g",
+        },
+      },
+    },
+  },
+  Query: &ast.GQLMultiLinearQueryStatement{
+    Statements: []ast.GQLLinearQueryStatement{
+      &ast.GQLSimpleLinearQueryStatement{
+        Statements: []ast.GQLPrimitiveQueryStatement{
+          &ast.GQLMatch{
+            Optional: -1,
+            Match:    8,
+            Pattern:  &ast.GQLGraphPattern{
+              Paths: []*ast.GQLTopLevelPathPattern{
+                &ast.GQLTopLevelPathPattern{
+                  Path: &ast.GQLPathPattern{
+                    Terms: []*ast.GQLPathTerm{
+                      &ast.GQLPathTerm{
+                        Primary: &ast.GQLNodePattern{
+                          Lparen:  14,
+                          Rparen:  16,
+                          Pattern: &ast.GQLElementPatternFiller{
+                            EmptyPos: -1,
+                            Variable: &ast.Ident{
+                              NamePos: 15,
+                              NameEnd: 16,
+                              Name:    "n",
+                            },
+                          },
+                        },
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+          &ast.GQLReturn{
+            Return: 18,
+            Items:  []*ast.GQLReturnItem{
+              &ast.GQLReturnItem{
+                Star: -1,
+                Expr: &ast.Ident{
+                  NamePos: 25,
+                  NameEnd: 26,
+                  Name:    "n",
+                },
+              },
+            },
+          },
+        },
+      },
+    },
+  },
+}
+
+--- SQL
+GRAPH g MATCH (n) RETURN n

testdata/result/query/!bad_trailing_token_query.sql.txt

@@ -0,0 +1,27 @@
+--- !bad_trailing_token_query.sql
+SELECT 1 !
+
+--- Error
+syntax error: testdata/input/query/!bad_trailing_token_query.sql:1:10: expected token: <eof>, but: !
+  1|  SELECT 1 !
+   |           ^
+
+
+--- AST
+&ast.QueryStatement{
+  Query: &ast.Select{
+    Results: []ast.SelectItem{
+      &ast.ExprSelectItem{
+        Expr: &ast.IntLiteral{
+          ValuePos: 7,
+          ValueEnd: 8,
+          Base:     10,
+          Value:    "1",
+        },
+      },
+    },
+  },
+}
+
+--- SQL
+SELECT 1

testdata/result/statement/!bad_trailing_token_dml.sql.txt

@@ -0,0 +1,31 @@
+--- !bad_trailing_token_dml.sql
+DELETE FROM t WHERE TRUE !
+
+--- Error
+syntax error: testdata/input/dml/!bad_trailing_token_dml.sql:1:26: expected token: <eof>, but: !
+  1|  DELETE FROM t WHERE TRUE !
+   |                           ^
+
+
+--- AST
+&ast.Delete{
+  TableName: &ast.Path{
+    Idents: []*ast.Ident{
+      &ast.Ident{
+        NamePos: 12,
+        NameEnd: 13,
+        Name:    "t",
+      },
+    },
+  },
+  Where: &ast.Where{
+    Where: 14,
+    Expr:  &ast.BoolLiteral{
+      ValuePos: 20,
+      Value:    true,
+    },
+  },
+}
+
+--- SQL
+DELETE FROM t WHERE TRUE