From 2d08f627d31df7447d0e7b46e8068deee1a86031 Mon Sep 17 00:00:00 2001 From: Zikani Nyirenda Mwase Date: Wed, 25 Mar 2026 02:50:23 +0200 Subject: [PATCH] feature: property based testing and fuzzing actions --- cmd/run.go | 1 + lib.go | 26 +++- playwright/actions.go | 39 +++++ playwright/playwright.go | 154 +++++++++++++++++-- playwright/playwright_test.go | 4 +- playwright/property_testing.go | 263 +++++++++++++++++++++++++++++++++ property_test_example.basi | 22 +++ 7 files changed, 490 insertions(+), 19 deletions(-) create mode 100644 playwright/property_testing.go create mode 100644 property_test_example.basi diff --git a/cmd/run.go b/cmd/run.go index b85d76b..4b3edc6 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -51,6 +51,7 @@ func (r *RunCmd) Run(globals *Globals) error { Browser: cmp.Or(parsed.GetMetaFieldString("Browsers"), globals.Browser), Headless: headless, Actions: actions, + Context: playwright.NewExecutionContext(), } } else if strings.HasSuffix(r.File, ".yaml") || strings.HasSuffix(r.File, ".yml") { diff --git a/lib.go b/lib.go index f9a197d..d993fac 100644 --- a/lib.go +++ b/lib.go @@ -80,6 +80,12 @@ var actions = []string{ "ExpectId", "ExpectJSProperty", "ExpectToMatchAriaSnapshot", + // Property-based testing actions + "Always", + "Fuzz", + "Extract", + "Eventually", + "Next", } func lexerActionsFromMap() string { @@ -95,6 +101,8 @@ var ( {`Action`, lexerActionsFromMap()}, {`MetaField`, `ID|Title|URL|Description|Headless|Browser`}, // TODO: support ScreenSizes, Extends {`Ident`, `[a-zA-Z][a-zA-Z_\d]*`}, + {`Variable`, `\$[a-zA-Z][a-zA-Z_\d]*`}, + {`Number`, `\d+`}, {`String`, `"(?:\\.|[^"])*"`}, {`Selector`, `"(?:\\.|[^"])*"`}, {"comment", `[#;][^\n]*`}, @@ -145,11 +153,27 @@ type MetaField struct { type Action struct { // Pos lexer.Position Action string `@Action` - Selector *Selector `@@` + Selector *Selector `[ @@ ]` Arguments *String `[ @@ ]` + Variable *Variable `[ @@ ]` + Number *Number `[ @@ ]` comment *string `@comment` } +type Variable struct { + // Pos lexer.Position + Variable string `@Variable` +} + +func (Variable) value() {} + +type Number struct { + // Pos lexer.Position + Number int `@Number` +} + +func (Number) value() {} + type String struct { // Pos lexer.Position String string `@String` diff --git a/playwright/actions.go b/playwright/actions.go index 691e6d5..fe22eb1 100644 --- a/playwright/actions.go +++ b/playwright/actions.go @@ -79,6 +79,12 @@ var actionMap = map[string]ActionFunc{ "ExpectId": StubExpectAction, "ExpectJSProperty": StubExpectAction, "ExpectToMatchAriaSnapshot": StubExpectAction, + // Property-based testing actions + "Always": AlwaysAction, + "Fuzz": FuzzAction, + "Extract": ExtractAction, + "Eventually": EventuallyAction, + "Next": NextAction, } func castOptions[Dest any](action *ExecutorAction) (dest *Dest, err error) { @@ -499,3 +505,36 @@ func UploadMultipleFilesAction(page playwrightgo.Page, action *ExecutorAction) e } return page.Locator(action.Selector).First().SetInputFiles(files, *opts) } + +// Property-based testing actions + +func AlwaysAction(page playwrightgo.Page, action *ExecutorAction) error { + // This action registers an invariant - it doesn't execute anything + // The invariant will be checked after every mutating action + // Implementation is handled in the execution context + return nil +} + +func FuzzAction(page playwrightgo.Page, action *ExecutorAction) error { + // This action performs random interactions within a scope + // Implementation is handled in the execution context with timeouts + return nil +} + +func ExtractAction(page playwrightgo.Page, action *ExecutorAction) error { + // This action extracts a value from the page and stores it in a variable + // Implementation is handled in the execution context + return nil +} + +func EventuallyAction(page playwrightgo.Page, action *ExecutorAction) error { + // This action polls until a condition is met + // Implementation is handled in the execution context with timeouts + return nil +} + +func NextAction(page playwrightgo.Page, action *ExecutorAction) error { + // This action asserts the immediate next state + // Implementation is handled in the execution context + return nil +} diff --git a/playwright/playwright.go b/playwright/playwright.go index 88f1c43..caae475 100644 --- a/playwright/playwright.go +++ b/playwright/playwright.go @@ -17,20 +17,23 @@ import ( const Name = "playwright" type Executor struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Description string `json:"description,omitempty" yaml:"description,omitempty"` - URL string `json:"url" yaml:"url"` - Browser string `json:"browser" yaml:"browser"` - Device string `json:"device" yaml:"device"` - Actions []ExecutorAction `json:"actions" yaml:"actions"` - Headless bool `json:"headless" yaml:"headless"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + URL string `json:"url" yaml:"url"` + Browser string `json:"browser" yaml:"browser"` + Device string `json:"device" yaml:"device"` + Actions []ExecutorAction `json:"actions" yaml:"actions"` + Headless bool `json:"headless" yaml:"headless"` + Context *ExecutionContext `json:"-" yaml:"-"` // Execution context for property-based testing } type ExecutorAction struct { - Action string `json:"action"` // The action to perform, must be a valid/supported action - Selector string `json:"selector" yaml:"selector"` // DOM selector or expression - Content string `json:"content,omitempty" yaml:"content"` // Content for actions that require it - Options any `json:"options,omitempty" yaml:"options"` // Options applicable to the given action + Action string `json:"action"` // The action to perform, must be a valid/supported action + Selector string `json:"selector" yaml:"selector"` // DOM selector or expression + Content string `json:"content,omitempty" yaml:"content"` // Content for actions that require it + Variable string `json:"variable,omitempty" yaml:"variable"` // Variable name for Extract + Number int `json:"number,omitempty" yaml:"number"` // Number for Fuzz step count + Options any `json:"options,omitempty" yaml:"options"` // Options applicable to the given action } func NewExecutorAction(act *basi.Action) *ExecutorAction { @@ -38,10 +41,20 @@ func NewExecutorAction(act *basi.Action) *ExecutorAction { if act.Arguments != nil { args = act.Arguments.String } + variable := "" + if act.Variable != nil { + variable = act.Variable.Variable + } + number := 0 + if act.Number != nil { + number = act.Number.Number + } return &ExecutorAction{ Action: act.Action, Selector: act.Selector.Selector, Content: args, + Variable: variable, + Number: number, Options: nil, } } @@ -125,9 +138,18 @@ func (e *Executor) Run(ctx context.Context) (interface{}, error) { if err != nil { return nil, fmt.Errorf("could not goto: %w", err) } + // Set current origin for fuzzing boundary checks + if e.Context == nil { + e.Context = NewExecutionContext() + } + if currentURL := page.URL(); currentURL != "" { + if u, err := url.Parse(currentURL); err == nil { + e.Context.CurrentOrigin = u.Scheme + "://" + u.Host + } + } } - err = performActions(ctx, page, e.Actions) + err = performActions(ctx, page, e.Actions, e.Context) if err != nil { return nil, err } @@ -162,19 +184,111 @@ func (e *Executor) Run(ctx context.Context) (interface{}, error) { }, nil } -func performActions(ctx context.Context, page playwrightgo.Page, actions []ExecutorAction) error { +func performActions(ctx context.Context, page playwrightgo.Page, actions []ExecutorAction, execCtx *ExecutionContext) error { + if execCtx == nil { + execCtx = NewExecutionContext() + } assertions := playwrightgo.NewPlaywrightAssertions() var lastLocator playwrightgo.Locator for i, action := range actions { if action.Action == "" { return fmt.Errorf("action cannot be empty, please specify an action") } - if action.Selector == "" { - return fmt.Errorf("selector cannot be empty, please specify a selector") - } actionName := action.Action + // Handle Always action - register invariant + if actionName == "Always" { + // The embedded expect action is in the Selector field + invariant := Invariant{ + Action: action.Selector, + Selector: "", + Content: "", + } + execCtx.Invariants.Invariants = append(execCtx.Invariants.Invariants, invariant) + continue + } + + // Handle Extract action + if actionName == "Extract" { + locator := page.Locator(action.Content) + textContent, err := locator.TextContent() + if err != nil { + return fmt.Errorf("failed to extract text from selector %s: %w", action.Content, err) + } + execCtx.Variables.Set(action.Selector, textContent) + continue + } + + // Handle Fuzz action + if actionName == "Fuzz" { + // Parse the arguments: "stepCount scopeSelector ignoreSelector" + parts := strings.Fields(action.Selector) + if len(parts) < 1 { + return fmt.Errorf("Fuzz action requires at least a step count") + } + stepCount := 10 // default + if len(parts) >= 1 { + if parsedCount, err := strconv.Atoi(parts[0]); err == nil { + stepCount = parsedCount + } + } + scopeSelector := ".body" // default + if len(parts) >= 2 { + scopeSelector = parts[1] + } + ignoreSelector := "" // default + if len(parts) >= 3 { + ignoreSelector = parts[2] + } + + fuzzAction := ExecutorAction{ + Action: "Fuzz", + Selector: scopeSelector, + Content: ignoreSelector, + Number: stepCount, + } + err := performFuzz(ctx, page, &fuzzAction, execCtx) + if err != nil { + return fmt.Errorf("fuzz action failed: %w", err) + } + continue + } + + // Handle Eventually action + if actionName == "Eventually" { + // The embedded expect action is in the Selector field + embeddedActionObj := ExecutorAction{ + Action: action.Selector, + Selector: "", + Content: "", + } + err := performEventually(ctx, page, &embeddedActionObj, execCtx, assertions) + if err != nil { + return fmt.Errorf("eventually action failed: %w", err) + } + continue + } + + // Handle Next action + if actionName == "Next" { + // The embedded expect action is in the Selector field + embeddedActionObj := ExecutorAction{ + Action: action.Selector, + Selector: "", + Content: "", + } + err := performNext(page, &embeddedActionObj, execCtx, assertions) + if err != nil { + return fmt.Errorf("next action failed: %w", err) + } + continue + } + + if action.Selector == "" && actionName != "Extract" && actionName != "Fuzz" { + return fmt.Errorf("selector cannot be empty for action %s, please specify a selector", actionName) + } + if strings.HasPrefix(actionName, "Find") { if loc, err := tryFindLocator(page, action); err != nil { return fmt.Errorf("failed to find a element on the page using: '%s'", cmp.Or(action.Selector, action.Content)) @@ -226,6 +340,14 @@ func performActions(ctx context.Context, page playwrightgo.Page, actions []Execu if actErr != nil { return actErr } + + // Check invariants after mutating actions + if IsMutatingAction(actionName) { + err := execCtx.CheckInvariants(page, assertions) + if err != nil { + return err + } + } } return nil } diff --git a/playwright/playwright_test.go b/playwright/playwright_test.go index f3c4ef5..989191c 100644 --- a/playwright/playwright_test.go +++ b/playwright/playwright_test.go @@ -85,7 +85,7 @@ func TestPerformActions(t *testing.T) { t.Error("failed to set testPage content") } - err = performActions(context.Background(), page, testActions) + err = performActions(context.Background(), page, testActions, NewExecutionContext()) if err != nil { t.Errorf("failed to test actions %v", err) } @@ -132,7 +132,7 @@ func TestPerformFindOnEmptyElement(t *testing.T) { t.Error("failed to set testPage content") } - err = performActions(context.Background(), page, testActions) + err = performActions(context.Background(), page, testActions, NewExecutionContext()) if err == nil { t.Errorf("expected error while performing actions") } diff --git a/playwright/property_testing.go b/playwright/property_testing.go new file mode 100644 index 0000000..291416e --- /dev/null +++ b/playwright/property_testing.go @@ -0,0 +1,263 @@ +package playwright + +import ( + "context" + "fmt" + "math/rand" + "net/url" + "regexp" + "strings" + "time" + + playwrightgo "github.com/playwright-community/playwright-go" +) + +// InvariantRegistry holds global invariants that must be checked after every mutating action +type InvariantRegistry struct { + Invariants []Invariant +} + +// Invariant represents a global assertion +type Invariant struct { + Action string + Selector string + Content string +} + +// VariableStore holds extracted variables during test execution +type VariableStore struct { + Variables map[string]interface{} +} + +// NewVariableStore creates a new variable store +func NewVariableStore() *VariableStore { + return &VariableStore{ + Variables: make(map[string]interface{}), + } +} + +// Set sets a variable value +func (vs *VariableStore) Set(name string, value interface{}) { + vs.Variables[name] = value +} + +// Get gets a variable value +func (vs *VariableStore) Get(name string) interface{} { + return vs.Variables[name] +} + +// InterpolateVariables replaces variable references in strings +func (vs *VariableStore) InterpolateVariables(input string) string { + re := regexp.MustCompile(`\$[a-zA-Z][a-zA-Z_\d]*`) + return re.ReplaceAllStringFunc(input, func(match string) string { + varName := strings.TrimPrefix(match, "$") + if value := vs.Get(varName); value != nil { + return fmt.Sprintf("%v", value) + } + return match // Keep original if not found + }) +} + +// ExecutionContext holds the state during test execution +type ExecutionContext struct { + Invariants *InvariantRegistry + Variables *VariableStore + CurrentOrigin string +} + +// NewExecutionContext creates a new execution context +func NewExecutionContext() *ExecutionContext { + return &ExecutionContext{ + Invariants: &InvariantRegistry{}, + Variables: NewVariableStore(), + } +} + +// IsMutatingAction checks if an action modifies state +func IsMutatingAction(action string) bool { + mutatingActions := []string{ + "Click", "DoubleClick", "Doubleclick", "Tap", "Clear", "Fill", + "Check", "Uncheck", "FillCheckbox", "Press", "PressSequentially", + "Select", "SelectOption", "SelectMultipleOptions", "Type", + "Goto", "GoBack", "GoForward", "Refresh", + } + for _, ma := range mutatingActions { + if action == ma { + return true + } + } + return false +} + +// CheckInvariants runs all registered invariants +func (ec *ExecutionContext) CheckInvariants(page playwrightgo.Page, assertions playwrightgo.PlaywrightAssertions) error { + for _, invariant := range ec.Invariants.Invariants { + action := ExecutorAction{ + Action: invariant.Action, + Selector: invariant.Selector, + Content: ec.Variables.InterpolateVariables(invariant.Content), + } + err := performAssertion(assertions, page.Locator(invariant.Selector), &action) + if err != nil { + return fmt.Errorf("invariant failed: %s %s %s: %w", invariant.Action, invariant.Selector, invariant.Content, err) + } + } + return nil +} + +// performFuzz executes bounded random interactions +func performFuzz(ctx context.Context, page playwrightgo.Page, action *ExecutorAction, execCtx *ExecutionContext) error { + stepCount := action.Number + if stepCount <= 0 { + stepCount = 10 // default + } + + scopeSelector := action.Selector + ignoreSelector := action.Content + + // Create a timeout context for the entire fuzz operation + fuzzCtx, cancel := context.WithTimeout(ctx, 30*time.Second) // 30 second max + defer cancel() + + for i := 0; i < stepCount; i++ { + select { + case <-fuzzCtx.Done(): + return fmt.Errorf("fuzz operation timed out") + default: + } + + // Find actionable elements within scope + scopeLocator := page.Locator(scopeSelector) + actionableElements := scopeLocator.Locator("input, button, a, select, textarea") + + // Exclude ignored elements + if ignoreSelector != "" { + actionableElements = actionableElements.Locator(fmt.Sprintf(":not(%s)", ignoreSelector)) + } + + count, err := actionableElements.Count() + if err != nil || count == 0 { + continue // No actionable elements found + } + + // Select random element + randomIndex := rand.Intn(count) + randomElement := actionableElements.Nth(randomIndex) + + // Check if element is visible and enabled + isVisible, err := randomElement.IsVisible() + if err != nil || !isVisible { + continue + } + + isEnabled, err := randomElement.IsEnabled() + if err != nil || !isEnabled { + continue + } + + // Determine element type and perform appropriate action + tagName, err := randomElement.Evaluate("el => el.tagName.toLowerCase()", nil) + if err != nil { + continue + } + tagNameStr, ok := tagName.(string) + if !ok { + continue + } + + switch tagNameStr { + case "input": + inputType, _ := randomElement.GetAttribute("type") + switch inputType { + case "checkbox", "radio": + randomElement.Check() + case "text", "password", "email", "search", "tel", "url": + randomText := generateRandomString(10) + randomElement.Fill(randomText) + default: + randomElement.Click() + } + case "button", "a": + randomElement.Click() + case "select": + randomElement.SelectOption(playwrightgo.SelectOptionValues{Values: &[]string{""}}) + case "textarea": + randomText := generateRandomString(20) + randomElement.Fill(randomText) + } + + // Check invariants after each random action + assertions := playwrightgo.NewPlaywrightAssertions() + err = execCtx.CheckInvariants(page, assertions) + if err != nil { + return fmt.Errorf("invariant failed during fuzz step %d: %w", i+1, err) + } + + // Check if we left the origin (security boundary) + currentURL := page.URL() + if currentURL != "" { + if u, err := url.Parse(currentURL); err == nil { + currentOrigin := u.Scheme + "://" + u.Host + if currentOrigin != execCtx.CurrentOrigin { + // Navigate back to original origin + page.GoBack() + break + } + } + } + } + + return nil +} + +// performEventually polls until a condition is met +func performEventually(ctx context.Context, page playwrightgo.Page, action *ExecutorAction, execCtx *ExecutionContext, assertions playwrightgo.PlaywrightAssertions) error { + eventuallyCtx, cancel := context.WithTimeout(ctx, 5*time.Second) // 5 second timeout + defer cancel() + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-eventuallyCtx.Done(): + return fmt.Errorf("eventually condition not met within timeout") + case <-ticker.C: + // Interpolate variables in the action content + interpolatedContent := execCtx.Variables.InterpolateVariables(action.Content) + + // Create a temporary action with interpolated content + tempAction := *action + tempAction.Content = interpolatedContent + + locator := page.Locator(action.Selector) + err := performAssertion(assertions, locator, &tempAction) + if err == nil { + return nil // Condition met + } + } + } +} + +// performNext asserts the immediate next state +func performNext(page playwrightgo.Page, action *ExecutorAction, execCtx *ExecutionContext, assertions playwrightgo.PlaywrightAssertions) error { + // Interpolate variables in the action content + interpolatedContent := execCtx.Variables.InterpolateVariables(action.Content) + + // Create a temporary action with interpolated content + tempAction := *action + tempAction.Content = interpolatedContent + + locator := page.Locator(action.Selector) + return performAssertion(assertions, locator, &tempAction) +} + +// generateRandomString creates a random string of given length +func generateRandomString(length int) string { + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, length) + for i := range b { + b[i] = charset[rand.Intn(len(charset))] + } + return string(b) +} diff --git a/property_test_example.basi b/property_test_example.basi new file mode 100644 index 0000000..08fbc78 --- /dev/null +++ b/property_test_example.basi @@ -0,0 +1,22 @@ +Title: "Property Based Testing Example" +URL: "https://example.com" +Headless: "yes" + +# Define global invariants +Always "ExpectVisible #main-content" +Always "ExpectAccessibleName main" + +# Extract initial state +Extract "$initialTitle" "h1" + +# Perform some actions +Click "#some-button" + +# Assert immediate next state +Next "ExpectVisible .notification" + +# Perform fuzz testing +Fuzz "5 .form-container .btn-danger" + +# Assert eventual state with variable interpolation +Eventually "ExpectText $initialTitle h1" \ No newline at end of file