fix(ci): bound inner test jobs while preserving concurrency and coverage

.github/workflows/ci.yml

@@ -46,7 +46,9 @@ jobs:
         # Every platform uses two external runners. Stable name hashing keeps
         # the package-qualified selections disjoint and exhaustive even if
         # independently discovered test-list order differs.
-        run: go run ./internal/testparallel -race -workers ${{ runner.os == 'macOS' && 2 || 4 }} -shard-index ${{ matrix.shard }} -shard-count 2 ./...
+        # Bound each inner job to 150 discovered names so large packages run
+        # in multiple waves without increasing process concurrency or timeout.
+        run: go run ./internal/testparallel -race -workers ${{ runner.os == 'macOS' && 2 || 4 }} -max-tests-per-job 150 -shard-index ${{ matrix.shard }} -shard-count 2 ./...
 
   quality:
     runs-on: ubuntu-latest

docs/development/parallel-test-job-size.md

@@ -0,0 +1,38 @@
+# Bounded inner test jobs
+
+The [main CI run on 746406b](https://github.com/jxsl13/perfscan/actions/runs/34761114704)
+failed in macOS oldstable external shard 1: both checks jobs exhausted the
+existing 20-minute timeout (1200.721s and 1200.376s). The other 13 matrix jobs
+passed. Active stacks included analyzer fixture package loading and the
+owner-shaped end-to-end campaign, with many ordinary tests queued behind
+`t.Parallel`; the evidence did not identify an assertion failure or a single
+hung test.
+
+`-max-tests-per-job` now defaults to 150, and CI specifies that value explicitly.
+After stable external assignment, each package uses
+`max(workers, ceil(selected / max-tests-per-job))` balanced round-robin groups,
+capped by the number of selected names. Empty selections produce no jobs.
+This bounds how many discovered names share one process timeout without
+increasing concurrent Go test processes. macOS still uses two workers, Linux
+and Windows still use four, and every job retains race detection and the
+20-minute timeout. Benchmarks remain in their dedicated gate.
+
+Race-enabled discovery of the initial implementation on the local macOS checkout (`go test -race -p 2 -list .
+./...`) found 23 packages and 2004 ordinary names, including the added job-size
+regression. The checks package contains 1589 names: external shards 0 and 1
+select 765 and 824 respectively. With two workers, their former two inner jobs
+contained at most 383 and 412 names. The cap produces six jobs per external
+shard, containing at most 128 and 138 names. Across all packages, external
+shards 0 and 1 select 962 and 1042 names, and use 38 and 35 jobs rather than 34
+and 31. External selection and the complete test census are preserved.
+The final revision additionally keeps the original seven-name composition
+fixture and adds a separate large composition test; the census above predates
+that additional test.
+
+Focused ordinary and race tests cover stable balanced partitioning, the cap
+boundaries, empty/small selections, and exact-once coverage of 1001 names
+across both external shards. Discovery still respects the race build. These
+checks establish partition correctness, not a measured CI speedup or proof
+that every macOS job fits its time budget; the complete CI matrix must validate
+the runtime result. A name-count cap cannot guarantee wall time for a single
+heavy test or its subtests.

internal/ciworkflow/build_cache_test.go

@@ -309,7 +309,7 @@ func TestGoCacheChangePreservesAllCIGateCommands(t *testing.T) {
 	wanted := map[string][]string{
 		"test": {
 			"Build|matrix.shard == 0|go build ./...",
-			"Test in parallel||go run ./internal/testparallel -race -workers ${{ runner.os == 'macOS' && 2 || 4 }} -shard-index ${{ matrix.shard }} -shard-count 2 ./...",
+			"Test in parallel||go run ./internal/testparallel -race -workers ${{ runner.os == 'macOS' && 2 || 4 }} -max-tests-per-job 150 -shard-index ${{ matrix.shard }} -shard-count 2 ./...",
 		},
 		"quality": {
 			"gofmt||# analysistest fixtures under testdata/ need exact // want anchoring\n# (single-line loops etc.) that gofmt would break, so exclude them.\nout=$(gofmt -l . | grep -v /testdata/ || true)\nif [ -n \"$out\" ]; then echo \"gofmt needed on:\"; echo \"$out\"; exit 1; fi",

internal/testparallel/discovery_test.go

@@ -83,7 +83,7 @@ func TestDiscoveryBoundedConcurrentOrderedAndExhaustive(t *testing.T) {
 		}
 		for external := range 3 {
 			for index, pkg := range packages {
-				if got, wantJobs := makeTestJobs(pkg, result.names[index], 2, external, 3), makeTestJobs(pkg, want[index], 2, external, 3); !reflect.DeepEqual(got, wantJobs) {
+				if got, wantJobs := makeTestJobs(pkg, result.names[index], 2, 150, external, 3), makeTestJobs(pkg, want[index], 2, 150, external, 3); !reflect.DeepEqual(got, wantJobs) {
 					t.Fatalf("discovery changed exact external/inner job selection: %v vs %v", got, wantJobs)
 				}
 			}

internal/testparallel/main.go

@@ -36,6 +36,7 @@ func main() {
 	// enabled so ordinary test cases can run concurrently and CI does not become
 	// unnecessarily serial.
 	parallel := flag.Int("parallel", runtime.GOMAXPROCS(0), "maximum tests run in parallel within each shard")
+	maxTestsPerJob := flag.Int("max-tests-per-job", 150, "maximum discovered test names in each test process")
 	timeout := flag.Duration("timeout", 20*time.Minute, "timeout for each test shard")
 	race := flag.Bool("race", false, "run each shard with the race detector")
 	shardIndex := flag.Int("shard-index", 0, "zero-based external shard assigned to this process")
@@ -49,6 +50,10 @@ func main() {
 		_, _ = io.WriteString(os.Stderr, "testparallel: -parallel must be at least 1\n")
 		os.Exit(2)
 	}
+	if *maxTestsPerJob < 1 {
+		_, _ = io.WriteString(os.Stderr, "testparallel: -max-tests-per-job must be at least 1\n")
+		os.Exit(2)
+	}
 	if *timeout <= 0 {
 		_, _ = io.WriteString(os.Stderr, "testparallel: -timeout must be greater than zero\n")
 		os.Exit(2)
@@ -80,7 +85,7 @@ func main() {
 	tests, selected := 0, 0
 	for index, pkg := range packages {
 		tests += len(names[index])
-		packageJobs := makeTestJobs(pkg, names[index], *workers, *shardIndex, *shardCount)
+		packageJobs := makeTestJobs(pkg, names[index], *workers, *maxTestsPerJob, *shardIndex, *shardCount)
 		for _, job := range packageJobs {
 			selected += len(job.names)
 		}
@@ -103,9 +108,16 @@ func validateExternalShard(index, count int) error {
 	return nil
 }
 
-func makeTestJobs(pkg string, names []string, workers, externalIndex, externalCount int) []testJob {
+func makeTestJobs(pkg string, names []string, workers, maxTestsPerJob, externalIndex, externalCount int) []testJob {
 	selected := selectExternalShard(pkg, names, externalIndex, externalCount)
-	groups := partition(selected, workers)
+	// Job granularity is independent of process concurrency. Large packages
+	// need more than one wave of jobs so queued parallel tests do not all share
+	// the same process timeout. Round-robin partitioning keeps groups balanced.
+	count := workers
+	if len(selected) > 0 {
+		count = max(count, 1+(len(selected)-1)/maxTestsPerJob)
+	}
+	groups := partition(selected, count)
 	jobs := make([]testJob, 0, len(groups))
 	for shard, group := range groups {
 		jobs = append(jobs, testJob{pkg: pkg, shard: shard, shardCount: len(groups), names: group})

internal/testparallel/main_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 	"slices"
 	"strings"
@@ -150,7 +151,7 @@ func TestExternalAndWorkerShardsCoverEveryTestOnce(t *testing.T) {
 	names := []string{"TestA", "TestB", "TestC", "TestD", "TestE", "TestF", "TestG"}
 	seen := make(map[string]int, len(names))
 	for external := range 2 {
-		for _, job := range makeTestJobs("example.com/p", names, 3, external, 2) {
+		for _, job := range makeTestJobs("example.com/p", names, 3, 150, external, 2) {
 			if job.pkg != "example.com/p" || job.shardCount < 1 || job.shard >= job.shardCount {
 				t.Fatalf("invalid job metadata: %+v", job)
 			}
@@ -166,6 +167,67 @@ func TestExternalAndWorkerShardsCoverEveryTestOnce(t *testing.T) {
 	}
 }
 
+func TestCappedExternalAndInnerJobsCoverEveryTestOnce(t *testing.T) {
+	t.Parallel()
+	names := make([]string, 1001)
+	for i := range names {
+		names[i] = fmt.Sprintf("Test%04d", i)
+	}
+	seen := make(map[string]int, len(names))
+	for external := range 2 {
+		for _, job := range makeTestJobs("example.com/p", names, 2, 150, external, 2) {
+			if job.pkg != "example.com/p" || job.shardCount < 1 || job.shard >= job.shardCount {
+				t.Fatalf("invalid job metadata: %+v", job)
+			}
+			for _, name := range job.names {
+				if externalShardForName(job.pkg, name, 2) != external {
+					t.Fatalf("inner partition changed external assignment of %s", name)
+				}
+				seen[name]++
+			}
+		}
+	}
+	for _, name := range names {
+		if seen[name] != 1 {
+			t.Fatalf("%s appeared %d times after both sharding layers, want once", name, seen[name])
+		}
+	}
+}
+
+func TestJobsBoundLargePackagesIndependentlyOfWorkers(t *testing.T) {
+	t.Parallel()
+	for _, cap := range []int{1, 7, 150} {
+		for _, size := range []int{0, 1, 2, 149, 150, 151, 300, 301, 523} {
+			names := make([]string, size)
+			for i := range names {
+				names[i] = fmt.Sprintf("Test%04d", i)
+			}
+			jobs := makeTestJobs("example.com/p", names, 2, cap, 0, 1)
+			wantCount := min(size, max(2, (size+cap-1)/cap))
+			if len(jobs) != wantCount {
+				t.Fatalf("size %d: got %d jobs, want %d", size, len(jobs), wantCount)
+			}
+			if !reflect.DeepEqual(jobs, makeTestJobs("example.com/p", names, 2, cap, 0, 1)) {
+				t.Fatalf("size %d: job partition is unstable", size)
+			}
+			seen := make(map[string]int, size)
+			for _, job := range jobs {
+				if len(job.names) == 0 || len(job.names) > cap || len(job.names) < size/max(1, len(jobs)) {
+					t.Fatalf("size %d: unbalanced or oversized job: %+v", size, job)
+				}
+				for _, name := range job.names {
+					seen[name]++
+				}
+			}
+			for _, name := range names {
+				if seen[name] != 1 {
+					t.Fatalf("size %d: %s selected %d times, want once", size, name, seen[name])
+				}
+			}
+		}
+	}
+}
+
 func TestPatternIsAnchoredAndEscaped(t *testing.T) {
 	t.Parallel()
 	if got, want := testPattern([]string{"TestA/B", "TestC+"}), `^(TestA/B|TestC\+)$`; got != want {