[bot-detection] Add custom JS fingerprinting capabilities

pkg/acquisition/modules/appsec/config.go

@@ -130,9 +130,9 @@ func loadCertPool(caCertPath string, logger log.FieldLogger) (*x509.CertPool, er
 }
 
 // expandAppsecConfigEntry resolves a single appsec_config(s) entry into the list
-// of appsec-config item names to load. A literal entry is returned untouched. An entry containing a glob meta-character 
-// is matched against the installed appsec-configs with the same matcher used
-// to expand appsec-rule patterns; it errors when no installed config matches.
+// of appsec-config item names to load. A literal entry is returned untouched.
+// An entry containing a glob meta-character is matched against the installed appsec-configs,
+// errors when no installed config matches.
 func expandAppsecConfigEntry(entry string, hub *cwhub.Hub) ([]string, error) {
 	if !strings.ContainsAny(entry, "*?") {
 		return []string{entry}, nil
@@ -281,6 +281,11 @@ func (w *Source) Configure(ctx context.Context, yamlConfig []byte, logger *log.E
 			return fmt.Errorf("unable to build challenge options: %w", err)
 		}
 
+		// cwhub manages the script.
+		if customJS := appsecCfg.LoadCustomJS(w.hub.GetDataDir()); customJS != "" {
+			challengeOpts = append(challengeOpts, challenge.WithCustomJS(customJS))
+		}
+
 		challengeRuntime, err := challenge.NewChallengeRuntime(ctx, challengeOpts...)
 		if err != nil {
 			return fmt.Errorf("unable to create challenge runtime: %w", err)

pkg/appsec/appsec.go

@@ -1275,6 +1275,19 @@ func (w *AppsecRuntimeConfig) ProcessOnChallengeRules(ctx context.Context, state
 			map[string]string{"Content-Type": "application/javascript", "Cache-Control": "public, max-age=3600"}, nil)
 	}
 
+	// Unreferenced when no appsec-config shipped a script, so the path then
+	// falls through to normal request handling.
+	//
+	// Never cached: the script changes whenever an operator changes the shipped
+	// detections, and a returning visitor running old detections against new
+	// scoring rules would be silent.
+	if path == challenge.ChallengeCustomJSPath {
+		if customJS := w.ChallengeRuntime.CustomJS(); customJS != "" {
+			return w.setChallengeResponse(state, http.StatusOK, customJS,
+				map[string]string{"Content-Type": "application/javascript", "Cache-Control": "no-cache, no-store"}, nil)
+		}
+	}
+
 	// Challenge submission: validate, give on_challenge_submit hooks a chance
 	// to reject the submission, then issue (or deny) the cookie. Per-route
 	// on_challenge inspection happens on subsequent cookie-bearing requests.

pkg/appsec/challenge/challenge.go

@@ -31,6 +31,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"text/template"
 	"time"
 
@@ -52,6 +53,8 @@ const (
 	ChallengeSubmitPath    = "/crowdsec-internal/challenge/submit"
 	ChallengePowWorkerPath = "/crowdsec-internal/challenge/pow-worker.js"
 	ChallengeFPScannerPath = "/crowdsec-internal/challenge/fpscanner.js"
+	// ChallengeCustomJSPath serves the custom detection script; see WithCustomJS.
+	ChallengeCustomJSPath = "/crowdsec-internal/challenge/custom.js"
 )
 
 // Sentinel errors (reasons) returned by ValidateChallengeResponse.
@@ -78,11 +81,17 @@ const cryptoObfuscationPoolDefaultSize = 1
 // outlive the per-epoch signing window without widening forgery exposure.
 const defaultCookieTTL = 12 * time.Hour
 
+// DefaultCustomJSTimeout is the wall-clock all detection hooks share when
+// custom_js_timeout is unset. It is time the visitor spends on the challenge
+// page, so it is deliberately short.
+const DefaultCustomJSTimeout = 500 * time.Millisecond
+
 // DefaultChallengeCSP is the Content-Security-Policy header used on the
 // challenge page when the operator hasn't configured a custom one. Allows
-// inline script/style (the challenge runtime injects both) and blob workers
-// (the PoW worker is loaded from a blob URL).
-const DefaultChallengeCSP = "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; worker-src 'self' blob:;"
+// inline script/style (the challenge runtime injects both), blob workers
+// (the PoW worker is loaded from a blob URL) and WebAssembly compilation
+// (hub-shipped detection modules can use it).
+const DefaultChallengeCSP = "default-src 'self'; script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; worker-src 'self' blob:;"
 
 //go:embed challenge.html.tmpl
 var htmlTemplate string
@@ -125,6 +134,23 @@ type ChallengeRuntime struct {
 	// dynamic_module.go.
 	challengeCode string
 
+	// customJS is the concatenated detection script shipped through the hub.
+	customJS string
+
+	// customJSTimeout is the budget those hooks share, sent to the browser on
+	// each challenge page.
+	customJSTimeout time.Duration
+
+	// customJSVersion is a digest of customJS. It is reported at startup so an
+	// operator can tell which build of the shipped detections an engine is
+	// actually running; it is not part of the script URL.
+	customJSVersion string
+
+	// customOverflowWarned latches the first cookie-overflow warning. Overflow is
+	// a property of the deployment (a detector reporting too much), not of the
+	// visitor, so warning per submission repeats it for every visitor.
+	customOverflowWarned atomic.Bool
+
 	powDifficulty int
 
 	// keys derives per-epoch sign keys (per-challenge secret / PoW MAC HMACs)
@@ -183,12 +209,29 @@ type runtimeOptions struct {
 	maxCookieLen              int
 	cryptoObfuscationPoolSize int
 	spentSetMaxEntries        int
+	customJS                  string
+	customJSTimeout           time.Duration
 	logger                    *log.Entry // nil → default "challenge" sublogger
 	// skipPreWarm drops the constructor's synchronous obfuscation and the
 	// background pre-warmer. Only withoutPreWarm (challenge_test.go) sets it.
 	skipPreWarm bool
 }
 
+// WithCustomJS sets the hub-distributed detection script. Empty disables it.
+func WithCustomJS(src string) Option {
+	return func(o *runtimeOptions) {
+		o.customJS = src
+	}
+}
+
+// WithCustomJSTimeout sets the shared detection-hook budget; zero or negative
+// is ignored, leaving DefaultCustomJSTimeout.
+func WithCustomJSTimeout(d time.Duration) Option {
+	return func(o *runtimeOptions) {
+		o.customJSTimeout = d
+	}
+}
+
 func WithLogger(logger *log.Entry) Option {
 	return func(o *runtimeOptions) {
 		o.logger = logger
@@ -426,6 +469,11 @@ func NewChallengeRuntime(ctx context.Context, opts ...Option) (*ChallengeRuntime
 		cookieTTL = defaultCookieTTL
 	}
 
+	customJSTimeout := resolvedOpts.customJSTimeout
+	if customJSTimeout <= 0 {
+		customJSTimeout = DefaultCustomJSTimeout
+	}
+
 	maxCookieLen := resolvedOpts.maxCookieLen
 	if maxCookieLen <= 0 {
 		maxCookieLen = MaxCookieLen
@@ -476,9 +524,12 @@ func NewChallengeRuntime(ctx context.Context, opts ...Option) (*ChallengeRuntime
 		maxCookieLen:       maxCookieLen,
 		htmlTpl:            htmlTpl,
 		spent:              newSpentSet(spentSetMaxEntries),
+		customJSTimeout:    customJSTimeout,
 		logger:             logger,
 	}
 
+	challengeRuntime.setCustomJS(resolvedOpts.customJS)
+
 	// Load the build-time-obfuscated challenge code from the baked-in bundle so
 	// we can serve immediately.
 	if err := challengeRuntime.seedCacheFromInitialBundle(); err != nil {
@@ -505,13 +556,22 @@ func NewChallengeRuntime(ctx context.Context, opts ...Option) (*ChallengeRuntime
 		go challengeRuntime.dynamicModulePreWarmer(runCtx)
 	}
 
-	logger.WithFields(log.Fields{
+	fields := log.Fields{
 		"rotation_interval": rotationInterval,
 		"cookie_ttl":        cookieTTL,
 		"max_cookie_len":    maxCookieLen,
 		"pow_difficulty":    defaultPowDifficulty,
 		"crypto_pool_size":  cryptoPoolSize,
-	}).Info("WAF challenge runtime initialized")
+	}
+
+	// Only with a script loaded: the timeout bounds nothing without hooks, and an
+	// empty version on every challenge-mode startup reads as a failed load.
+	if challengeRuntime.customJS != "" {
+		fields["custom_js_version"] = challengeRuntime.customJSVersion
+		fields["custom_js_timeout"] = customJSTimeout
+	}
+
+	logger.WithFields(fields).Info("WAF challenge runtime initialized")
 
 	return challengeRuntime, nil
 }
@@ -585,17 +645,26 @@ func (c *ChallengeRuntime) GetChallengePage(ctx context.Context, userAgent strin
 		return "", fmt.Errorf("build dynamic key module: %w", err)
 	}
 
+	// Empty drops the script tag, rather than pointing every challenge page at
+	// an empty file.
+	customJSPath := ""
+	if c.customJS != "" {
+		customJSPath = ChallengeCustomJSPath
+	}
+
 	var renderedPage strings.Builder
 
 	if err := c.htmlTpl.Execute(&renderedPage, map[string]interface{}{
-		"JSChallenge":   challengeCode,
-		"DynamicModule": dynamicModule,
-		"FPScannerPath": ChallengeFPScannerPath,
-		"PowDifficulty": difficulty,
-		"PowPrefix":     powSalt,
-		"PowMAC":        powMAC,
-		"Timestamp":     ts,
-		"R":             r,
+		"JSChallenge":       challengeCode,
+		"DynamicModule":     dynamicModule,
+		"FPScannerPath":     ChallengeFPScannerPath,
+		"CustomJSPath":      customJSPath,
+		"CustomJSTimeoutMS": c.customJSTimeout.Milliseconds(),
+		"PowDifficulty":     difficulty,
+		"PowPrefix":         powSalt,
+		"PowMAC":            powMAC,
+		"Timestamp":         ts,
+		"R":                 r,
 	}); err != nil {
 		return "", fmt.Errorf("render challenge page: %w", err)
 	}
@@ -705,6 +774,7 @@ func (c *ChallengeRuntime) ValidateChallengeResponse(request *http.Request, body
 			"k_epoch": fmt.Sprintf("%x", signKey),
 			"fsid":    fpData.FSID,
 			"is_bot":  fpData.FastBotDetection,
+			"custom":  fpData.CustomKeys(),
 		}).Debug("validated submission")
 	}
 
@@ -721,6 +791,15 @@ func (c *ChallengeRuntime) ValidateChallengeResponse(request *http.Request, body
 	// rotation); the browser Max-Age below matches so both expire together.
 	notAfter := time.Now().Add(c.cookieTTL).Unix()
 	cookieValue, err := sealCookieV0(envelope, c.keys.MasterCookieKey(), notAfter, 0, "", []byte(request.UserAgent()), c.maxCookieLen)
+
+	// If the cookie is too large, try to drop custom detection as a last chance to fit it within the size limit.
+	if errors.Is(err, ErrCookieTooLarge) && envelope.GetFingerprint().GetCustom() != nil {
+		c.logCustomOverflow(&fpData)
+
+		envelope.Fingerprint.Custom = nil
+		cookieValue, err = sealCookieV0(envelope, c.keys.MasterCookieKey(), notAfter, 0, "", []byte(request.UserAgent()), c.maxCookieLen)
+	}
+
 	if err != nil {
 		return nil, FingerprintData{}, 0, fmt.Errorf("failed to seal challenge cookie: %w", err)
 	}
@@ -794,3 +873,36 @@ func (c *ChallengeRuntime) ValidCookie(ck *http.Cookie, userAgent string) (*Cook
 		AllowlistReason: envelope.AllowlistReason,
 	}, nil
 }
+
+// setCustomJS keeps the script and the digest reported for it in step.
+func (c *ChallengeRuntime) setCustomJS(src string) {
+	c.customJS = src
+	c.customJSVersion = CustomJSVersion(src)
+}
+
+// CustomJS returns the detection script the dispatcher serves at
+// ChallengeCustomJSPath, empty when none is loaded.
+func (c *ChallengeRuntime) CustomJS() string {
+	if c == nil {
+		return ""
+	}
+
+	return c.customJS
+}
+
+// logCustomOverflow reports a dropped custom map. The first one is the operator's
+// signal; the rest are the same deployment fact repeated once per visitor, so they
+// go to debug. CustomKeys allocates, hence the level check.
+func (c *ChallengeRuntime) logCustomOverflow(fpData *FingerprintData) {
+	const msg = "custom detections do not fit in the cookie, dropped"
+
+	if c.customOverflowWarned.CompareAndSwap(false, true) {
+		c.log().WithField("custom", fpData.CustomKeys()).Warn(msg)
+
+		return
+	}
+
+	if c.log().Logger.IsLevelEnabled(log.DebugLevel) {
+		c.log().WithField("custom", fpData.CustomKeys()).Debug(msg)
+	}
+}

pkg/appsec/challenge/challenge.html.tmpl

@@ -388,10 +388,13 @@
       })();
     </script>
 
-    <script>var _powD={{.PowDifficulty}},_powP="{{.PowPrefix}}",_powM="{{.PowMAC}}",_ts="{{.Timestamp}}",_r="{{.R}}"</script>
+    <script>var _powD={{.PowDifficulty}},_powP="{{.PowPrefix}}",_powM="{{.PowMAC}}",_ts="{{.Timestamp}}",_r="{{.R}}",_cjsT={{.CustomJSTimeoutMS}}</script>
     <!-- Classic script: runs before the deferred module below, so the scanner
          global is ready when the challenge code runs. -->
     <script src="{{.FPScannerPath}}"></script>
+    {{if .CustomJSPath}}<!-- Classic script like the scanner, so the hooks are
+         registered before the module below runs them. -->
+    <script src="{{.CustomJSPath}}"></script>{{end}}
     <script type="module">
 {{.JSChallenge}}
 {{.DynamicModule}}

pkg/appsec/challenge/challenge.js

@@ -383,6 +383,55 @@ function reportChallengeStatus(status) {
 
 const CSEC_HOOK_NAME = "__CSEC_CHALLENGE_HOOK_v1__";
 
+// --- Custom detections ---
+// Hub-shipped scripts push a function onto this global; each may mutate the
+// collected fingerprint to clear an fpscanner false positive or add fp.custom
+// entries for appsec-rules to score. reservedStrings in
+// js/obfuscate/obfuscate.js keeps the name literal, so the separately-built
+// bundles agree on the symbol.
+const CSEC_CUSTOM_NAME = "__CSEC_CUSTOM_DETECT_v1__";
+
+// Budget for all hooks combined: the visitor is watching a spinner, so one that
+// hangs must not hold them there. Injected per request from custom_js_timeout;
+// the fallback matches DefaultCustomJSTimeout.
+const CSEC_CUSTOM_BUDGET_MS = typeof _cjsT !== "undefined" ? _cjsT : 500;
+
+// Each hook is isolated — a throw, rejection or timeout drops only its own
+// effect, so a broken script cannot cost a visitor the page.
+async function applyCustomDetections(fp) {
+  const hooks = globalThis[CSEC_CUSTOM_NAME];
+  if (!Array.isArray(hooks) || hooks.length === 0) {
+    return;
+  }
+
+  const deadline = new Promise((resolve) =>
+    setTimeout(resolve, CSEC_CUSTOM_BUDGET_MS),
+  );
+
+  for (const hook of hooks) {
+    if (typeof hook !== "function") {
+      continue;
+    }
+    try {
+      await Promise.race([Promise.resolve(hook(fp)), deadline]);
+    } catch (_) {
+      // Deliberately swallowed; see above.
+    }
+  }
+
+  // collectFingerprint set this before the hooks ran, so a cleared detection
+  // would otherwise leave the summary flag stuck on. fsid is left alone: it
+  // hashes the pre-hook bitmask, which keeps ids comparable between
+  // deployments running different scripts.
+  try {
+    fp.fastBotDetection = Object.values(fp.fastBotDetectionDetails).some(
+      (d) => d && d.detected,
+    );
+  } catch (_) {
+    // Malformed details; leave the flag as fpscanner set it.
+  }
+}
+
 async function runChallenge(epochKey) {
   // Fail closed if the fpscanner bundle didn't load.
   const Scanner = globalThis.CrowdsecFingerprintScanner;
@@ -405,6 +454,9 @@ async function runChallenge(epochKey) {
     return;
   }
 
+  // Runs before anything is signed, so hooks can still correct fpscanner.
+  await applyCustomDetections(fpResult);
+
   // Per-challenge secret s = HMAC(K_epoch, r). Never transmitted; the server
   // derives the same s from its per-epoch key and the cleartext r. epochKey is
   // hex-encoded K; HMAC over its raw bytes.

pkg/appsec/challenge/challenge_test.go

@@ -785,3 +785,28 @@ func TestChallengePageChecksCookiesEnabled(t *testing.T) {
 		require.Contains(t, htmlTemplate, want)
 	}
 }
+
+// The default CSP has to permit WebAssembly, because hub-shipped detection
+// modules compile a module to read CPU-level behavior. It must NOT permit
+// JavaScript eval: some automation frameworks inject their spoofing that way
+// and this policy is what refuses them, so the two tokens are not
+// interchangeable even though one contains the other as a substring.
+func TestDefaultChallengeCSP(t *testing.T) {
+	tests := []struct {
+		name  string
+		token string
+		want  bool
+	}{
+		{"allows wasm compilation", "'wasm-unsafe-eval'", true},
+		{"still forbids js eval", " 'unsafe-eval'", false},
+		{"keeps inline script", "'unsafe-inline'", true},
+		{"keeps blob workers", "worker-src 'self' blob:", true},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.want, strings.Contains(DefaultChallengeCSP, tc.token),
+				"token %q in %q", tc.token, DefaultChallengeCSP)
+		})
+	}
+}