Fix per-key validation metrics judging every key by the whole target dict

src/inspect_scout/_recorder/validation.py

@@ -159,11 +159,22 @@ def compute_validation_metrics(
         # Per-key metrics
         per_key: dict[str, ValidationMetrics] = {}
         for entry in with_targets:
-            target_positive = is_positive_value(entry.target)
             if isinstance(entry.valid, dict):
                 for key, valid in entry.valid.items():
                     if key not in per_key:
                         per_key[key] = ValidationMetrics()
+                    # The expected target is per key: a dict target (produced by
+                    # dict- and label-based validation) carries one expectation
+                    # per key, so each key must be judged against its own target.
+                    # Using is_positive_value on the whole dict would always be
+                    # truthy, collapsing every key to a positive expectation and
+                    # making tn/fp impossible (precision pinned at 1.0,
+                    # specificity/accuracy always None). Fall back to the scalar
+                    # target for the legacy scalar-target-with-dict-valid shape.
+                    if isinstance(entry.target, dict):
+                        target_positive = is_positive_value(entry.target.get(key))
+                    else:
+                        target_positive = is_positive_value(entry.target)
                     _update_metrics(per_key[key], target_positive, valid)
 
         # Total is sum of per-key

tests/recorder/test_summary.py

@@ -228,6 +228,61 @@ def test_mixed_with_legacy(self) -> None:
         assert metrics.precision == 1.0
         assert metrics.recall == 1.0
 
+    def test_dict_target_uses_per_key_expectation(self) -> None:
+        """Dict targets carry one expectation per key (the production shape).
+
+        This is what dict- and label-based validation produce: ``target`` is a
+        dict keyed like ``valid``. Key ``b`` is expected negative (target False)
+        in both entries, so a scanner that correctly finds it absent is a true
+        negative and one that flags it is a false positive. Judging ``b`` against
+        the whole target dict (always truthy) would instead record tp/fn, pinning
+        precision at 1.0 and leaving specificity undefined.
+        """
+        validations = [
+            # b correctly found absent -> true negative for b
+            ValidationEntry(
+                id="t1", target={"a": True, "b": False}, valid={"a": True, "b": True}
+            ),
+            # b wrongly flagged present -> false positive for b
+            ValidationEntry(
+                id="t2", target={"a": True, "b": False}, valid={"a": True, "b": False}
+            ),
+        ]
+        result = compute_validation_metrics(validations)
+        assert result is not None
+        _, per_key = result
+        assert per_key is not None
+        # b is a negative target throughout: one tn, one fp, no positives.
+        assert per_key["b"].tp == 0
+        assert per_key["b"].fn == 0
+        assert per_key["b"].tn == 1
+        assert per_key["b"].fp == 1
+        assert per_key["b"].specificity == 0.5
+        # a is a positive target and passes both times: two true positives.
+        assert per_key["a"].tp == 2
+        assert per_key["a"].fp == 0
+
+    def test_labels_shape_negative_expectation_scored(self) -> None:
+        """Label-based validation (target == the labels dict) scores negatives.
+
+        Label validation stores the labels dict as ``target``. A label expected
+        absent that the scanner correctly leaves unflagged must count as a true
+        negative, not a true positive.
+        """
+        validations = [
+            ValidationEntry(
+                id="r1",
+                target={"deception": True, "sandbagging": False},
+                valid={"deception": True, "sandbagging": True},
+            ),
+        ]
+        result = compute_validation_metrics(validations)
+        assert result is not None
+        _, per_key = result
+        assert per_key is not None
+        assert per_key["sandbagging"].tn == 1
+        assert per_key["sandbagging"].tp == 0
+
 
 class TestValidationResults:
     """Tests for ValidationResults model."""