Sidecar strings skip the required-tag check

Copilot review of PR hed-standard#1437. A sidecar string is a fragment of a row, so the sidecar stage no longer runs the required-tag check on it; required tags split between a sidecar string and the HED column validate clean. The unique-tag check still runs there, since a unique tag repeated inside a fragment is repeated in the row and hed-tests has a sidecar-only TAG_NOT_UNIQUE case. run_full_string_checks takes required_tags and unique_tags in place of whole_row; the HED-cell pass passes both False.

Author: VisLab

hed/validator/hed_validator.py

@@ -94,14 +94,16 @@ def run_basic_checks(self, hed_string, allow_placeholders) -> list[dict]:
         issues += self._def_validator.validate_def_tags(hed_string)
         return issues
 
-    def run_full_string_checks(self, hed_string, whole_row=True) -> list[dict]:
+    def run_full_string_checks(self, hed_string, required_tags=True, unique_tags=True) -> list[dict]:
         """Run all full-string validation checks on a HED string.
 
         Parameters:
             hed_string (HedString): The HED string to validate.
-            whole_row (bool): If False, the string is one cell of a row whose other columns also carry HED,
-                so the checks that need every tag of the row (required and unique tags) are skipped. The
-                group-level checks still apply: a group in the cell is a group of the row.
+            required_tags (bool): Run the required-tag check. Pass False for a fragment of a row (a HED cell,
+                a sidecar string): the tag it lacks may come from another column, so the check needs the
+                assembled row.
+            unique_tags (bool): Run the unique-tag check. A unique tag repeated inside a fragment is repeated
+                in the row too, so this check is safe on a fragment; pass False to leave it to assembly.
 
         Returns:
             list[dict]: A list of issues found during validation. Each issue is represented as a dictionary.
@@ -111,14 +113,25 @@ def run_full_string_checks(self, hed_string, whole_row=True) -> list[dict]:
             - Each check is a callable function that takes `hed_string` as input and returns a list of issues.
             - If any check returns issues, the method stops and returns those issues immediately.
             - If no issues are found, an empty list is returned.
+            - The group-level checks (tag placement, reserved groups, duplicates, Onset and Offset) always
+              run: a group in a fragment is a group of the row.
 
         """
+
+        def row_tag_checks(string_obj):
+            tags = string_obj.get_all_tags()
+            issues = []
+            if required_tags:
+                issues += self._group_validator.check_for_required_tags(tags)
+            if unique_tags:
+                issues += self._group_validator.check_multiple_unique_tags_exist(tags)
+            return issues
+
         checks = [
+            row_tag_checks,
             self._group_validator.run_tag_level_validators,
             self._def_validator.validate_onset_offset,
         ]
-        if whole_row:
-            checks.insert(0, self._group_validator.run_all_tags_validators)
 
         for check in checks:
             issues = check(hed_string)  # Call each function with `hed_string`

hed/validator/sidecar_validator.py

@@ -123,8 +123,11 @@ def _validate_sidecar(self, sidecar, extra_def_dicts, error_handler) -> list[dic
                             modified_string = df_util.replace_ref(modified_string, f"{{{ref}}}", ref_dict[ref])
                         hed_string_obj = HedString(modified_string, hed_schema=self._schema, def_dict=sidecar_def_dict)
 
+                        # A sidecar string is a fragment of a row: a tag it lacks may come from another
+                        # column, so the required-tag check waits for assembly. A unique tag repeated in
+                        # the fragment is repeated in the row, so that check runs here.
                         error_handler.push_error_context(ErrorContext.HED_STRING, hed_string_obj)
-                        new_issues += hed_validator.run_full_string_checks(hed_string_obj)
+                        new_issues += hed_validator.run_full_string_checks(hed_string_obj, required_tags=False)
                         error_handler.add_context_and_filter(new_issues)
                         issues += new_issues
                         error_handler.pop_error_context()  # Hed string

hed/validator/spreadsheet_validator.py

@@ -36,7 +36,8 @@ class SpreadsheetValidator:
 
     The stages run in order and each stops the run when it finds an error (never on a warning alone):
 
-    1. **Sidecar**: every template and categorical string of the sidecar (``SidecarValidator``).
+    1. **Sidecar**: every template and categorical string of the sidecar (``SidecarValidator``). A sidecar
+       string is a fragment of a row, so the required-tag check waits for assembly.
     2. **Column values**: the column structure (mapper issues, sidecar references to columns the table
        lacks), then each value column's distinct values against the units or value class of its ``#``
        tag, and each categorical column's distinct values against its sidecar keys.
@@ -394,7 +395,9 @@ def validate_hed_column(self, column_name, distinct, error_handler, row_offset=0
             hed_string = HedString(text, self._schema)
             string_issues = self._hed_validator.run_basic_checks(hed_string, allow_placeholders=False)
             if full_string and not check_for_any_errors(string_issues):
-                string_issues += self._hed_validator.run_full_string_checks(hed_string, whole_row=False)
+                string_issues += self._hed_validator.run_full_string_checks(
+                    hed_string, required_tags=False, unique_tags=False
+                )
             issues += self._report_distinct(string_issues, hed_string, rows, column_name, error_handler, row_offset)
         return issues
 
@@ -405,7 +408,9 @@ def _validate_hed_cells(self, column_name, distinct, error_handler, row_offset)
         issues = []
         for text, rows in distinct.items():
             hed_string = HedString(text, self._schema)
-            string_issues = self._hed_validator.run_full_string_checks(hed_string, whole_row=False)
+            string_issues = self._hed_validator.run_full_string_checks(
+                hed_string, required_tags=False, unique_tags=False
+            )
             issues += self._report_distinct(string_issues, hed_string, rows, column_name, error_handler, row_offset)
         return issues
 

tests/validator/test_hed_validator.py

@@ -229,6 +229,28 @@ def test_duplicate_group_in_definition(self):
         issues = test_string.validate(hed_schema)
         self.assertEqual(len(issues), 1)
 
+    def test_run_full_string_checks_row_tag_flags(self):
+        """The required-tag and unique-tag checks need the whole row; each can be left out for a fragment,
+        while the group-level checks always run."""
+        schema_path = os.path.join(self.hed_base_dir, "HED8.0.0_added_tests.mediawiki")
+        validator = HedValidator(schema.load_schema(schema_path))
+
+        missing_required = HedString("Event, (Onset)", validator._hed_schema)
+        codes = [issue["code"] for issue in validator.run_full_string_checks(missing_required)]
+        self.assertEqual(codes, ["REQUIRED_TAG_MISSING", "REQUIRED_TAG_MISSING"])
+        codes = [issue["code"] for issue in validator.run_full_string_checks(missing_required, required_tags=False)]
+        self.assertEqual(codes, ["TEMPORAL_TAG_ERROR"])
+
+        repeated_unique = HedString(
+            "Action, Animal-agent, (Event-context, (Red, Blue)), (Event-context, (Green, Yellow))",
+            validator._hed_schema,
+        )
+        codes = [issue["code"] for issue in validator.run_full_string_checks(repeated_unique)]
+        self.assertEqual(codes, ["TAG_NOT_UNIQUE"])
+        self.assertEqual(validator.run_full_string_checks(repeated_unique, unique_tags=False), [])
+        codes = [issue["code"] for issue in validator.run_full_string_checks(repeated_unique, required_tags=False)]
+        self.assertEqual(codes, ["TAG_NOT_UNIQUE"])
+
     def test_severity_enum_not_string_github_hedit_161(self):
         """Regression test for critical bug: severity should be ErrorSeverity enum, not string.
 

tests/validator/test_staged_validation.py

@@ -293,6 +293,23 @@ def test_required_and_unique_tags_are_checked_only_on_assembled_rows(self):
         issues = validator.validate(ListColumnSource(columns, base_input=frame))
         self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.REQUIRED_TAG_MISSING] * 2)
 
+        # Required tags split between a sidecar string and the HED cell: clean through a real assembled input,
+        # so stage 1 must not hold the sidecar string alone to the required-tag rule either.
+        split = _sidecar({"condition": {"HED": {"a": "Action"}}})
+        frame = _tabular([["onset", "duration", "condition", "HED"], [1.0, 0, "a", "Animal-agent"]], sidecar=split)
+        self.assertEqual(validator.validate(frame), [])
+        self.assertEqual(
+            validator.validate(ListColumnSource({"condition": ["a"], "HED": ["Animal-agent"]}), sidecar=split), []
+        )
+
+        # A unique tag repeated inside one sidecar string is repeated in every row that uses it, so stage 1
+        # still reports it (hed-tests TAG_NOT_UNIQUE has a sidecar-only case).
+        repeated = _sidecar(
+            {"condition": {"HED": {"a": "(Event-context, (Red, Blue)), (Event-context, (Green, Yellow))"}}}
+        )
+        issues = self.validator.validate(ListColumnSource({"condition": ["a"]}), sidecar=repeated)
+        self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.TAG_NOT_UNIQUE])
+
         # The same holds for a unique tag repeated within one cell.
         columns = {
             "onset": [1.0],