diff --git a/.gitignore b/.gitignore index dfaf1a91..c27dd533 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .vscode +.venv __pycache__ __MACOSX .DS_Store diff --git a/README.md b/README.md index a9d57f75..a672c067 100644 --- a/README.md +++ b/README.md @@ -632,7 +632,7 @@ optional arguments: usage: fluster.py run [-h] [-j JOBS] [-t TIMEOUT] [-ff] [-q] [-ts TESTSUITES [TESTSUITES ...]] [-tv TESTVECTORS [TESTVECTORS ...]] [-sv SKIPVECTORS [SKIPVECTORS ...]] [-d DECODERS [DECODERS ...]] [-s] -[-so SUMMARY_OUTPUT] [-f {md,csv,junitxml}] [-k] [-th THRESHOLD] +[-so SUMMARY_OUTPUT] [-f {md,csv,junitxml}] [-k] [-p PROFILES [PROFILES ...]] [-th THRESHOLD] [-tth TIME_THRESHOLD] [-v] optional arguments: @@ -658,6 +658,8 @@ optional arguments: -f {md,csv,json,junitxml}, --format {md,csv,json,junitxml} specify the format for the summary file -k, --keep keep output files generated during the test + -p PROFILES [PROFILES ...], --profiles PROFILES [PROFILES ...] + run only test vectors matching the given profiles (e.g. 'VP9 Profile 0') -th THRESHOLD, --threshold THRESHOLD set exit code to 2 if threshold tests are not success. exit code is 0 otherwise @@ -667,6 +669,20 @@ optional arguments: -v, --verbose show stdout and stderr of commands executed ``` +**Profile Filtering:** +- The `-p/--profiles` option filters test vectors by their codec profile. Only vectors + with a matching profile are run; vectors without profile metadata are excluded. +- Multiple profiles can be specified (OR logic). +- Profile names are case-insensitive (e.g. `-p Main`, `-p "VP9 Profile 0"`). +- VP9 profiles are automatically inferred from test vector names + (`vp90-` prefix → Profile 0, `vp91-` → Profile 1, etc.). +- Vectors without a profile field in the JSON file are always excluded + when profile filtering is active. +- Examples: + - `./fluster.py run -ts JVT-AVC_V1 -p Baseline` runs only Baseline profile vectors + - `./fluster.py run -ts VP9-TEST-VECTORS -p "VP9 Profile 0"` runs only Profile 0 vectors + - `./fluster.py run -ts JVT-AVC_V1 -p Baseline Main High` runs vectors matching any of the three profiles + ### Download ```bash @@ -777,6 +793,7 @@ options: ./fluster.py reference --help usage: fluster.py reference [-h] [-j JOBS] [-t TIMEOUT] [-q] [-v] + [-p PROFILES [PROFILES ...]] decoder testsuites [testsuites ...] positional arguments: @@ -791,6 +808,9 @@ optional arguments: timeout in secs for each decoding. Defaults to 30 secs -q, --quiet don't show every test run -v, --verbose show stdout and stderr of commands executed + -p PROFILES [PROFILES ...], --profiles PROFILES [PROFILES ...] + run only test vectors for the given profiles (e.g. + 'VP9 Profile 0') ``` ## Report diff --git a/fluster/codec.py b/fluster/codec.py index 20014b4b..b9de5e64 100644 --- a/fluster/codec.py +++ b/fluster/codec.py @@ -16,6 +16,7 @@ # License along with this library. If not, see . from enum import Enum +from typing import List, Optional class Codec(Enum): @@ -67,52 +68,84 @@ class OutputFormat(Enum): class Profile(Enum): - """Profile""" + """Profile - NONE = "None" + Each member carries a ``codecs`` attribute identifying the codecs it + applies to. Some profiles are shared among several codecs. + """ + + codecs: List[Codec] + + NONE = ("None", [Codec.NONE]) # H.264 - CONSTRAINED_BASELINE = "Constrained Baseline" - BASELINE = "Baseline" - EXTENDED = "Extended" - MAIN = "Main" - HIGH = "High" - HIGH_10 = "High 10" - HIGH_10_INTRA = "High 10 Intra" - HIGH_4_2_2 = "High 4:2:2" - HIGH_4_2_2_INTRA = "High 4:2:2 Intra" - HIGH_4_4_4_INTRA = "High 4:4:4 Intra" - HIGH_4_4_4_PREDICTIVE = "High 4:4:4 Predictive" - CAVLC_4_4_4 = "CAVLC 4:4:4" - CAVLC_4_4_4_INTRA = "CAVLC 4:4:4 Intra" - - MAIN_10 = "Main 10" - MAIN_STILL_PICTURE = "Main Still Picture" - MAIN_4_2_2_10 = "Main 4:2:2 10" - MAIN_4_4_4_12 = "Main 4:4:4 12" + CONSTRAINED_BASELINE = ("Constrained Baseline", [Codec.H264]) + BASELINE = ("Baseline", [Codec.H264]) + EXTENDED = ("Extended", [Codec.H264]) + HIGH = ("High", [Codec.H264, Codec.AV1]) + HIGH_10 = ("High 10", [Codec.H264]) + HIGH_10_INTRA = ("High 10 Intra", [Codec.H264]) + HIGH_4_2_2 = ("High 4:2:2", [Codec.H264]) + HIGH_4_2_2_INTRA = ("High 4:2:2 Intra", [Codec.H264]) + HIGH_4_4_4_INTRA = ("High 4:4:4 Intra", [Codec.H264]) + HIGH_4_4_4_PREDICTIVE = ("High 4:4:4 Predictive", [Codec.H264]) + CAVLC_4_4_4 = ("CAVLC 4:4:4", [Codec.H264]) + CAVLC_4_4_4_INTRA = ("CAVLC 4:4:4 Intra", [Codec.H264]) + + MAIN_4_2_2_10 = ("Main 4:2:2 10", [Codec.H264]) + MAIN_4_4_4_12 = ("Main 4:4:4 12", [Codec.H264]) + + # Shared between H.264, H.265, AV1 and MPEG2 video + MAIN = ("Main", [Codec.H264, Codec.H265, Codec.AV1, Codec.MPEG2_VIDEO]) + + # Shared between H.265 and H.266 + MAIN_10 = ("Main 10", [Codec.H265, Codec.H266]) + + # H.265 + MAIN_STILL_PICTURE = ("Main Still Picture", [Codec.H265]) # H.266 - MAIN_10_4_4_4 = "Main 10 4:4:4" - MAIN_10_STILL_PICTURE = "Main 10 Still Picture" - MAIN_10_4_4_4_STILL_PICTURE = "Main 10 4:4:4 Still Picture" - MULTILAYER_MAIN_10 = "Multilayer Main 10" - MULTILAYER_MAIN_10_4_4_4 = "Multilayer Main 10 4:4:4" + MAIN_10_4_4_4 = ("Main 10 4:4:4", [Codec.H266]) + MAIN_10_STILL_PICTURE = ("Main 10 Still Picture", [Codec.H266]) + MAIN_10_4_4_4_STILL_PICTURE = ("Main 10 4:4:4 Still Picture", [Codec.H266]) + MULTILAYER_MAIN_10 = ("Multilayer Main 10", [Codec.H266]) + MULTILAYER_MAIN_10_4_4_4 = ("Multilayer Main 10 4:4:4", [Codec.H266]) # MPEG2 video - PROFILE_4_2_2 = "4:2:2" - SIMPLE = "Simple" + PROFILE_4_2_2 = ("4:2:2", [Codec.MPEG2_VIDEO]) + SIMPLE = ("Simple", [Codec.MPEG2_VIDEO]) # MPEG4 video - SIMPLE_PROFILE = "Simple Profile" - ADVANCED_SIMPLE_PROFILE = "Advanced Simple Profile" - SIMPLE_STUDIO_PROFILE = "Simple Studio Profile" - ERROR_RESILIENT_SIMPLE_SCALABLE_PROFILE = "Error Resilient Simple Scalable Profile" + SIMPLE_PROFILE = ("Simple Profile", [Codec.MPEG4_VIDEO]) + ADVANCED_SIMPLE_PROFILE = ("Advanced Simple Profile", [Codec.MPEG4_VIDEO]) + SIMPLE_STUDIO_PROFILE = ("Simple Studio Profile", [Codec.MPEG4_VIDEO]) + ERROR_RESILIENT_SIMPLE_SCALABLE_PROFILE = ( + "Error Resilient Simple Scalable Profile", + [Codec.MPEG4_VIDEO], + ) # AAC - AAC_MAIN = "AAC Main" - AAC_LC = "AAC Low Complexity" - AAC_SSR = "AAC Scalable Sampling Rate" - AAC_LTP = "AAC Long Term Prediction" - ER_AAC_LC = "Error Resilient AAC Low Complexity" - ER_AAC_ELD = "Error Resilient AAC Enhanced Low Delay" - ER_AAC_LD = "Error Resilient AAC Low Delay" + AAC_MAIN = ("AAC Main", [Codec.AAC]) + AAC_LC = ("AAC Low Complexity", [Codec.AAC]) + AAC_SSR = ("AAC Scalable Sampling Rate", [Codec.AAC]) + AAC_LTP = ("AAC Long Term Prediction", [Codec.AAC]) + ER_AAC_LC = ("Error Resilient AAC Low Complexity", [Codec.AAC]) + ER_AAC_ELD = ("Error Resilient AAC Enhanced Low Delay", [Codec.AAC]) + ER_AAC_LD = ("Error Resilient AAC Low Delay", [Codec.AAC]) + + # VP9 + VP9_PROFILE_0 = ("VP9 Profile 0", [Codec.VP9]) + VP9_PROFILE_1 = ("VP9 Profile 1", [Codec.VP9]) + VP9_PROFILE_2 = ("VP9 Profile 2", [Codec.VP9]) + VP9_PROFILE_3 = ("VP9 Profile 3", [Codec.VP9]) + + def __new__(cls, display_name: str, codecs: Optional[List[Codec]] = None) -> "Profile": + if codecs is None: + for member in cls: + if member.value == display_name: + return member + raise ValueError(f"{display_name!r} is not a valid {cls.__qualname__}") + obj = object.__new__(cls) + obj._value_ = display_name + obj.codecs = codecs + return obj diff --git a/fluster/fluster.py b/fluster/fluster.py index b9b1b078..ce6a3964 100644 --- a/fluster/fluster.py +++ b/fluster/fluster.py @@ -58,6 +58,7 @@ def __init__( verbose: bool = False, summary_output: str = "", summary_format: str = "", + profiles: Optional[List[str]] = None, ): self.jobs = jobs self.timeout = timeout @@ -77,6 +78,8 @@ def __init__( self.verbose = verbose self.summary_output = summary_output self.summary_format = summary_format + self.profiles_names = profiles + self.profiles: Set[Profile] = set() def to_test_suite_context( self, @@ -98,6 +101,7 @@ def to_test_suite_context( skip_vectors=skip_vectors, keep_files=self.keep_files, verbose=self.verbose, + profiles=self.profiles, ) return ts_context @@ -233,6 +237,25 @@ def list_test_suites( if len(self.test_suites) == 0: print(f' No test suites found in "{self.test_suites_dir}"') + def list_profiles(self, codec: Optional[Codec] = None) -> None: + """List all available profiles""" + print("\nList of available profiles:") + profiles_dict: Dict[Codec, List[Profile]] = {} + for profile in Profile: + if profile == Profile.NONE: + continue + for current_codec in profile.codecs: + if current_codec not in profiles_dict: + profiles_dict[current_codec] = [] + profiles_dict[current_codec].append(profile) + + for current_codec, profile_list in sorted(profiles_dict.items(), key=lambda item: item[0].value): + if codec and codec != current_codec: + continue + print(f"\n{current_codec}") + for profile in sorted(profile_list, key=lambda p: p.value): + print(f" {profile.value}") + @staticmethod def _get_matches(in_list: List[str], check_list: List[Any], name: str) -> List[Any]: if in_list: @@ -258,6 +281,14 @@ def _normalize_context(self, ctx: Context) -> None: ctx.test_vectors_names = [x.lower() for x in ctx.test_vectors_names] if ctx.skip_vectors_names: ctx.skip_vectors_names = [x.lower() for x in ctx.skip_vectors_names] + if ctx.profiles_names: + profile_values = {p.value.lower(): p for p in Profile} + for name in ctx.profiles_names: + if name.lower() in profile_values: + ctx.profiles.add(profile_values[name.lower()]) + else: + available = ", ".join(sorted(p.value for p in Profile if p != Profile.NONE)) + sys.exit(f"Unknown profile '{name}'. Available profiles: {available}") ctx.test_suites = self._get_matches(ctx.test_suites_names, self.test_suites, "test suite") ctx.decoders = self._get_matches(ctx.decoders_names, self.decoders, "decoders") diff --git a/fluster/main.py b/fluster/main.py index 178dfa66..4977fb6f 100644 --- a/fluster/main.py +++ b/fluster/main.py @@ -280,6 +280,12 @@ def _add_run_cmd(self, subparsers: Any) -> None: help="set exit code to 3 if test suite takes longer than threshold seconds. exit code is 0 otherwise", type=float, ) + subparser.add_argument( + "-p", + "--profiles", + help="run only test vectors for the given profiles (e.g. 'VP9 Profile 0')", + nargs="+", + ) subparser.add_argument( "-v", "--verbose", @@ -326,6 +332,12 @@ def _add_reference_cmd(self, subparsers: Any) -> None: help="show stdout and stderr of commands executed", action="store_true", ) + subparser.add_argument( + "-p", + "--profiles", + help="run only test vectors for the given profiles (e.g. 'VP9 Profile 0')", + nargs="+", + ) subparser.set_defaults(func=self._reference_cmd) def _add_download_cmd(self, subparsers: Any) -> None: @@ -372,6 +384,7 @@ def _add_download_cmd(self, subparsers: Any) -> None: def _list_cmd(args: Any, fluster: Fluster) -> None: fluster.list_test_suites(show_test_vectors=args.testvectors, test_suites=args.testsuites, codec=args.codec) fluster.list_decoders(check=args.check, verbose=args.verbose, codec=args.codec) + fluster.list_profiles(codec=args.codec) @staticmethod def _run_cmd(args: Any, fluster: Fluster) -> None: @@ -392,6 +405,7 @@ def _run_cmd(args: Any, fluster: Fluster) -> None: verbose=args.verbose, summary_output=args.summary_output, summary_format=args.format, + profiles=args.profiles, ) try: fluster.run_test_suites(context) @@ -411,6 +425,7 @@ def _reference_cmd(args: Any, fluster: Fluster) -> None: quiet=args.quiet, verbose=args.verbose, reference=True, + profiles=args.profiles, ) try: fluster.run_test_suites(context) diff --git a/fluster/test_suite.py b/fluster/test_suite.py index d0602749..b6ab016b 100644 --- a/fluster/test_suite.py +++ b/fluster/test_suite.py @@ -30,7 +30,7 @@ from unittest.result import TestResult from fluster import utils -from fluster.codec import Codec +from fluster.codec import Codec, Profile from fluster.decoder import Decoder, get_reference_decoder_for_codec from fluster.test import MD5ComparisonTest, PixelComparisonTest, ReferenceComparisonTest, SampleComparisonTest, Test from fluster.test_vector import TestVector, TestVectorResult @@ -101,6 +101,7 @@ def __init__( verbose: bool = False, reference_decoder: Optional[Decoder] = None, test_vector_names: Optional[Set[str]] = None, + profiles: Optional[Set[Profile]] = None, ): self.jobs = jobs self.decoder = decoder @@ -116,6 +117,7 @@ def __init__( self.verbose = verbose self.reference_decoder = reference_decoder self.test_vector_names = test_vector_names + self.profiles = profiles class TestMethod(Enum): @@ -168,6 +170,18 @@ def clone(self) -> "TestSuite": """Create a deep copy of the object""" return copy.deepcopy(self) + @staticmethod + def _infer_vp9_profile(name: str) -> Optional[Profile]: + if name.startswith("vp90-"): + return Profile.VP9_PROFILE_0 + if name.startswith("vp91-"): + return Profile.VP9_PROFILE_1 + if name.startswith("vp92-"): + return Profile.VP9_PROFILE_2 + if name.startswith("vp93-"): + return Profile.VP9_PROFILE_3 + return None + @classmethod def from_json_file(cls: Type["TestSuite"], filename: str, resources_dir: str) -> "TestSuite": """Create a TestSuite instance from a file""" @@ -184,7 +198,15 @@ def from_json_file(cls: Type["TestSuite"], filename: str, resources_dir: str) -> data.pop("test_vectors_not_run", None) data.pop("test_vectors_not_supported", None) data.pop("time_taken", None) - return cls(filename, resources_dir, **data) + test_suite = cls(filename, resources_dir, **data) + # Infer VP9 profile from filename when not explicitly set + if test_suite.codec == Codec.VP9: + for test_vector in test_suite.test_vectors.values(): + if test_vector.profile is None: + inferred = cls._infer_vp9_profile(test_vector.name) + if inferred: + test_vector.profile = inferred + return test_suite def to_json_file(self, filename: str) -> None: """Serialize the test suite to a file""" @@ -581,6 +603,8 @@ def generate_tests(self, ctx: Context) -> List[Test]: name_lower = test_vector.name.lower() if ctx.test_vector_names is not None and name_lower not in ctx.test_vector_names: continue + if ctx.profiles and test_vector.profile not in ctx.profiles: + continue if ctx.skip_vectors and name_lower in ctx.skip_vectors: skip = True