From 7dd77a231e06479ac2ea6c8aefc51be6aa8a83b8 Mon Sep 17 00:00:00 2001 From: Aamir Nazir Date: Wed, 5 Aug 2026 12:21:30 +0400 Subject: [PATCH 01/43] [Example] Add TorchAO to GPTQModel Test Requirement (#4164) ### Changes Add TorchAO to GPTQModel Test Requirement and pin it to `0.17.0` ### Reason for changes It was installed transitively before with torch. Recent release of torchao(`0.18.0`) was being installed this way which caused unexpected error. (Failing action: https://github.com/openvinotoolkit/nncf/actions/runs/30928990933/job/92221632392) ### Related tickets ### Tests [Test examples](https://github.com/openvinotoolkit/nncf/actions/runs/30984266172) - success --- .../llm_compression/torch/gptq_model_convertor/requirements.txt | 1 + .../torch/gptq_model_convertor/requirements_extra.txt | 1 + tests/integration/gptq_model/requirements.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/examples/llm_compression/torch/gptq_model_convertor/requirements.txt b/examples/llm_compression/torch/gptq_model_convertor/requirements.txt index c722ad61b06..b23ca1689b3 100644 --- a/examples/llm_compression/torch/gptq_model_convertor/requirements.txt +++ b/examples/llm_compression/torch/gptq_model_convertor/requirements.txt @@ -1,4 +1,5 @@ torch==2.9.0 +torchao==0.17.0 # TODO(AlexanderDokuchaev): remove transformers next release gptqmodel https://github.com/ModelCloud/GPTQModel/issues/2328 transformers==4.57.6 numpy==2.2.6 diff --git a/examples/llm_compression/torch/gptq_model_convertor/requirements_extra.txt b/examples/llm_compression/torch/gptq_model_convertor/requirements_extra.txt index c408b9b30ce..4408ddf9ff0 100644 --- a/examples/llm_compression/torch/gptq_model_convertor/requirements_extra.txt +++ b/examples/llm_compression/torch/gptq_model_convertor/requirements_extra.txt @@ -1,5 +1,6 @@ gptqmodel==5.6.12 torch==2.9.0 +torchao==0.17.0 # TODO(AlexanderDokuchaev): remove transformers next release gptqmodel https://github.com/ModelCloud/GPTQModel/issues/2328 transformers==4.57.6 huggingface_hub==0.36.2 diff --git a/tests/integration/gptq_model/requirements.txt b/tests/integration/gptq_model/requirements.txt index 86eb9ee9a7a..0f1a15a4e60 100644 --- a/tests/integration/gptq_model/requirements.txt +++ b/tests/integration/gptq_model/requirements.txt @@ -1,3 +1,4 @@ torch==2.9.0 +torchao==0.17.0 transformers==4.57.6 pytest==9.0.3 From 0d930a52500c18ba611b84a75cdaf72320886ee4 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 13 Aug 2026 13:42:53 +0400 Subject: [PATCH 02/43] init --- .../weights_compression/Usage.md | 34 +++ src/nncf/__init__.py | 3 + .../torch/fx/quantization/quantize_model.py | 3 + src/nncf/onnx/quantization/quantize_model.py | 3 + .../openvino/quantization/quantize_model.py | 3 + src/nncf/openvino/rt_info.py | 33 ++- src/nncf/quantization/advanced_parameters.py | 33 +++ .../weight_compression/algorithm.py | 219 ++++++++++++++++++ .../algorithms/weight_compression/config.py | 2 + src/nncf/quantization/quantize_model.py | 11 + src/nncf/scopes.py | 100 ++++++-- .../quantization/quantize_model.py | 3 + .../template_test_weights_compression.py | 202 ++++++++++++++++ .../quantization/test_weights_compression.py | 56 +++++ 14 files changed, 683 insertions(+), 22 deletions(-) diff --git a/docs/usage/post_training_compression/weights_compression/Usage.md b/docs/usage/post_training_compression/weights_compression/Usage.md index 9cc2d9165a9..29420508e4b 100644 --- a/docs/usage/post_training_compression/weights_compression/Usage.md +++ b/docs/usage/post_training_compression/weights_compression/Usage.md @@ -108,6 +108,40 @@ from nncf import compress_weights, CompressWeightsMode compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_ASYM, group_size=64, ratio=0.9) # model is openvino.Model object ``` +#### Custom precision for specific layers + +- The `mode`, `ratio` and `backup_mode` parameters define the precision of a layer indirectly: the mixed-precision + algorithm decides which layers are compressed to the primary precision, and the rest is compressed to the backup one. + The `custom_annotation` parameter makes it possible to assign a compression configuration to certain layers + explicitly. A typical use case is a Mixture-of-Experts model, where the experts are compressed to 4 bits, while the + attention and the router layers are kept in 8 bits to preserve accuracy. + +```python +import nncf +from nncf import compress_weights, CompressWeightsMode + +compressed_model = compress_weights( + model, # model is openvino.Model object + mode=CompressWeightsMode.INT4_SYM, + group_size=64, + ratio=1.0, + custom_annotation=[ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*self_attn.*", ".*router.*"]), + config=nncf.WeightCompressionConfig(mode=CompressWeightsMode.INT8_ASYM, group_size=-1), + ), + ], +) +``` + +- The scope of an annotation is defined by the same rules as [the ignored scope](/docs/usage/IgnoredScope.md): node names, + regular expressions, operation types and subgraphs. + The configuration given by an annotation takes precedence over the decision made by the algorithm, namely over the + mixed-precision assignment, the `all_layers` option and the backup precision of the embeddings and the last linear + layer. A layer that is matched by both the `ignored_scope` and a custom annotation is compressed with the + user-defined configuration and a warning is logged. If several annotations match the same layer, the last one takes + precedence. + #### Data-aware methods - Accuracy of the 4-bit compressed models can be improved by using data-aware mixed-precision algorithm. It is capable to find outliers in the input activations and assign different quantization precision to minimize accuracy degradation. diff --git a/src/nncf/__init__.py b/src/nncf/__init__.py index 475d7e836fc..8760a0c5f95 100644 --- a/src/nncf/__init__.py +++ b/src/nncf/__init__.py @@ -65,8 +65,11 @@ from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters as AdvancedQuantizationParameters from nncf.quantization.advanced_parameters import AdvancedScaleEstimationParameters as AdvancedScaleEstimationParameters from nncf.quantization.advanced_parameters import AdvancedSmoothQuantParameters as AdvancedSmoothQuantParameters +from nncf.quantization.advanced_parameters import CustomAnnotation as CustomAnnotation from nncf.quantization.advanced_parameters import GroupSizeFallbackMode as GroupSizeFallbackMode from nncf.quantization.advanced_parameters import OverflowFix as OverflowFix +from nncf.quantization.algorithms.weight_compression.config import WeightCompressionConfig as WeightCompressionConfig +from nncf.scopes import CustomAnnotationScope as CustomAnnotationScope from nncf.scopes import IgnoredScope as IgnoredScope from nncf.scopes import Subgraph as Subgraph from nncf.version import __version__ as __version__ diff --git a/src/nncf/experimental/torch/fx/quantization/quantize_model.py b/src/nncf/experimental/torch/fx/quantization/quantize_model.py index 8eff430895b..fe859a61958 100644 --- a/src/nncf/experimental/torch/fx/quantization/quantize_model.py +++ b/src/nncf/experimental/torch/fx/quantization/quantize_model.py @@ -37,6 +37,7 @@ from nncf.parameters import TargetDevice from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters +from nncf.quantization.advanced_parameters import CustomAnnotation from nncf.quantization.algorithms.post_training.algorithm import PostTrainingQuantization from nncf.quantization.algorithms.weight_compression.algorithm import WeightCompression from nncf.scopes import IgnoredScope @@ -132,6 +133,7 @@ def compress_weights_impl( backup_mode: BackupMode, compression_format: CompressionFormat, advanced_parameters: AdvancedCompressionParameters | None = None, + custom_annotation: list[CustomAnnotation] | None = None, ) -> torch.fx.GraphModule: """ Implementation of the `compress_weights()` method for the Torch Fx backend. @@ -151,6 +153,7 @@ def compress_weights_impl( backup_mode, compression_format, advanced_parameters, + custom_annotation, ) graph = build_graph(model) compressed_model = compression_algorithm.apply(model, graph, dataset=dataset) diff --git a/src/nncf/onnx/quantization/quantize_model.py b/src/nncf/onnx/quantization/quantize_model.py index d76179564c9..e8634f8ce64 100644 --- a/src/nncf/onnx/quantization/quantize_model.py +++ b/src/nncf/onnx/quantization/quantize_model.py @@ -46,6 +46,7 @@ from nncf.quantization.advanced_parameters import AdvancedAccuracyRestorerParameters from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters +from nncf.quantization.advanced_parameters import CustomAnnotation from nncf.quantization.advanced_parameters import QuantizationParameters from nncf.quantization.algorithms.accuracy_control.algorithm import QuantizationAccuracyRestorer from nncf.quantization.algorithms.accuracy_control.algorithm import calculate_accuracy_drop @@ -329,6 +330,7 @@ def compress_weights_impl( backup_mode: BackupMode, compression_format: CompressionFormat, advanced_parameters: AdvancedCompressionParameters | None = None, + custom_annotation: list[CustomAnnotation] | None = None, ) -> onnx.ModelProto: if model.opset_import[0].version < 13: msg = "ONNX models with opset version < 13 do not support per-channel quantization." @@ -361,6 +363,7 @@ def compress_weights_impl( backup_mode, compression_format, advanced_parameters, + custom_annotation, ) graph = build_graph(model) diff --git a/src/nncf/openvino/quantization/quantize_model.py b/src/nncf/openvino/quantization/quantize_model.py index 19031fb1674..d42cbb1f88f 100644 --- a/src/nncf/openvino/quantization/quantize_model.py +++ b/src/nncf/openvino/quantization/quantize_model.py @@ -42,6 +42,7 @@ from nncf.quantization.advanced_parameters import AdvancedAccuracyRestorerParameters from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters +from nncf.quantization.advanced_parameters import CustomAnnotation from nncf.quantization.advanced_parameters import convert_to_dict_recursively from nncf.quantization.algorithms.accuracy_control.algorithm import QuantizationAccuracyRestorer from nncf.quantization.algorithms.accuracy_control.algorithm import calculate_accuracy_drop @@ -379,6 +380,7 @@ def compress_weights_impl( backup_mode: BackupMode, compression_format: CompressionFormat, advanced_parameters: AdvancedCompressionParameters | None = None, + custom_annotation: list[CustomAnnotation] | None = None, ) -> ov.Model: """ Implementation of the `compress_weights()` method for the OpenVINO backend. @@ -400,6 +402,7 @@ def compress_weights_impl( backup_mode, compression_format, advanced_parameters, + custom_annotation, ) statistics_points = None diff --git a/src/nncf/openvino/rt_info.py b/src/nncf/openvino/rt_info.py index 87582316bbb..2f5ebc3ccd8 100644 --- a/src/nncf/openvino/rt_info.py +++ b/src/nncf/openvino/rt_info.py @@ -16,7 +16,9 @@ import nncf from nncf.common.logging import nncf_logger -from nncf.scopes import IgnoredScope +from nncf.quantization.advanced_parameters import CustomAnnotation +from nncf.quantization.algorithms.weight_compression.config import WeightCompressionConfig +from nncf.scopes import BaseScope def exclude_empty_fields(value: dict[str, Any]) -> dict[str, Any]: @@ -33,6 +35,21 @@ def exclude_empty_fields(value: dict[str, Any]) -> dict[str, Any]: return value +def compression_config_to_dict(config: WeightCompressionConfig) -> dict[str, Any]: + """ + Converts a weight compression config into a dictionary suitable for dumping into Model's meta section. + + Codebook values are reduced to their number since the values themselves are too large to be dumped. + + :param config: Weight compression config. + :return: Dictionary with the compression config parameters. + """ + value: dict[str, Any] = {"mode": config.mode.value, "group_size": config.group_size} + if config.codebook_values is not None: + value["codebook_size"] = config.codebook_values.size + return value + + def dump_parameters( model: ov.Model, parameters: dict[str, Any], algo_name: str | None = "quantization", path: list[str] | None = None ) -> None: @@ -48,13 +65,23 @@ def dump_parameters( path = path if path else [] for key, value in parameters.items(): # Special condition for composed fields like IgnoredScope - if isinstance(value, IgnoredScope): + if isinstance(value, BaseScope): value = exclude_empty_fields(asdict(value)) if bool(value): - dump_parameters(model, value, algo_name, [key]) + dump_parameters(model, value, algo_name, path + [key]) continue # The default value in case empty ignored_scope parameter passed value = [] + # Special condition for the list of custom annotations + elif isinstance(value, (list, tuple)) and any(isinstance(item, CustomAnnotation) for item in value): + for i, annotation in enumerate(value): + dump_parameters( + model, + {"scope": annotation.scope, "config": compression_config_to_dict(annotation.config)}, + algo_name, + path + [key, str(i)], + ) + continue rt_path = ["nncf", algo_name] + path + [key] model.set_rt_info(str(value), rt_path) diff --git a/src/nncf/quantization/advanced_parameters.py b/src/nncf/quantization/advanced_parameters.py index df39661f164..67d9486dced 100644 --- a/src/nncf/quantization/advanced_parameters.py +++ b/src/nncf/quantization/advanced_parameters.py @@ -24,7 +24,9 @@ from nncf.common.quantization.structs import QuantizationScheme as QuantizationMode from nncf.common.utils.api_marker import api from nncf.parameters import StrEnum +from nncf.quantization.algorithms.weight_compression.config import WeightCompressionConfig from nncf.quantization.range_estimator import RangeEstimatorParameters +from nncf.scopes import CustomAnnotationScope from nncf.tensor import TensorDataType TTensor = Any @@ -451,6 +453,37 @@ class AdvancedCompressionParameters: ) +@api(canonical_alias="nncf.CustomAnnotation") +@dataclass +class CustomAnnotation: + """ + Binds a user-defined weight compression configuration to a portion of a model. + + The configuration takes precedence over the one assigned by the weight compression algorithm for all + matched nodes, including the assignment made by the mixed precision algorithm and the `ignored_scope`, + `all_layers` and `backup_mode` options. + + Example: + + .. code-block:: python + + import nncf + + annotation = nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=['.*self_attn.*', '.*router.*']), + config=nncf.WeightCompressionConfig(mode=nncf.CompressWeightsMode.INT8_ASYM, group_size=-1), + ) + + :param scope: Defines the portion of a model to annotate. + :type scope: nncf.CustomAnnotationScope + :param config: Weight compression configuration to apply to the matched nodes. + :type config: nncf.WeightCompressionConfig + """ + + scope: CustomAnnotationScope = field(default_factory=CustomAnnotationScope) + config: WeightCompressionConfig = field(default_factory=WeightCompressionConfig) + + @api() @dataclass class AdvancedAccuracyRestorerParameters: diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 20c3de05102..ac5365c57ab 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -38,6 +38,7 @@ from nncf.parameters import CompressWeightsMode from nncf.parameters import SensitivityMetric from nncf.quantization.advanced_parameters import AdvancedCompressionParameters +from nncf.quantization.advanced_parameters import CustomAnnotation from nncf.quantization.advanced_parameters import GroupSizeFallbackMode from nncf.quantization.advanced_parameters import convert_to_dict_recursively from nncf.quantization.algorithms.algorithm import Algorithm @@ -50,6 +51,7 @@ from nncf.quantization.algorithms.weight_compression.scale_estimation import ScaleEstimation from nncf.quantization.algorithms.weight_compression.weight_lowering import WeightCompressionConfig from nncf.quantization.algorithms.weight_compression.weight_lowering import get_reduction_channel_size +from nncf.scopes import CustomAnnotationScope from nncf.scopes import IgnoredScope from nncf.scopes import get_ignored_node_names_from_ignored_scope from nncf.tensor import Tensor @@ -85,6 +87,7 @@ def get_weight_compression_configuration( sensitivity_metric: SensitivityMetric | None = None, backup_mode: BackupMode | None = None, advanced_parameters: AdvancedCompressionParameters | None = None, + custom_annotation: list[CustomAnnotation] | None = None, ) -> dict[str, Any]: """ Generates a configuration dictionary for weight compression based on the provided parameters. @@ -134,9 +137,56 @@ def get_weight_compression_configuration( ), "backup_mode": backup_mode, "advanced_parameters": advanced_parameters or AdvancedCompressionParameters(), + "custom_annotation": custom_annotation, } +def check_custom_annotation(custom_annotation: list[CustomAnnotation] | None) -> None: + """ + Validates the user-defined custom annotation. + + :param custom_annotation: List of custom annotations to validate. + """ + if custom_annotation is None: + return + + if not isinstance(custom_annotation, (list, tuple)): + msg = ( + "The custom_annotation option expects a list of nncf.CustomAnnotation instances, " + f"but {type(custom_annotation).__name__} is given." + ) + raise nncf.ValidationError(msg) + + for annotation in custom_annotation: + if not isinstance(annotation, CustomAnnotation): + msg = ( + "The custom_annotation option expects a list of nncf.CustomAnnotation instances, " + f"but an element of type {type(annotation).__name__} is given." + ) + raise nncf.ValidationError(msg) + + if not isinstance(annotation.config, WeightCompressionConfig): + msg = ( + "The config of a custom annotation must be an nncf.WeightCompressionConfig instance, " + f"but {type(annotation.config).__name__} is given." + ) + raise nncf.ValidationError(msg) + + if not isinstance(annotation.scope, CustomAnnotationScope): + msg = ( + "The scope of a custom annotation must be an nncf.CustomAnnotationScope instance, " + f"but {type(annotation.scope).__name__} is given." + ) + raise nncf.ValidationError(msg) + + if annotation.config.is_codebook and annotation.config.codebook_values is None: + msg = ( + f"Codebook values must be provided in the custom annotation config for the " + f"{annotation.config.mode.value} compression mode." + ) + raise nncf.ValidationError(msg) + + def check_user_compression_configuration( mode: CompressWeightsMode, subset_size: int, @@ -153,10 +203,12 @@ def check_user_compression_configuration( backup_mode: BackupMode | None, compression_format: CompressionFormat | None, advanced_parameters: AdvancedCompressionParameters | None, + custom_annotation: list[CustomAnnotation] | None = None, ) -> None: """ Validates the user's weight compression configuration for correctness. """ + check_custom_annotation(custom_annotation) if mode in INT8_MODES: if (ratio and ratio != 1) or (group_size and group_size != -1): msg = ( @@ -319,6 +371,7 @@ def __init__( backup_mode: BackupMode, compression_format: CompressionFormat = CompressionFormat.DQ, advanced_parameters: AdvancedCompressionParameters | None = None, + custom_annotation: list[CustomAnnotation] | None = None, ): """ :param mode: Defines a mode for weight compression. @@ -365,6 +418,9 @@ def __init__( FP8_E4M3 stands for FP8 format with E4M3 values sharing group-level fp16 scale. :param compression_format: Describes the format in which the model is saved after weight compression. :param advanced_parameters: advanced parameters for algorithms in compression pipeline. + :param custom_annotation: A list of user-defined weight compression configurations bound to portions of + the model. The configuration of a custom annotation takes precedence over the one assigned by the + algorithm for all the matched nodes. """ super().__init__() self._mode = mode @@ -387,6 +443,7 @@ def __init__( self._advanced_parameters = ( advanced_parameters if advanced_parameters is not None else AdvancedCompressionParameters() ) + self._custom_annotation = custom_annotation criterion_cls = MIXED_PRECISION_CRITERIA.get(self._sensitivity_metric) self._mixed_precision_algo = criterion_cls(self._ratio, self._subset_size) @@ -1089,6 +1146,163 @@ def get_weight_compression_parameters( return all_weight_params, ratio_defining_params, skipped_weight_params + def _get_custom_annotation_configs(self, graph: NNCFGraph) -> dict[str, WeightCompressionConfig]: + """ + Maps node names matched by the custom annotation to the user-defined weight compression configuration. + + If several annotations match the same node, the last one takes precedence. + + :param graph: NNCFGraph instance. + :return: A mapping from a node name to the user-defined weight compression configuration. + """ + node_name_to_config: dict[str, WeightCompressionConfig] = {} + for annotation in self._custom_annotation: + annotated_names = get_ignored_node_names_from_ignored_scope( + annotation.scope, graph, strict=annotation.scope.validate, scope_kind="Annotated" + ) + overlapped_names = sorted(annotated_names & node_name_to_config.keys()) + if overlapped_names: + nncf_logger.warning( + "Several custom annotations match the same nodes. The last matched annotation takes " + "precedence for the following nodes:\n\t" + "\n\t".join(overlapped_names) + ) + for node_name in annotated_names: + node_name_to_config[node_name] = annotation.config + return node_name_to_config + + def _validate_custom_annotation_group_size(self, weight_params: list[WeightCompressionParameters]) -> None: + """ + Validates that the group size defined by the custom annotation is divisible by the reduction channel size + for all the annotated weights. Group size fallback modes are not applied to the annotated weights since + the group size is defined by the user explicitly. + + :param weight_params: List of annotated weight parameters. + """ + failed_nodes = [] + for w_params in weight_params: + group_size = w_params.compression_config.group_size + if group_size == -1: + continue + reduction_channel_size, _ = get_reduction_channel_size(w_params.weight_shape, w_params.reduction_axes) + if reduction_channel_size % group_size != 0: + failed_nodes.append((w_params.node_with_weight.node_name, reduction_channel_size, group_size)) + if failed_nodes: + names = "\n\t".join( + f'"{name}" (channel size: {channel_size}, group size: {group_size})' + for name, channel_size, group_size in failed_nodes + ) + msg = ( + "Failed to apply group-wise quantization with the group size value defined by the custom " + "annotation. Ensure that the group size is divisible by the channel size. Failed nodes:\n\t" + names + ) + raise nncf.InvalidGroupSizeError(msg) + + def apply_custom_annotation( + self, + graph: NNCFGraph, + all_weight_params: list[WeightCompressionParameters], + ratio_defining_params: list[WeightCompressionParameters], + skipped_weight_params: list[WeightCompressionParameters], + ) -> tuple[ + list[WeightCompressionParameters], + list[WeightCompressionParameters], + list[WeightCompressionParameters], + ]: + """ + Assigns the user-defined weight compression configurations to the weight parameters matched by the + custom annotation. + + The annotated parameters are excluded from the ratio-defining ones, so that neither the mixed precision + algorithm nor the `all_layers` and the last layer rules can override the user-defined configuration. + The parameters that were skipped, e.g. by the ignored scope, are returned back to the compressed ones + if they are matched by the custom annotation. + + :param graph: NNCFGraph instance. + :param all_weight_params: List of all weight compression parameters that can be compressed. + :param ratio_defining_params: List of ratio-defining parameters. + :param skipped_weight_params: List of weight compression parameters that are not compressed. + :return: The updated tuple of all, ratio-defining and skipped weight compression parameters. + """ + if not self._custom_annotation: + return all_weight_params, ratio_defining_params, skipped_weight_params + + node_name_to_config = self._get_custom_annotation_configs(graph) + compressed_weight_names = set(w_params.weight_name for w_params in all_weight_params) + annotated_weight_params = [] + restored_weight_params = [] + unsupported_node_names = [] + + def annotate(w_params: WeightCompressionParameters) -> bool: + config = node_name_to_config.get(w_params.node_with_weight.node_name) + if config is None: + return False + if not self.is_weight_compression_supported(w_params.weight_dtype, config.mode): + unsupported_node_names.append(w_params.node_with_weight.node_name) + return False + # A deep copy is required since some algorithms, e.g. codebook estimation, update the compression + # config of a weight parameter in place. + w_params.compression_config = copy.deepcopy(config) + annotated_weight_params.append(w_params) + return True + + for w_params in all_weight_params: + annotate(w_params) + + for w_params in skipped_weight_params: + # A weight that is already compressed under another node, e.g. a shared weight, must not be restored + # to avoid processing the same weight more than once. + if w_params.weight_name in compressed_weight_names: + continue + if annotate(w_params): + compressed_weight_names.add(w_params.weight_name) + restored_weight_params.append(w_params) + + if unsupported_node_names: + nncf_logger.warning( + "The compression mode defined by the custom annotation is not supported for the data type of the " + "following nodes. They are kept with the configuration assigned by the algorithm:\n\t" + + "\n\t".join(sorted(unsupported_node_names)) + ) + + matched_node_names = set( + w_params.node_with_weight.node_name + for w_params in all_weight_params + skipped_weight_params + if w_params.node_with_weight.node_name in node_name_to_config + ) + not_applied_node_names = node_name_to_config.keys() - matched_node_names + if not_applied_node_names: + nncf_logger.warning( + "The following nodes are matched by the custom annotation, but have no weight to compress. " + "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(not_applied_node_names)) + ) + + self._validate_custom_annotation_group_size(annotated_weight_params) + + if restored_weight_params: + nncf_logger.warning( + "The following nodes are excluded from the compression, e.g. by the ignored scope, but are matched " + "by the custom annotation. They will be compressed with the user-defined configuration:\n\t" + + "\n\t".join(sorted(w_params.node_with_weight.node_name for w_params in restored_weight_params)) + ) + restored_weight_names = set(w_params.weight_name for w_params in restored_weight_params) + skipped_weight_params = [ + w_params for w_params in skipped_weight_params if w_params.weight_name not in restored_weight_names + ] + # Keep the topological order of the weight parameters + node_name_to_position = {node.node_name: i for i, node in enumerate(graph.topological_sort())} + all_weight_params = sorted( + all_weight_params + restored_weight_params, + key=lambda wp: node_name_to_position[wp.node_with_weight.node_name], + ) + + # The user-defined configuration takes precedence over the mixed precision assignment + annotated_weight_names = set(w_params.weight_name for w_params in annotated_weight_params) + ratio_defining_params = [ + w_params for w_params in ratio_defining_params if w_params.weight_name not in annotated_weight_names + ] + + return all_weight_params, ratio_defining_params, skipped_weight_params + def apply( self, model: TModel, @@ -1102,6 +1316,10 @@ def apply( all_weight_params, ratio_defining_params, skipped_weight_params = self.get_weight_compression_parameters( model, graph ) + # Assign the user-defined weight compression configurations + all_weight_params, ratio_defining_params, skipped_weight_params = self.apply_custom_annotation( + graph, all_weight_params, ratio_defining_params, skipped_weight_params + ) # Collect statistics for the weights compression statistics, statistic_points = self.collect_statistics_and_statistic_points( model, graph, statistic_points, dataset, ratio_defining_params, all_weight_params @@ -1228,6 +1446,7 @@ def apply_with_parameters( "backup_mode": self._backup_mode.value, "compression_format": self._compression_format.value, "advanced_parameters": convert_to_dict_recursively(self._advanced_parameters), + "custom_annotation": self._custom_annotation or [], }, algo_name="weight_compression", ) diff --git a/src/nncf/quantization/algorithms/weight_compression/config.py b/src/nncf/quantization/algorithms/weight_compression/config.py index 5e99da33329..c987d0003c0 100644 --- a/src/nncf/quantization/algorithms/weight_compression/config.py +++ b/src/nncf/quantization/algorithms/weight_compression/config.py @@ -16,6 +16,7 @@ import numpy as np from nncf.common.graph.graph import NNCFNode +from nncf.common.utils.api_marker import api from nncf.errors import InternalError from nncf.errors import ValidationError from nncf.parameters import CompressWeightsMode @@ -23,6 +24,7 @@ from nncf.tensor.definitions import TensorDataType +@api(canonical_alias="nncf.WeightCompressionConfig") @dataclass class WeightCompressionConfig: """ diff --git a/src/nncf/quantization/quantize_model.py b/src/nncf/quantization/quantize_model.py index cdb2e0bdba3..568ff9e39b4 100644 --- a/src/nncf/quantization/quantize_model.py +++ b/src/nncf/quantization/quantize_model.py @@ -30,6 +30,7 @@ from nncf.quantization.advanced_parameters import AdvancedAccuracyRestorerParameters from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters +from nncf.quantization.advanced_parameters import CustomAnnotation from nncf.quantization.algorithms.accuracy_control.evaluator import MetricResults from nncf.quantization.algorithms.hyperparameter_tuner.algorithm import HyperparameterTuner from nncf.quantization.algorithms.hyperparameter_tuner.param_grid import get_quantization_param_grids @@ -424,6 +425,7 @@ def compress_weights( backup_mode: BackupMode | None = None, compression_format: CompressionFormat = CompressionFormat.DQ, advanced_parameters: AdvancedCompressionParameters | None = None, + custom_annotation: list[CustomAnnotation] | None = None, ) -> TModel: """ Compress model weights. @@ -492,6 +494,13 @@ def compress_weights( :type compression_format: nncf.CompressionFormat :param advanced_parameters: Advanced parameters for compression algorithms. :type advanced_parameters: nncf.AdvancedCompressionParameters + :param custom_annotation: A list of user-defined weight compression configurations bound to portions of the + model. For all the matched nodes the given configuration is used instead of the one assigned by the + algorithm, which makes it possible to compress certain layers, e.g. attention or MoE router layers, to a + precision different from the one defined by `mode`, `ratio` and `backup_mode`. The custom annotation + takes precedence over the mixed precision assignment, the `ignored_scope` and the `all_layers` options. + If several annotations match the same node, the last one takes precedence. + :type custom_annotation: list[nncf.CustomAnnotation] :return: The non-trainable model with compressed weights. """ backend = get_backend(model) @@ -671,6 +680,7 @@ def compress_weights( backup_mode, compression_format, advanced_parameters, + custom_annotation, ) weight_compression_configuration = get_weight_compression_configuration( mode, @@ -686,6 +696,7 @@ def compress_weights( sensitivity_metric, backup_mode, advanced_parameters, + custom_annotation, ) return compression_weights_impl( diff --git a/src/nncf/scopes.py b/src/nncf/scopes.py index 93a6c34bb32..4dd7ab1c4c4 100644 --- a/src/nncf/scopes.py +++ b/src/nncf/scopes.py @@ -60,9 +60,29 @@ def get_ignored_node_names_from_subgraph(graph: NNCFGraph, subgraph: Subgraph) - return list(sorted(ignored_names)) +@dataclass +class BaseScope: + """ + Defines a portion of a model by a set of matching rules against the model graph. + + :param names: List of node names. + :param patterns: List of regular expressions that define patterns for names of nodes. + :param types: List of operation types. + :param subgraphs: List of subgraphs. + :param validate: If set to True, then an error will be raised if any rule does not match + in the model graph. + """ + + names: list[str] = field(default_factory=list) + patterns: list[str] = field(default_factory=list) + types: list[str] = field(default_factory=list) + subgraphs: list[Subgraph] = field(default_factory=list) + validate: bool = True + + @api(canonical_alias="nncf.IgnoredScope") @dataclass -class IgnoredScope: +class IgnoredScope(BaseScope): r""" Provides an option to specify portions of model to be excluded from compression. @@ -108,14 +128,48 @@ class IgnoredScope: :type types: bool """ - names: list[str] = field(default_factory=list) - patterns: list[str] = field(default_factory=list) - types: list[str] = field(default_factory=list) - subgraphs: list[Subgraph] = field(default_factory=list) - validate: bool = True + +@api(canonical_alias="nncf.CustomAnnotationScope") +@dataclass +class CustomAnnotationScope(BaseScope): + r""" + Provides an option to specify portions of model to be annotated with a custom compression configuration. + + The custom annotation scope defines model nodes for which the user-defined compression configuration is + used instead of the one assigned by the algorithm. Matching rules are the same as for + :class:`nncf.IgnoredScope`. + + Example: + + .. code-block:: python + + import nncf + + # Annotate by node name: + node_names = ['node_1', 'node_2', 'node_3'] + scope = nncf.CustomAnnotationScope(names=node_names) + + # Annotate using regular expressions: + patterns = ['.*self_attn.*', '.*router.*'] + scope = nncf.CustomAnnotationScope(patterns=patterns) + + **Note:** Operation types must be specified according to the model framework. + + :param names: List of annotated node names. + :type names: List[str] + :param patterns: List of regular expressions that define patterns for names of annotated nodes. + :type patterns: List[str] + :param types: List of annotated operation types. + :type types: List[str] + :param subgraphs: List of annotated subgraphs. + :type subgraphs: List[Subgraph] + :param validate: If set to True, then an error will be raised if any annotation rule does not match + in the model graph. + :type validate: bool + """ -def get_difference_ignored_scope(ignored_scope_1: IgnoredScope, ignored_scope_2: IgnoredScope) -> IgnoredScope: +def get_difference_ignored_scope(ignored_scope_1: BaseScope, ignored_scope_2: BaseScope) -> IgnoredScope: """ Returns ignored scope with rules from 'ignored_scope_1' not presented at 'ignored_scope_2' @@ -153,7 +207,7 @@ def convert_ignored_scope_to_list(ignored_scope: IgnoredScope | None) -> list[st def get_matched_ignored_scope_info( - ignored_scope: IgnoredScope, nncf_graphs: list[NNCFGraph] + ignored_scope: BaseScope, nncf_graphs: list[NNCFGraph] ) -> tuple[IgnoredScope, dict[str, set[str]]]: """ Returns matched ignored scope for provided graphs along with all found matches. @@ -201,43 +255,50 @@ def get_matched_ignored_scope_info( return matched_ignored_scope, matches -def _info_matched_ignored_scope(matches: dict[str, set[str]]) -> None: +def _info_matched_ignored_scope(matches: dict[str, set[str]], scope_kind: str = "Ignored") -> None: """ Log matches. :param matches: Matches. + :param scope_kind: Human-readable kind of the scope to mention in the message. """ for rule_type, rules in matches.items(): if rules: - nncf_logger.info(f"{len(rules)} ignored nodes were found by {rule_type} in the NNCFGraph") + nncf_logger.info(f"{len(rules)} {scope_kind.lower()} nodes were found by {rule_type} in the NNCFGraph") -def _error_unmatched_ignored_scope(unmatched_ignored_scope: IgnoredScope) -> str: +def _error_unmatched_ignored_scope(unmatched_ignored_scope: IgnoredScope, scope_kind: str = "Ignored") -> str: """ Returns an error message for unmatched ignored scope. :param unmatched_ignored_scope: Unmatched ignored scope. + :param scope_kind: Human-readable kind of the scope to mention in the message. :return str: Error message. """ err_msg = "\n" for ignored_type in ("names", "types", "patterns"): unmatched_rules = getattr(unmatched_ignored_scope, ignored_type) if unmatched_rules: - err_msg += f"Ignored nodes that matches {ignored_type} {unmatched_rules} were not found in the NNCFGraph.\n" + err_msg += ( + f"{scope_kind} nodes that matches {ignored_type} {unmatched_rules} were not found in the NNCFGraph.\n" + ) for subgraph in unmatched_ignored_scope.subgraphs: err_msg += ( - f"Ignored nodes that matches subgraph with input names {subgraph.inputs} " + f"{scope_kind} nodes that matches subgraph with input names {subgraph.inputs} " f"and output names {subgraph.outputs} were not found in the NNCFGraph.\n" ) return err_msg -def _check_ignored_scope_strictly_matched(ignored_scope: IgnoredScope, matched_ignored_scope: IgnoredScope) -> None: +def _check_ignored_scope_strictly_matched( + ignored_scope: BaseScope, matched_ignored_scope: IgnoredScope, scope_kind: str = "Ignored" +) -> None: """ Passes when ignored_scope and matched_ignored_scope are equal, otherwise - raises ValidationError. :param ignored_scope: Ignored scope. :param matched_ignored_scope: Matched ignored scope. + :param scope_kind: Human-readable kind of the scope to mention in the error message. """ unmatched_ignored_scope = get_difference_ignored_scope(ignored_scope, matched_ignored_scope) if ( @@ -246,11 +307,11 @@ def _check_ignored_scope_strictly_matched(ignored_scope: IgnoredScope, matched_i or unmatched_ignored_scope.patterns or unmatched_ignored_scope.subgraphs ): - raise nncf.ValidationError(_error_unmatched_ignored_scope(unmatched_ignored_scope)) + raise nncf.ValidationError(_error_unmatched_ignored_scope(unmatched_ignored_scope, scope_kind)) def get_ignored_node_names_from_ignored_scope( - ignored_scope: IgnoredScope, nncf_graph: NNCFGraph, strict: bool = True + ignored_scope: BaseScope, nncf_graph: NNCFGraph, strict: bool = True, scope_kind: str = "Ignored" ) -> set[str]: """ Returns ignored names according to ignored scope and NNCFGraph. @@ -260,16 +321,17 @@ def get_ignored_node_names_from_ignored_scope( :param ignored_scope: Ignored scope. :param nncf_graph: Graph. :param strict: Whether all ignored_scopes must match at least one node or not. + :param scope_kind: Human-readable kind of the scope to mention in the error message. :return: NNCF node names from given NNCFGraph specified in given ignored scope. """ matched_ignored_scope, matches = get_matched_ignored_scope_info(ignored_scope, [nncf_graph]) if strict: - _check_ignored_scope_strictly_matched(ignored_scope, matched_ignored_scope) - _info_matched_ignored_scope(matches) + _check_ignored_scope_strictly_matched(ignored_scope, matched_ignored_scope, scope_kind) + _info_matched_ignored_scope(matches, scope_kind) return {name for match in matches.values() for name in match} -def validate_ignored_scope(ignored_scope: IgnoredScope, nncf_graphs: list[NNCFGraph]) -> None: +def validate_ignored_scope(ignored_scope: BaseScope, nncf_graphs: list[NNCFGraph]) -> None: """ Passes whether all rules at 'ignored_scope' have matches at provided graphs, otherwise - raises ValidationError. diff --git a/src/nncf/torch/function_hook/quantization/quantize_model.py b/src/nncf/torch/function_hook/quantization/quantize_model.py index ac3e502fdd9..67170951452 100644 --- a/src/nncf/torch/function_hook/quantization/quantize_model.py +++ b/src/nncf/torch/function_hook/quantization/quantize_model.py @@ -26,6 +26,7 @@ from nncf.parameters import TargetDevice from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters +from nncf.quantization.advanced_parameters import CustomAnnotation from nncf.quantization.algorithms.post_training.algorithm import PostTrainingQuantization from nncf.quantization.algorithms.weight_compression.algorithm import WeightCompression from nncf.quantization.quantize_model import warning_model_no_batchwise_support @@ -100,6 +101,7 @@ def compress_weights_impl( backup_mode: BackupMode, compression_format: CompressionFormat, advanced_parameters: AdvancedCompressionParameters | None = None, + custom_annotation: list[CustomAnnotation] | None = None, ) -> torch.nn.Module: """ Implementation of the `compress_weights()` method for the PyTorch backend. @@ -119,6 +121,7 @@ def compress_weights_impl( backup_mode, compression_format, advanced_parameters, + custom_annotation, ) graph = build_graph(model) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index c72c1964f2b..28499dfac68 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -926,6 +926,208 @@ def test_group_size_fallback_modes( f"Expected {ref_num_group_sizes} group size values, but got {num_group_sizes}." ) + # Test Custom Annotation + + def _get_sequential_matmul_model(self): + """Returns a backend model with a sequence of MatMul layers, ready to be compressed.""" + return self.wrap_model( + self.get_sequential_matmul_model(transpose_a=False), np.ones([1, 4, 4], dtype=np.float32) + ) + + def _compress_and_get_configs(self, mocker, **kwargs) -> dict[str, WeightCompressionConfig]: + """ + Compresses a model and returns the compression config assigned to each compressed weight node. + + :param mocker: Mocker fixture. + :param kwargs: Arguments for the `compress_weights` function. + :return: A mapping from a node name to the assigned compression config. + """ + spy = mocker.spy(WeightCompression, "apply_with_parameters") + compress_weights(**kwargs) + all_weight_params = spy.call_args.args[5] + return {wp.node_with_weight.node_name: wp.compression_config for wp in all_weight_params} + + def test_custom_annotation(self, mocker): + """ + Checks that the config given by the custom annotation is assigned to the matched nodes, overriding both + the primary precision and the backup precision of the last layer, and that the rest of the nodes is not + affected. + """ + common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=False) + reference_configs = self._compress_and_get_configs( + mocker, model=self._get_sequential_matmul_model(), **common_kwargs + ) + node_names = list(reference_configs) + # The last layer is compressed to the backup precision since all_layers is False + assert reference_configs[node_names[-1]].mode == CompressWeightsMode.INT8_ASYM + + annotated_config = WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1) + last_layer_config = WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4) + configs = self._compress_and_get_configs( + mocker, + model=self._get_sequential_matmul_model(), + **common_kwargs, + custom_annotation=[ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[node_names[1]]), + config=annotated_config, + ), + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[node_names[-1]]), + config=last_layer_config, + ), + ], + ) + + assert configs[node_names[1]] == annotated_config + assert configs[node_names[-1]] == last_layer_config + for node_name in node_names[2:-1]: + assert configs[node_name] == reference_configs[node_name] + + def test_custom_annotation_over_mixed_precision(self, mocker): + """ + Checks that an annotated node is excluded from the ratio-defining parameters, so that the mixed precision + algorithm can not override the config given by the custom annotation. + """ + annotated_config = WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=-1) + common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=0.5, group_size=-1, all_layers=True) + node_names = list( + self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) + ) + + mixed_precision_spy = mocker.spy(WeightCompression, "apply_mixed_precision") + configs = self._compress_and_get_configs( + mocker, + model=self._get_sequential_matmul_model(), + **common_kwargs, + custom_annotation=[ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[node_names[1]]), + config=annotated_config, + ) + ], + ) + + ratio_defining_params = mixed_precision_spy.call_args.args[1] + ratio_defining_names = [wp.node_with_weight.node_name for wp in ratio_defining_params] + assert node_names[1] not in ratio_defining_names + assert configs[node_names[1]] == annotated_config + + def test_custom_annotation_over_ignored_scope(self, mocker): + """ + Checks that a node from the ignored scope is compressed with the config given by the custom annotation + and that a warning is logged. + """ + annotated_config = WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1) + common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) + node_names = list( + self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) + ) + ignored_node_name = node_names[1] + + with patch.object(nncf_logger, "warning") as mock_warning: + configs = self._compress_and_get_configs( + mocker, + model=self._get_sequential_matmul_model(), + **common_kwargs, + ignored_scope=IgnoredScope(names=[ignored_node_name]), + custom_annotation=[ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[ignored_node_name]), + config=annotated_config, + ) + ], + ) + + assert configs[ignored_node_name] == annotated_config + assert list(configs) == node_names, "The topological order of the weight parameters is not kept." + warning_messages = [args[0] for args, _ in mock_warning.call_args_list] + warn_msg = "are excluded from the compression, e.g. by the ignored scope, but are matched" + assert any(warn_msg in msg for msg in warning_messages) + + def test_custom_annotation_overlap(self, mocker): + """ + Checks that the last matched annotation takes precedence and that a warning is logged. + """ + common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) + node_names = list( + self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) + ) + first_config = WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1) + last_config = WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4) + + with patch.object(nncf_logger, "warning") as mock_warning: + configs = self._compress_and_get_configs( + mocker, + model=self._get_sequential_matmul_model(), + **common_kwargs, + custom_annotation=[ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[node_names[1], node_names[2]]), + config=first_config, + ), + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[node_names[2]]), + config=last_config, + ), + ], + ) + + assert configs[node_names[1]] == first_config + assert configs[node_names[2]] == last_config + warning_messages = [args[0] for args, _ in mock_warning.call_args_list] + warn_msg = "Several custom annotations match the same nodes" + assert any(warn_msg in msg for msg in warning_messages) + + def test_custom_annotation_without_weights(self, mocker): + """ + Checks that a warning is logged when the custom annotation matches only the nodes that have no weight. + """ + model = self._get_sequential_matmul_model() + input_node_name = nncf.build_graph(model).get_input_nodes()[0].node_name + + with patch.object(nncf_logger, "warning") as mock_warning: + self._compress_and_get_configs( + mocker, + model=self._get_sequential_matmul_model(), + mode=CompressWeightsMode.INT4_SYM, + ratio=1.0, + group_size=-1, + custom_annotation=[ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[input_node_name]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + ], + ) + + warning_messages = [args[0] for args, _ in mock_warning.call_args_list] + warn_msg = "are matched by the custom annotation, but have no weight to compress" + assert any(warn_msg in msg for msg in warning_messages) + + def test_custom_annotation_with_invalid_group_size(self, mocker): + """ + Checks that an error is raised when the group size given by the custom annotation is not divisible by the + channel size. + """ + common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) + node_names = list( + self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) + ) + + with pytest.raises(InvalidGroupSizeError) as exc_info: + compress_weights( + model=self._get_sequential_matmul_model(), + **common_kwargs, + custom_annotation=[ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[node_names[1]]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_SYM, group_size=3), + ) + ], + ) + assert "group size value defined by the custom annotation" in str(exc_info.value) + @pytest.mark.parametrize("is_3d_weights", [True, False]) @pytest.mark.parametrize("dataset", [None, np.ones([2, 8, 8], dtype=np.float32)]) @pytest.mark.parametrize("prefer_data_aware_scaling", [True, False]) diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 3d363a4015a..2fce7d0222f 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -923,6 +923,62 @@ def test_raise_error_with_unsupported_params_for_int4(mode, params): compress_weights(ov.Model([], []), mode=mode, **params) +@pytest.mark.parametrize( + "custom_annotation", + ( + nncf.CustomAnnotation(), + ["anything"], + [nncf.CustomAnnotation(scope=nncf.IgnoredScope())], + [nncf.CustomAnnotation(config="anything")], + [ + nncf.CustomAnnotation( + config=WeightCompressionConfig(mode=CompressWeightsMode.CODEBOOK, codebook_values=None) + ) + ], + ), + ids=["not_a_list", "not_an_annotation", "wrong_scope", "wrong_config", "codebook_without_values"], +) +def test_raise_error_with_invalid_custom_annotation(custom_annotation): + with pytest.raises(nncf.ValidationError): + compress_weights(ov.Model([], []), mode=CompressWeightsMode.INT4_SYM, custom_annotation=custom_annotation) + + +@pytest.mark.parametrize( + "custom_annotation, expected_dump", + [ + (None, {"": "[]"}), + ( + [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=["MatMul.*"], types=["MatMul"]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ), + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=["MatMul_1"]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), + ), + ], + { + "0/scope/patterns": "['MatMul.*']", + "0/scope/types": "['MatMul']", + "0/config": "{'mode': 'int8_sym', 'group_size': -1}", + "1/scope/names": "['MatMul_1']", + "1/config": "{'mode': 'int4_asym', 'group_size': 3}", + }, + ), + ], + ids=["without_annotation", "with_annotation"], +) +def test_dump_custom_annotation(custom_annotation, expected_dump): + model = IntegerModel().ov_model + compressed_model = compress_weights(model, mode=CompressWeightsMode.INT8_ASYM, custom_annotation=custom_annotation) + + rt_path = ["nncf", "weight_compression", "custom_annotation"] + for key, value in expected_dump.items(): + path = rt_path + key.split("/") if key else rt_path + assert compressed_model.get_rt_info(path).astype(str) == value + + @pytest.mark.parametrize("mode", [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3, CompressWeightsMode.NVFP4]) class TestUnsupportedParams: @pytest.mark.parametrize( From feb3cfaa6d013684b48328abec18796dbe476723 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 13 Aug 2026 15:08:24 +0400 Subject: [PATCH 03/43] change ingored scope util function names to common name --- .../sparsify_activations_impl.py | 6 +- .../sparsify_activations/target_scope.py | 8 +- .../openvino/quantization/quantize_model.py | 4 +- .../algorithms/min_max/algorithm.py | 4 +- .../weight_compression/algorithm.py | 10 +- src/nncf/scopes.py | 139 ++++++++---------- .../function_hook/pruning/prune_model.py | 4 +- src/nncf/version.py | 2 +- tests/common/test_ignored_scope.py | 6 +- tests/common/test_scopes.py | 4 +- 10 files changed, 87 insertions(+), 100 deletions(-) diff --git a/src/nncf/experimental/torch/sparsify_activations/sparsify_activations_impl.py b/src/nncf/experimental/torch/sparsify_activations/sparsify_activations_impl.py index d63087ceb1a..17da6016f03 100644 --- a/src/nncf/experimental/torch/sparsify_activations/sparsify_activations_impl.py +++ b/src/nncf/experimental/torch/sparsify_activations/sparsify_activations_impl.py @@ -27,7 +27,7 @@ from nncf.experimental.torch.sparsify_activations.target_scope import TargetScope from nncf.experimental.torch.sparsify_activations.target_scope import get_target_node_names_from_target_scope from nncf.scopes import IgnoredScope -from nncf.scopes import get_ignored_node_names_from_ignored_scope +from nncf.scopes import get_node_names_from_scope from nncf.torch.model_creation import is_wrapped_model from nncf.torch.model_creation import wrap_model @@ -181,9 +181,7 @@ def _get_target_sparsity_by_node(self, graph: NNCFGraph) -> dict[NNCFNode, float :return: A dictionary with nodes and the corresponding target sparsity level. """ supported_metatypes = self._backend_entity.supported_metatypes - ignored_names = get_ignored_node_names_from_ignored_scope( - self._ignored_scope, graph, strict=self._ignored_scope.validate - ) + ignored_names = get_node_names_from_scope(self._ignored_scope, graph, strict=self._ignored_scope.validate) target_sparsity_by_node = {} for scope, target_sparsity in self._target_sparsity_by_scope.items(): target_names = get_target_node_names_from_target_scope(scope, graph, strict=scope.validate) diff --git a/src/nncf/experimental/torch/sparsify_activations/target_scope.py b/src/nncf/experimental/torch/sparsify_activations/target_scope.py index efc98223dd4..103cbc70551 100644 --- a/src/nncf/experimental/torch/sparsify_activations/target_scope.py +++ b/src/nncf/experimental/torch/sparsify_activations/target_scope.py @@ -14,8 +14,8 @@ import nncf from nncf.common.graph.graph import NNCFGraph from nncf.scopes import IgnoredScope -from nncf.scopes import get_difference_ignored_scope -from nncf.scopes import get_matched_ignored_scope_info +from nncf.scopes import get_difference_scope +from nncf.scopes import get_matched_scope_info @dataclass @@ -84,7 +84,7 @@ def get_target_node_names_from_target_scope( :param strict: Whether target_scope must match at least one node or not. :return: NNCF node names from the given graph matched by target scope. """ - matched_target_scope, matches = get_matched_ignored_scope_info(target_scope, [nncf_graph]) + matched_target_scope, matches = get_matched_scope_info(target_scope, [nncf_graph]) if strict: _check_target_scope_strictly_matched(target_scope, matched_target_scope) return set().union(*matches.values()) @@ -97,7 +97,7 @@ def _check_target_scope_strictly_matched(target_scope: TargetScope, matched_targ :param target_scope: The given target scope. :param matched_target_scope: The actual target scope matched in a graph. """ - unmatched_scope = get_difference_ignored_scope(target_scope, matched_target_scope) + unmatched_scope = get_difference_scope(target_scope, matched_target_scope) error_messages = [] for match_type in ("names", "types", "patterns", "subgraphs"): unmatched_rules = getattr(unmatched_scope, match_type) diff --git a/src/nncf/openvino/quantization/quantize_model.py b/src/nncf/openvino/quantization/quantize_model.py index d42cbb1f88f..f71ce7d7106 100644 --- a/src/nncf/openvino/quantization/quantize_model.py +++ b/src/nncf/openvino/quantization/quantize_model.py @@ -56,7 +56,7 @@ from nncf.quantization.statistics_caching import cache_weight_compression_statistics from nncf.quantization.statistics_caching import register_statistics_for_algorithm from nncf.scopes import IgnoredScope -from nncf.scopes import validate_ignored_scope +from nncf.scopes import validate_scope TTensor = TypeVar("TTensor") @@ -97,7 +97,7 @@ def _extract_all_subgraphs(model: ov.Model, current_id: str) -> None: main_model_graph_id = "main_model_graph" _extract_all_subgraphs(model, main_model_graph_id) if ignored_scope and ignored_scope.validate: - validate_ignored_scope(ignored_scope, graphs.values()) + validate_scope(ignored_scope, graphs.values()) ignored_scope = IgnoredScope( ignored_scope.names, ignored_scope.patterns, ignored_scope.types, ignored_scope.subgraphs, validate=False ) diff --git a/src/nncf/quantization/algorithms/min_max/algorithm.py b/src/nncf/quantization/algorithms/min_max/algorithm.py index 34330dd5b39..ebc9644c4c3 100644 --- a/src/nncf/quantization/algorithms/min_max/algorithm.py +++ b/src/nncf/quantization/algorithms/min_max/algorithm.py @@ -69,7 +69,7 @@ from nncf.quantization.range_estimator import RangeEstimatorParametersSet from nncf.quantization.range_estimator import StatisticsType from nncf.scopes import IgnoredScope -from nncf.scopes import get_ignored_node_names_from_ignored_scope +from nncf.scopes import get_node_names_from_scope TModel = TypeVar("TModel") @@ -613,7 +613,7 @@ def _get_ignored_names( :param ignored_patterns: Ignored patterns. :return: Ignored node names and ignore reason for quantization. """ - user_ignored_names = get_ignored_node_names_from_ignored_scope( + user_ignored_names = get_node_names_from_scope( self._ignored_scope, nncf_graph, strict=self._ignored_scope.validate ) autogenerated_ignored_names = get_node_names_matching_graph_pattern(inference_nncf_graph, ignored_patterns) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index ac5365c57ab..3550476bce7 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -53,7 +53,7 @@ from nncf.quantization.algorithms.weight_compression.weight_lowering import get_reduction_channel_size from nncf.scopes import CustomAnnotationScope from nncf.scopes import IgnoredScope -from nncf.scopes import get_ignored_node_names_from_ignored_scope +from nncf.scopes import get_node_names_from_scope from nncf.tensor import Tensor from nncf.tensor import functions as fns from nncf.tensor.definitions import TensorDataType @@ -563,9 +563,7 @@ def get_ignored_node_names(self, nncf_graph: NNCFGraph) -> set[str]: :param nncf_graph: The NNCF graph to analyze. :return: A set of node names to be ignored during weight compression. """ - ignored_names = get_ignored_node_names_from_ignored_scope( - self._ignored_scope, nncf_graph, strict=self._ignored_scope.validate - ) + ignored_names = get_node_names_from_scope(self._ignored_scope, nncf_graph, strict=self._ignored_scope.validate) autogenerated_ignored_names = get_node_names_matching_graph_pattern( nncf_graph, self._backend_entity.get_ignored_patterns() @@ -915,7 +913,7 @@ def _get_ignored_scope_weight_statistics(self, model: TModel, graph: NNCFGraph) :param graph: Model graph. :return: A list of weight sizes for the ignored nodes. """ - ignored_names = get_ignored_node_names_from_ignored_scope(self._ignored_scope, graph, strict=False) + ignored_names = get_node_names_from_scope(self._ignored_scope, graph, strict=False) weighted_metatypes = ( self._backend_entity.matmul_metatypes + self._backend_entity.embedding_metatypes @@ -1157,7 +1155,7 @@ def _get_custom_annotation_configs(self, graph: NNCFGraph) -> dict[str, WeightCo """ node_name_to_config: dict[str, WeightCompressionConfig] = {} for annotation in self._custom_annotation: - annotated_names = get_ignored_node_names_from_ignored_scope( + annotated_names = get_node_names_from_scope( annotation.scope, graph, strict=annotation.scope.validate, scope_kind="Annotated" ) overlapped_names = sorted(annotated_names & node_name_to_config.keys()) diff --git a/src/nncf/scopes.py b/src/nncf/scopes.py index 4dd7ab1c4c4..f24ebdb5821 100644 --- a/src/nncf/scopes.py +++ b/src/nncf/scopes.py @@ -36,28 +36,28 @@ class Subgraph: outputs: list[str] = field(default_factory=list) -def get_ignored_node_names_from_subgraph(graph: NNCFGraph, subgraph: Subgraph) -> list[str]: +def get_node_names_from_subgraph(graph: NNCFGraph, subgraph: Subgraph) -> list[str]: """ - Returns all names that should be ignored according to given subgraph. + Returns all names that are matched by the given subgraph. :param graph: Given NNCFGraph. :param subgraph: Given subgraph instance. - :return: All names that should be ignored according to given subgraph. + :return: All names that are matched by the given subgraph. """ - ignored_names = set() + matched_names = set() for start_node_name in subgraph.inputs: for end_node_name in subgraph.outputs: if start_node_name == end_node_name: # For networkx<3.3 nx.get_all_simple_paths returns empty path for this case node = graph.get_node_by_name(start_node_name) - ignored_names.add(node.node_name) + matched_names.add(node.node_name) continue for path in graph.get_all_simple_paths(start_node_name, end_node_name): for node_key in path: node = graph.get_node_by_key(node_key) - ignored_names.add(node.node_name) + matched_names.add(node.node_name) - return list(sorted(ignored_names)) + return list(sorted(matched_names)) @dataclass @@ -169,20 +169,20 @@ class CustomAnnotationScope(BaseScope): """ -def get_difference_ignored_scope(ignored_scope_1: BaseScope, ignored_scope_2: BaseScope) -> IgnoredScope: +def get_difference_scope(scope_1: BaseScope, scope_2: BaseScope) -> IgnoredScope: """ - Returns ignored scope with rules from 'ignored_scope_1' not presented at 'ignored_scope_2' + Returns scope with rules from 'scope_1' not presented at 'scope_2' - :param ignored_scope_1: First ignored scope. - :param ignored_scope_2: Second ignored scope. - :return: Ignored scope. + :param scope_1: First scope. + :param scope_2: Second scope. + :return: Scope with the rules difference. """ return IgnoredScope( - names=list(set(ignored_scope_1.names) - set(ignored_scope_2.names)), - patterns=list(set(ignored_scope_1.patterns) - set(ignored_scope_2.patterns)), - types=list(set(ignored_scope_1.types) - set(ignored_scope_2.types)), - subgraphs=[subgraph for subgraph in ignored_scope_1.subgraphs if subgraph not in ignored_scope_2.subgraphs], - validate=ignored_scope_1.validate, + names=list(set(scope_1.names) - set(scope_2.names)), + patterns=list(set(scope_1.patterns) - set(scope_2.patterns)), + types=list(set(scope_1.types) - set(scope_2.types)), + subgraphs=[subgraph for subgraph in scope_1.subgraphs if subgraph not in scope_2.subgraphs], + validate=scope_1.validate, ) @@ -206,56 +206,54 @@ def convert_ignored_scope_to_list(ignored_scope: IgnoredScope | None) -> list[st return results -def get_matched_ignored_scope_info( - ignored_scope: BaseScope, nncf_graphs: list[NNCFGraph] -) -> tuple[IgnoredScope, dict[str, set[str]]]: +def get_matched_scope_info(scope: BaseScope, nncf_graphs: list[NNCFGraph]) -> tuple[IgnoredScope, dict[str, set[str]]]: """ - Returns matched ignored scope for provided graphs along with all found matches. - The resulted ignored scope consist of all matched rules. + Returns matched scope for provided graphs along with all found matches. + The resulted scope consist of all matched rules. The found matches consist of a dictionary with a rule name as a key and matched node names as a value. - :param ignored_scope: Ignored scope instance. + :param scope: Scope instance. :param nncf_graphs: Graphs. - :returns: Matched ignored scope along with all matches. + :returns: Matched scope along with all matches. """ names, patterns, types, subgraphs_numbers = set(), set(), set(), set() # type: ignore matches = {"names": names, "patterns": set(), "types": set(), "subgraphs": set()} for graph in nncf_graphs: - if ignored_scope.names or ignored_scope.patterns: + if scope.names or scope.patterns: node_names = set(node.node_name for node in graph.nodes.values()) - for ignored_node_name in filter(lambda name: name in node_names, ignored_scope.names): - names.add(ignored_node_name) + for matched_node_name in filter(lambda name: name in node_names, scope.names): + names.add(matched_node_name) - for str_pattern in ignored_scope.patterns: + for str_pattern in scope.patterns: pattern = re.compile(str_pattern) pattern_matched_names = set(filter(pattern.match, node_names)) if pattern_matched_names: matches["patterns"].update(pattern_matched_names) patterns.add(str_pattern) - for node in graph.get_nodes_by_types(ignored_scope.types): + for node in graph.get_nodes_by_types(scope.types): matches["types"].add(node.node_name) types.add(node.node_type) - for i, subgraph in enumerate(ignored_scope.subgraphs): - names_from_subgraph = get_ignored_node_names_from_subgraph(graph, subgraph) + for i, subgraph in enumerate(scope.subgraphs): + names_from_subgraph = get_node_names_from_subgraph(graph, subgraph) if names_from_subgraph: matches["subgraphs"].update(names_from_subgraph) subgraphs_numbers.add(i) - matched_ignored_scope = IgnoredScope( + matched_scope = IgnoredScope( names=list(names), patterns=list(patterns), types=list(types), - subgraphs=[subgraph for i, subgraph in enumerate(ignored_scope.subgraphs) if i in subgraphs_numbers], - validate=ignored_scope.validate, + subgraphs=[subgraph for i, subgraph in enumerate(scope.subgraphs) if i in subgraphs_numbers], + validate=scope.validate, ) - return matched_ignored_scope, matches + return matched_scope, matches -def _info_matched_ignored_scope(matches: dict[str, set[str]], scope_kind: str = "Ignored") -> None: +def _info_matched_scope(matches: dict[str, set[str]], scope_kind: str = "Ignored") -> None: """ Log matches. @@ -267,22 +265,22 @@ def _info_matched_ignored_scope(matches: dict[str, set[str]], scope_kind: str = nncf_logger.info(f"{len(rules)} {scope_kind.lower()} nodes were found by {rule_type} in the NNCFGraph") -def _error_unmatched_ignored_scope(unmatched_ignored_scope: IgnoredScope, scope_kind: str = "Ignored") -> str: +def _error_unmatched_scope(unmatched_scope: IgnoredScope, scope_kind: str = "Ignored") -> str: """ - Returns an error message for unmatched ignored scope. + Returns an error message for unmatched scope. - :param unmatched_ignored_scope: Unmatched ignored scope. + :param unmatched_scope: Unmatched scope. :param scope_kind: Human-readable kind of the scope to mention in the message. :return str: Error message. """ err_msg = "\n" - for ignored_type in ("names", "types", "patterns"): - unmatched_rules = getattr(unmatched_ignored_scope, ignored_type) + for rule_type in ("names", "types", "patterns"): + unmatched_rules = getattr(unmatched_scope, rule_type) if unmatched_rules: err_msg += ( - f"{scope_kind} nodes that matches {ignored_type} {unmatched_rules} were not found in the NNCFGraph.\n" + f"{scope_kind} nodes that matches {rule_type} {unmatched_rules} were not found in the NNCFGraph.\n" ) - for subgraph in unmatched_ignored_scope.subgraphs: + for subgraph in unmatched_scope.subgraphs: err_msg += ( f"{scope_kind} nodes that matches subgraph with input names {subgraph.inputs} " f"and output names {subgraph.outputs} were not found in the NNCFGraph.\n" @@ -290,53 +288,46 @@ def _error_unmatched_ignored_scope(unmatched_ignored_scope: IgnoredScope, scope_ return err_msg -def _check_ignored_scope_strictly_matched( - ignored_scope: BaseScope, matched_ignored_scope: IgnoredScope, scope_kind: str = "Ignored" -) -> None: +def _check_scope_strictly_matched(scope: BaseScope, matched_scope: IgnoredScope, scope_kind: str = "Ignored") -> None: """ - Passes when ignored_scope and matched_ignored_scope are equal, otherwise - raises ValidationError. + Passes when scope and matched_scope are equal, otherwise - raises ValidationError. - :param ignored_scope: Ignored scope. - :param matched_ignored_scope: Matched ignored scope. + :param scope: Scope. + :param matched_scope: Matched scope. :param scope_kind: Human-readable kind of the scope to mention in the error message. """ - unmatched_ignored_scope = get_difference_ignored_scope(ignored_scope, matched_ignored_scope) - if ( - unmatched_ignored_scope.names - or unmatched_ignored_scope.types - or unmatched_ignored_scope.patterns - or unmatched_ignored_scope.subgraphs - ): - raise nncf.ValidationError(_error_unmatched_ignored_scope(unmatched_ignored_scope, scope_kind)) - - -def get_ignored_node_names_from_ignored_scope( - ignored_scope: BaseScope, nncf_graph: NNCFGraph, strict: bool = True, scope_kind: str = "Ignored" + unmatched_scope = get_difference_scope(scope, matched_scope) + if unmatched_scope.names or unmatched_scope.types or unmatched_scope.patterns or unmatched_scope.subgraphs: + raise nncf.ValidationError(_error_unmatched_scope(unmatched_scope, scope_kind)) + + +def get_node_names_from_scope( + scope: BaseScope, nncf_graph: NNCFGraph, strict: bool = True, scope_kind: str = "Ignored" ) -> set[str]: """ - Returns ignored names according to ignored scope and NNCFGraph. - If strict is True, raises nncf.ValidationError if any ignored rule was not matched. + Returns matched names according to the scope and NNCFGraph. + If strict is True, raises nncf.ValidationError if any rule was not matched. If strict is False, returns all possible matches. - :param ignored_scope: Ignored scope. + :param scope: Scope. :param nncf_graph: Graph. - :param strict: Whether all ignored_scopes must match at least one node or not. + :param strict: Whether all scope rules must match at least one node or not. :param scope_kind: Human-readable kind of the scope to mention in the error message. - :return: NNCF node names from given NNCFGraph specified in given ignored scope. + :return: NNCF node names from given NNCFGraph specified in given scope. """ - matched_ignored_scope, matches = get_matched_ignored_scope_info(ignored_scope, [nncf_graph]) + matched_scope, matches = get_matched_scope_info(scope, [nncf_graph]) if strict: - _check_ignored_scope_strictly_matched(ignored_scope, matched_ignored_scope, scope_kind) - _info_matched_ignored_scope(matches, scope_kind) + _check_scope_strictly_matched(scope, matched_scope, scope_kind) + _info_matched_scope(matches, scope_kind) return {name for match in matches.values() for name in match} -def validate_ignored_scope(ignored_scope: BaseScope, nncf_graphs: list[NNCFGraph]) -> None: +def validate_scope(scope: BaseScope, nncf_graphs: list[NNCFGraph]) -> None: """ - Passes whether all rules at 'ignored_scope' have matches at provided graphs, otherwise - raises ValidationError. + Passes whether all rules at 'scope' have matches at provided graphs, otherwise - raises ValidationError. - :param ignored_scope: Ignored scope. + :param scope: Scope. :param nncf_graphs: Graphs. """ - matched_ignored_scope, _ = get_matched_ignored_scope_info(ignored_scope, nncf_graphs) - _check_ignored_scope_strictly_matched(ignored_scope, matched_ignored_scope) + matched_scope, _ = get_matched_scope_info(scope, nncf_graphs) + _check_scope_strictly_matched(scope, matched_scope) diff --git a/src/nncf/torch/function_hook/pruning/prune_model.py b/src/nncf/torch/function_hook/pruning/prune_model.py index d04243e2390..d9bedff34eb 100644 --- a/src/nncf/torch/function_hook/pruning/prune_model.py +++ b/src/nncf/torch/function_hook/pruning/prune_model.py @@ -17,7 +17,7 @@ from nncf.common.logging import nncf_logger from nncf.parameters import PruneMode from nncf.scopes import IgnoredScope -from nncf.scopes import get_ignored_node_names_from_ignored_scope +from nncf.scopes import get_node_names_from_scope from nncf.torch.function_hook.nncf_graph.layer_attributes import PT2OpLayerAttributes from nncf.torch.function_hook.nncf_graph.nncf_graph_builder import build_nncf_graph from nncf.torch.function_hook.pruning.magnitude.algo import apply_magnitude_pruning @@ -89,7 +89,7 @@ def get_prunable_parameters(graph: PTNNCFGraph, ignored_scope: IgnoredScope | No """ ignored_names: set[str] = set() if ignored_scope is not None: - ignored_names = get_ignored_node_names_from_ignored_scope(ignored_scope, graph) + ignored_names = get_node_names_from_scope(ignored_scope, graph) nodes_with_weights = graph.get_nodes_by_metatypes(OPERATORS_WITH_WEIGHTS_METATYPES) parameters_to_sparsity: set[str] = set() diff --git a/src/nncf/version.py b/src/nncf/version.py index 8b00527f51f..d60058159c5 100644 --- a/src/nncf/version.py +++ b/src/nncf/version.py @@ -9,4 +9,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.4.0" +__version__ = "3.4.0.dev0+32e2e5dffdirty" diff --git a/tests/common/test_ignored_scope.py b/tests/common/test_ignored_scope.py index dcdb58e0c52..4117a93cc56 100644 --- a/tests/common/test_ignored_scope.py +++ b/tests/common/test_ignored_scope.py @@ -16,7 +16,7 @@ from nncf.common.graph.operator_metatypes import OutputNoopMetatype from nncf.scopes import IgnoredScope from nncf.scopes import Subgraph -from nncf.scopes import get_ignored_node_names_from_ignored_scope +from nncf.scopes import get_node_names_from_scope from tests.common.quantization.metatypes import Conv2dTestMetatype from tests.common.quantization.metatypes import LinearTestMetatype from tests.common.quantization.mock_graphs import NodeWithType @@ -74,7 +74,7 @@ def __init__(self, conv_metatype, linear_metatype): @pytest.mark.parametrize("ignored_scope,ref_ignored_names", IGNORED_SCOPES_TEST_DATA) def test_ignored_scopes(ignored_scope, ref_ignored_names): nncf_graph = NNCFGraphToTestIgnoredScope(CONV_TYPE, LINEAR_TYPE).nncf_graph - ignored_names = get_ignored_node_names_from_ignored_scope(ignored_scope, nncf_graph) + ignored_names = get_node_names_from_scope(ignored_scope, nncf_graph) assert sorted(ignored_names) == ref_ignored_names @@ -90,4 +90,4 @@ def test_ignored_scopes(ignored_scope, ref_ignored_names): def test_wrong_ignored_scopes(ignored_scope): nncf_graph = NNCFGraphToTestIgnoredScope(CONV_TYPE, LINEAR_TYPE).nncf_graph with pytest.raises(nncf.ValidationError): - get_ignored_node_names_from_ignored_scope(ignored_scope, nncf_graph) + get_node_names_from_scope(ignored_scope, nncf_graph) diff --git a/tests/common/test_scopes.py b/tests/common/test_scopes.py index 58167ac41ce..91ce47bb9dd 100644 --- a/tests/common/test_scopes.py +++ b/tests/common/test_scopes.py @@ -14,7 +14,7 @@ from nncf.common.scopes import get_not_matched_scopes from nncf.scopes import IgnoredScope from nncf.scopes import Subgraph -from nncf.scopes import get_difference_ignored_scope +from nncf.scopes import get_difference_scope @pytest.mark.parametrize( @@ -74,4 +74,4 @@ def test_get_not_matched_scopes(scope, ref): ), ) def test_ignored_scope_diff(scope_1, scope_2, ref): - assert get_difference_ignored_scope(scope_1, scope_2) == ref + assert get_difference_scope(scope_1, scope_2) == ref From 5a63d447500e361cc2b806f0af8b03be712beae8 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 13 Aug 2026 16:36:55 +0400 Subject: [PATCH 04/43] group size fallback takes precedence over custom annotation --- .../weight_compression/algorithm.py | 170 +++++++++++------- .../template_test_weights_compression.py | 39 +++- 2 files changed, 145 insertions(+), 64 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 3550476bce7..ddacf9e34c7 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -708,24 +708,28 @@ def apply_mixed_precision( def validate_group_size( self, - ratio_defining_params: list[WeightCompressionParameters], + weight_params: list[WeightCompressionParameters], ) -> None: """ Validates that the configured group size is divisible by the reduction channel size for all weights. - :param ratio_defining_params: Information about weights that are used for calculating ratio between primary and - backup precisions. + :param weight_params: Information about weights to validate. Typically, the weights that are used for + calculating ratio between primary and backup precisions. """ - # Check if group size is valid for each weight in ratio_defining_params + # Check if group size is valid for each weight in weight_params failed_nodes = [] - for w_params in ratio_defining_params: + for w_params in weight_params: if w_params.compression_config is None or w_params.compression_config.group_size == -1: continue + group_size = w_params.compression_config.group_size reduction_channel_size, _ = get_reduction_channel_size(w_params.weight_shape, w_params.reduction_axes) - if reduction_channel_size % w_params.compression_config.group_size != 0: - failed_nodes.append((w_params.node_with_weight.node_name, reduction_channel_size)) + if reduction_channel_size % group_size != 0: + failed_nodes.append((w_params.node_with_weight.node_name, reduction_channel_size, group_size)) if len(failed_nodes) > 0: - names = "\n\t".join(f'"{name}" (channel size: {channel_size})' for name, channel_size in failed_nodes) + names = "\n\t".join( + f'"{name}" (channel size: {channel_size}, group size: {group_size})' + for name, channel_size, group_size in failed_nodes + ) msg = ( f"Failed to apply group-wise quantization with group size value {self._group_size}.\n" "Ensure that the group size is divisible by the channel size, " @@ -733,29 +737,52 @@ def validate_group_size( ) raise nncf.InvalidGroupSizeError(msg) + @staticmethod + def _get_requested_group_sizes( + weight_params: list[WeightCompressionParameters], group_size: int | None = None + ) -> dict[str, int]: + """ + Returns the group size that is requested for each of the given weight parameters. + + :param weight_params: List of weight parameters. + :param group_size: Group size requested for all the given weight parameters. If None, the group size is + taken from the compression config of each weight parameter. + :return: A dictionary mapping weight names to the requested group size values. + """ + if group_size is not None: + return {w_params.weight_name: group_size for w_params in weight_params} + return {w_params.weight_name: w_params.compression_config.group_size for w_params in weight_params} + def _handle_ignore_group_size_fallback( self, all_weight_params: list[WeightCompressionParameters], - ratio_defining_params: list[WeightCompressionParameters], + target_weight_params: list[WeightCompressionParameters], skipped_weight_params: list[WeightCompressionParameters], + group_sizes: dict[str, int], ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters], list[WeightCompressionParameters]]: """ - Removes nodes that cannot be quantized with the specified group size from the lists of weight parameters. - """ - if self._group_size == -1: - return all_weight_params, ratio_defining_params, skipped_weight_params + Removes nodes that cannot be quantized with the requested group size from the lists of weight parameters. + :param all_weight_params: List of all weight parameters that can be compressed. + :param target_weight_params: List of weight parameters to check. + :param skipped_weight_params: List of weight parameters that are not compressed. + :param group_sizes: A dictionary mapping weight names to the requested group size values. + :return: The updated tuple of all, target and skipped weight compression parameters. + """ nodes_to_exclude = {} - for w_params in ratio_defining_params: + for w_params in target_weight_params: + group_size = group_sizes[w_params.weight_name] + if group_size == -1: + continue reduction_channel_size, _ = get_reduction_channel_size(w_params.weight_shape, w_params.reduction_axes) - if reduction_channel_size % self._group_size != 0: + if reduction_channel_size % group_size != 0: nodes_to_exclude[w_params.node_with_weight.node_name] = w_params.weight_shape skipped_weight_params.append(dataclasses.replace(w_params, compression_config=None)) if nodes_to_exclude: - ratio_defining_params = [ + target_weight_params = [ w_params - for w_params in ratio_defining_params + for w_params in target_weight_params if w_params.node_with_weight.node_name not in nodes_to_exclude ] all_weight_params = [ @@ -765,36 +792,36 @@ def _handle_ignore_group_size_fallback( ] nncf_logger.warning( - f"Group-wise quantization with group size {self._group_size} can't be applied to some nodes. " + "Group-wise quantization with the requested group size can't be applied to some nodes. " "They will be ignored and kept with original precision.\n" "Consider changing group size value or setting group size fallback parameter to ADJUST, which enables " "automatic adjustment to smaller group size values." ) - return all_weight_params, ratio_defining_params, skipped_weight_params + return all_weight_params, target_weight_params, skipped_weight_params def _handle_adjust_group_size_fallback( - self, weight_params: list[WeightCompressionParameters] + self, weight_params: list[WeightCompressionParameters], group_sizes: dict[str, int] ) -> tuple[list[WeightCompressionParameters], dict[str, int]]: """ - Calculates adjusted group size for weight parameters that cannot be quantized with the specified group size. + Calculates adjusted group size for weight parameters that cannot be quantized with the requested group size. + :param weight_params: List of weight parameters to process. + :param group_sizes: A dictionary mapping weight names to the requested group size values. :return: A tuple containing two elements: - - A list of weight parameters that can be quantized with the specified or adjusted group size. + - A list of weight parameters that can be quantized with the requested or adjusted group size. - A dictionary mapping weight names to their group size values. """ - if self._group_size == -1: - return weight_params, {w_params.weight_name: self._group_size for w_params in weight_params} - group_size_values = {} valid_weight_params = [] invalid_weight_params = [] adjusted_weight_params = [] for w_params in weight_params: + group_size = group_sizes[w_params.weight_name] reduction_channel_size, _ = get_reduction_channel_size(w_params.weight_shape, w_params.reduction_axes) - if reduction_channel_size % self._group_size == 0: + if group_size == -1 or reduction_channel_size % group_size == 0: valid_weight_params.append(w_params) - group_size_values[w_params.weight_name] = self._group_size + group_size_values[w_params.weight_name] = group_size continue # The maximal power of two that divides reduction_channel_size @@ -810,8 +837,7 @@ def _handle_adjust_group_size_fallback( if adjusted_weight_params: # Adjusted group size value for some nodes nncf_logger.info( - f"Some nodes can't be quantized with the specified group size of {self._group_size}. " - "Adjusted group size values will be used." + "Some nodes can't be quantized with the requested group size. Adjusted group size values will be used." ) if invalid_weight_params: @@ -1129,14 +1155,15 @@ def get_weight_compression_parameters( ratio_defining_params = self._get_ratio_defining_params(all_weight_params, is_last_layer_skipped) # Handle group size fallback modes + group_size_values = self._get_requested_group_sizes(ratio_defining_params, self._group_size) if self._group_size_fallback_mode == GroupSizeFallbackMode.IGNORE: all_weight_params, ratio_defining_params, skipped_weight_params = self._handle_ignore_group_size_fallback( - all_weight_params, ratio_defining_params, skipped_weight_params + all_weight_params, ratio_defining_params, skipped_weight_params, group_size_values ) if self._group_size_fallback_mode == GroupSizeFallbackMode.ADJUST: - ratio_defining_params, group_size_values = self._handle_adjust_group_size_fallback(ratio_defining_params) - else: - group_size_values = {w_params.weight_name: self._group_size for w_params in ratio_defining_params} + ratio_defining_params, group_size_values = self._handle_adjust_group_size_fallback( + ratio_defining_params, group_size_values + ) # Set each ratio defining parameter to primary config for weight_param in ratio_defining_params: @@ -1168,33 +1195,6 @@ def _get_custom_annotation_configs(self, graph: NNCFGraph) -> dict[str, WeightCo node_name_to_config[node_name] = annotation.config return node_name_to_config - def _validate_custom_annotation_group_size(self, weight_params: list[WeightCompressionParameters]) -> None: - """ - Validates that the group size defined by the custom annotation is divisible by the reduction channel size - for all the annotated weights. Group size fallback modes are not applied to the annotated weights since - the group size is defined by the user explicitly. - - :param weight_params: List of annotated weight parameters. - """ - failed_nodes = [] - for w_params in weight_params: - group_size = w_params.compression_config.group_size - if group_size == -1: - continue - reduction_channel_size, _ = get_reduction_channel_size(w_params.weight_shape, w_params.reduction_axes) - if reduction_channel_size % group_size != 0: - failed_nodes.append((w_params.node_with_weight.node_name, reduction_channel_size, group_size)) - if failed_nodes: - names = "\n\t".join( - f'"{name}" (channel size: {channel_size}, group size: {group_size})' - for name, channel_size, group_size in failed_nodes - ) - msg = ( - "Failed to apply group-wise quantization with the group size value defined by the custom " - "annotation. Ensure that the group size is divisible by the channel size. Failed nodes:\n\t" + names - ) - raise nncf.InvalidGroupSizeError(msg) - def apply_custom_annotation( self, graph: NNCFGraph, @@ -1274,8 +1274,6 @@ def annotate(w_params: WeightCompressionParameters) -> bool: "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(not_applied_node_names)) ) - self._validate_custom_annotation_group_size(annotated_weight_params) - if restored_weight_params: nncf_logger.warning( "The following nodes are excluded from the compression, e.g. by the ignored scope, but are matched " @@ -1299,8 +1297,53 @@ def annotate(w_params: WeightCompressionParameters) -> bool: w_params for w_params in ratio_defining_params if w_params.weight_name not in annotated_weight_names ] + # The group size fallback modes are applied to the annotated weights as well. The resulting group size is + # validated together with the rest of the weights in the apply() method. + all_weight_params, annotated_weight_params, skipped_weight_params = self._handle_group_size_fallback( + all_weight_params, annotated_weight_params, skipped_weight_params + ) + return all_weight_params, ratio_defining_params, skipped_weight_params + def _handle_group_size_fallback( + self, + all_weight_params: list[WeightCompressionParameters], + annotated_weight_params: list[WeightCompressionParameters], + skipped_weight_params: list[WeightCompressionParameters], + ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters], list[WeightCompressionParameters]]: + """ + Applies the configured group size fallback mode to the weights annotated by the custom annotation. + + The group size of an annotated weight is defined by the user, so the fallback mode is applied to that value + instead of the group size configured for the algorithm. + + :param all_weight_params: List of all weight compression parameters that can be compressed. + :param annotated_weight_params: List of annotated weight compression parameters. + :param skipped_weight_params: List of weight compression parameters that are not compressed. + :return: The updated tuple of all, annotated and skipped weight compression parameters. + """ + group_sizes = self._get_requested_group_sizes(annotated_weight_params) + if self._group_size_fallback_mode == GroupSizeFallbackMode.IGNORE: + all_weight_params, annotated_weight_params, skipped_weight_params = self._handle_ignore_group_size_fallback( + all_weight_params, annotated_weight_params, skipped_weight_params, group_sizes + ) + elif self._group_size_fallback_mode == GroupSizeFallbackMode.ADJUST: + valid_weight_params, group_size_values = self._handle_adjust_group_size_fallback( + annotated_weight_params, group_sizes + ) + valid_weight_names = set(w_params.weight_name for w_params in valid_weight_params) + for w_params in annotated_weight_params: + if w_params.weight_name in valid_weight_names: + w_params.compression_config = dataclasses.replace( + w_params.compression_config, group_size=group_size_values[w_params.weight_name] + ) + else: + # A valid adjusted group size wasn't found, the annotated config can not be applied + w_params.compression_config = self._get_backup_config(w_params.weight_dtype) + annotated_weight_params = valid_weight_params + + return all_weight_params, annotated_weight_params, skipped_weight_params + def apply( self, model: TModel, @@ -1324,7 +1367,8 @@ def apply( ) # Apply Mixed precision algorithm to ratio defining parameters self.apply_mixed_precision(ratio_defining_params, model, graph, statistic_points) - self.validate_group_size(ratio_defining_params) + # All the weights are validated, since the group size of a weight can also be defined by the custom annotation + self.validate_group_size(all_weight_params) # Print statistics nncf_logger.info( diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 28499dfac68..0c760e27b49 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -1126,7 +1126,44 @@ def test_custom_annotation_with_invalid_group_size(self, mocker): ) ], ) - assert "group size value defined by the custom annotation" in str(exc_info.value) + assert "Failed to apply group-wise quantization with group size value" in str(exc_info.value) + assert f'"{node_names[1]}" (channel size: 4, group size: 3)' in str(exc_info.value) + + @pytest.mark.parametrize( + "fallback_mode, ref_mode, ref_group_size", + [ + (nncf.GroupSizeFallbackMode.ADJUST, CompressWeightsMode.INT4_ASYM, 4), + (nncf.GroupSizeFallbackMode.IGNORE, None, None), + ], + ) + def test_custom_annotation_group_size_fallback(self, mocker, fallback_mode, ref_group_size, ref_mode): + """ + Checks that the group size fallback mode is applied to the group size defined by the custom annotation: + ADJUST replaces it with a valid value, IGNORE keeps the node in the original precision. + """ + common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) + node_names = list( + self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) + ) + + configs = self._compress_and_get_configs( + mocker, + model=self._get_sequential_matmul_model(), + **common_kwargs, + advanced_parameters=CompressionParams(group_size_fallback_mode=fallback_mode, min_adjusted_group_size=4), + custom_annotation=[ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[node_names[1]]), + # The channel size of the model is 4, so the group size of 3 is invalid + config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), + ) + ], + ) + + if ref_mode is None: + assert node_names[1] not in configs + else: + assert configs[node_names[1]] == WeightCompressionConfig(mode=ref_mode, group_size=ref_group_size) @pytest.mark.parametrize("is_3d_weights", [True, False]) @pytest.mark.parametrize("dataset", [None, np.ones([2, 8, 8], dtype=np.float32)]) From 7e75fd2cb0933809d53a99499c8b844166245938 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Mon, 31 Aug 2026 16:15:57 +0400 Subject: [PATCH 05/43] fix int8 group size --- .../weight_compression/algorithm.py | 12 ++++- src/nncf/scopes.py | 19 ++++---- tests/common/test_scopes.py | 12 ++++- .../quantization/test_weights_compression.py | 45 +++++++++++++------ 4 files changed, 63 insertions(+), 25 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index ddacf9e34c7..0742273c43d 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -141,7 +141,7 @@ def get_weight_compression_configuration( } -def check_custom_annotation(custom_annotation: list[CustomAnnotation] | None) -> None: +def validate_custom_annotation(custom_annotation: list[CustomAnnotation] | None) -> None: """ Validates the user-defined custom annotation. @@ -186,6 +186,14 @@ def check_custom_annotation(custom_annotation: list[CustomAnnotation] | None) -> ) raise nncf.ValidationError(msg) + if annotation.config.mode in INT8_MODES and annotation.config.group_size != -1: + msg = ( + "INT8 modes require per-channel quantization in 8 bit, so the group size of a custom annotation " + f"with the {annotation.config.mode.value} mode must be -1, but " + f"group_size={annotation.config.group_size} is given." + ) + raise nncf.ParameterNotSupportedError(msg) + def check_user_compression_configuration( mode: CompressWeightsMode, @@ -208,7 +216,7 @@ def check_user_compression_configuration( """ Validates the user's weight compression configuration for correctness. """ - check_custom_annotation(custom_annotation) + validate_custom_annotation(custom_annotation) if mode in INT8_MODES: if (ratio and ratio != 1) or (group_size and group_size != -1): msg = ( diff --git a/src/nncf/scopes.py b/src/nncf/scopes.py index f24ebdb5821..cda80dbb9c7 100644 --- a/src/nncf/scopes.py +++ b/src/nncf/scopes.py @@ -169,15 +169,17 @@ class CustomAnnotationScope(BaseScope): """ -def get_difference_scope(scope_1: BaseScope, scope_2: BaseScope) -> IgnoredScope: +def get_difference_scope(scope_1: BaseScope, scope_2: BaseScope) -> BaseScope: """ - Returns scope with rules from 'scope_1' not presented at 'scope_2' + Returns scope with rules from 'scope_1' not presented at 'scope_2'. + The returned scope has the same type as 'scope_1'. :param scope_1: First scope. :param scope_2: Second scope. :return: Scope with the rules difference. """ - return IgnoredScope( + cls = scope_1.__class__ + return cls( names=list(set(scope_1.names) - set(scope_2.names)), patterns=list(set(scope_1.patterns) - set(scope_2.patterns)), types=list(set(scope_1.types) - set(scope_2.types)), @@ -206,10 +208,10 @@ def convert_ignored_scope_to_list(ignored_scope: IgnoredScope | None) -> list[st return results -def get_matched_scope_info(scope: BaseScope, nncf_graphs: list[NNCFGraph]) -> tuple[IgnoredScope, dict[str, set[str]]]: +def get_matched_scope_info(scope: BaseScope, nncf_graphs: list[NNCFGraph]) -> tuple[BaseScope, dict[str, set[str]]]: """ Returns matched scope for provided graphs along with all found matches. - The resulted scope consist of all matched rules. + The resulted scope consist of all matched rules and has the same type as the given scope. The found matches consist of a dictionary with a rule name as a key and matched node names as a value. :param scope: Scope instance. @@ -243,7 +245,8 @@ def get_matched_scope_info(scope: BaseScope, nncf_graphs: list[NNCFGraph]) -> tu matches["subgraphs"].update(names_from_subgraph) subgraphs_numbers.add(i) - matched_scope = IgnoredScope( + cls = scope.__class__ + matched_scope = cls( names=list(names), patterns=list(patterns), types=list(types), @@ -265,7 +268,7 @@ def _info_matched_scope(matches: dict[str, set[str]], scope_kind: str = "Ignored nncf_logger.info(f"{len(rules)} {scope_kind.lower()} nodes were found by {rule_type} in the NNCFGraph") -def _error_unmatched_scope(unmatched_scope: IgnoredScope, scope_kind: str = "Ignored") -> str: +def _error_unmatched_scope(unmatched_scope: BaseScope, scope_kind: str = "Ignored") -> str: """ Returns an error message for unmatched scope. @@ -288,7 +291,7 @@ def _error_unmatched_scope(unmatched_scope: IgnoredScope, scope_kind: str = "Ign return err_msg -def _check_scope_strictly_matched(scope: BaseScope, matched_scope: IgnoredScope, scope_kind: str = "Ignored") -> None: +def _check_scope_strictly_matched(scope: BaseScope, matched_scope: BaseScope, scope_kind: str = "Ignored") -> None: """ Passes when scope and matched_scope are equal, otherwise - raises ValidationError. diff --git a/tests/common/test_scopes.py b/tests/common/test_scopes.py index 91ce47bb9dd..31ac6a23847 100644 --- a/tests/common/test_scopes.py +++ b/tests/common/test_scopes.py @@ -12,6 +12,7 @@ from nncf.common.graph import NNCFNode from nncf.common.scopes import get_not_matched_scopes +from nncf.scopes import CustomAnnotationScope from nncf.scopes import IgnoredScope from nncf.scopes import Subgraph from nncf.scopes import get_difference_scope @@ -71,7 +72,14 @@ def test_get_not_matched_scopes(scope, ref): subgraphs=[Subgraph(inputs=["A_input"], outputs=["A_output"])], ), ), + ( + CustomAnnotationScope(names=["A_name", "B_name"]), + CustomAnnotationScope(names=["B_name"]), + CustomAnnotationScope(names=["A_name"]), + ), ), ) -def test_ignored_scope_diff(scope_1, scope_2, ref): - assert get_difference_scope(scope_1, scope_2) == ref +def test_scope_diff(scope_1, scope_2, ref): + diff_scope = get_difference_scope(scope_1, scope_2) + assert type(diff_scope) is type(scope_1) + assert diff_scope == ref diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 2fce7d0222f..bcb76147265 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -924,22 +924,41 @@ def test_raise_error_with_unsupported_params_for_int4(mode, params): @pytest.mark.parametrize( - "custom_annotation", + "custom_annotation, error", ( - nncf.CustomAnnotation(), - ["anything"], - [nncf.CustomAnnotation(scope=nncf.IgnoredScope())], - [nncf.CustomAnnotation(config="anything")], - [ - nncf.CustomAnnotation( - config=WeightCompressionConfig(mode=CompressWeightsMode.CODEBOOK, codebook_values=None) - ) - ], + (nncf.CustomAnnotation(), nncf.ValidationError), + (["anything"], nncf.ValidationError), + ([nncf.CustomAnnotation(scope=nncf.IgnoredScope())], nncf.ValidationError), + ([nncf.CustomAnnotation(config="anything")], nncf.ValidationError), + ( + [ + nncf.CustomAnnotation( + config=WeightCompressionConfig(mode=CompressWeightsMode.CODEBOOK, codebook_values=None) + ) + ], + nncf.ValidationError, + ), + ( + [nncf.CustomAnnotation(config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=4))], + nncf.ParameterNotSupportedError, + ), + ( + [nncf.CustomAnnotation(config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_ASYM, group_size=4))], + nncf.ParameterNotSupportedError, + ), ), - ids=["not_a_list", "not_an_annotation", "wrong_scope", "wrong_config", "codebook_without_values"], + ids=[ + "not_a_list", + "not_an_annotation", + "wrong_scope", + "wrong_config", + "codebook_without_values", + "int8_sym_with_group_size", + "int8_asym_with_group_size", + ], ) -def test_raise_error_with_invalid_custom_annotation(custom_annotation): - with pytest.raises(nncf.ValidationError): +def test_raise_error_with_invalid_custom_annotation(custom_annotation, error): + with pytest.raises(error): compress_weights(ov.Model([], []), mode=CompressWeightsMode.INT4_SYM, custom_annotation=custom_annotation) From feb84179a59775ff705cc439caa7c0036a979c66 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 10 Sep 2026 23:07:11 +0400 Subject: [PATCH 06/43] fix failing precommit test --- .../template_test_weights_compression.py | 59 ++++++++----------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 2e6a10aa1c8..da18d1ce049 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -974,29 +974,34 @@ def _get_sequential_matmul_model(self): self.get_sequential_matmul_model(transpose_a=False), np.ones([1, 4, 4], dtype=np.float32) ) - def _compress_and_get_configs(self, mocker, **kwargs) -> dict[str, WeightCompressionConfig]: + def _compress_and_get_configs(self, **kwargs) -> dict[str, WeightCompressionConfig]: """ Compresses a model and returns the compression config assigned to each compressed weight node. - :param mocker: Mocker fixture. :param kwargs: Arguments for the `compress_weights` function. :return: A mapping from a node name to the assigned compression config. """ - spy = mocker.spy(WeightCompression, "apply_with_parameters") - compress_weights(**kwargs) - all_weight_params = spy.call_args.args[5] + captured_weight_params = [] + original_fn = WeightCompression.apply_with_parameters + + def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_params): + captured_weight_params.append(all_weight_params) + return original_fn(self, model, graph, dataset, statistics, all_weight_params) + + with patch.object(WeightCompression, "apply_with_parameters", apply_with_parameters): + compress_weights(**kwargs) + + all_weight_params = captured_weight_params[-1] return {wp.node_with_weight.node_name: wp.compression_config for wp in all_weight_params} - def test_custom_annotation(self, mocker): + def test_custom_annotation(self): """ Checks that the config given by the custom annotation is assigned to the matched nodes, overriding both the primary precision and the backup precision of the last layer, and that the rest of the nodes is not affected. """ common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=False) - reference_configs = self._compress_and_get_configs( - mocker, model=self._get_sequential_matmul_model(), **common_kwargs - ) + reference_configs = self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs) node_names = list(reference_configs) # The last layer is compressed to the backup precision since all_layers is False assert reference_configs[node_names[-1]].mode == CompressWeightsMode.INT8_ASYM @@ -1004,7 +1009,6 @@ def test_custom_annotation(self, mocker): annotated_config = WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1) last_layer_config = WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4) configs = self._compress_and_get_configs( - mocker, model=self._get_sequential_matmul_model(), **common_kwargs, custom_annotation=[ @@ -1031,13 +1035,10 @@ def test_custom_annotation_over_mixed_precision(self, mocker): """ annotated_config = WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=-1) common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=0.5, group_size=-1, all_layers=True) - node_names = list( - self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) - ) + node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) mixed_precision_spy = mocker.spy(WeightCompression, "apply_mixed_precision") configs = self._compress_and_get_configs( - mocker, model=self._get_sequential_matmul_model(), **common_kwargs, custom_annotation=[ @@ -1053,21 +1054,18 @@ def test_custom_annotation_over_mixed_precision(self, mocker): assert node_names[1] not in ratio_defining_names assert configs[node_names[1]] == annotated_config - def test_custom_annotation_over_ignored_scope(self, mocker): + def test_custom_annotation_over_ignored_scope(self): """ Checks that a node from the ignored scope is compressed with the config given by the custom annotation and that a warning is logged. """ annotated_config = WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1) common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) - node_names = list( - self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) - ) + node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) ignored_node_name = node_names[1] with patch.object(nncf_logger, "warning") as mock_warning: configs = self._compress_and_get_configs( - mocker, model=self._get_sequential_matmul_model(), **common_kwargs, ignored_scope=IgnoredScope(names=[ignored_node_name]), @@ -1085,20 +1083,17 @@ def test_custom_annotation_over_ignored_scope(self, mocker): warn_msg = "are excluded from the compression, e.g. by the ignored scope, but are matched" assert any(warn_msg in msg for msg in warning_messages) - def test_custom_annotation_overlap(self, mocker): + def test_custom_annotation_overlap(self): """ Checks that the last matched annotation takes precedence and that a warning is logged. """ common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) - node_names = list( - self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) - ) + node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) first_config = WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1) last_config = WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4) with patch.object(nncf_logger, "warning") as mock_warning: configs = self._compress_and_get_configs( - mocker, model=self._get_sequential_matmul_model(), **common_kwargs, custom_annotation=[ @@ -1119,7 +1114,7 @@ def test_custom_annotation_overlap(self, mocker): warn_msg = "Several custom annotations match the same nodes" assert any(warn_msg in msg for msg in warning_messages) - def test_custom_annotation_without_weights(self, mocker): + def test_custom_annotation_without_weights(self): """ Checks that a warning is logged when the custom annotation matches only the nodes that have no weight. """ @@ -1128,7 +1123,6 @@ def test_custom_annotation_without_weights(self, mocker): with patch.object(nncf_logger, "warning") as mock_warning: self._compress_and_get_configs( - mocker, model=self._get_sequential_matmul_model(), mode=CompressWeightsMode.INT4_SYM, ratio=1.0, @@ -1145,15 +1139,13 @@ def test_custom_annotation_without_weights(self, mocker): warn_msg = "are matched by the custom annotation, but have no weight to compress" assert any(warn_msg in msg for msg in warning_messages) - def test_custom_annotation_with_invalid_group_size(self, mocker): + def test_custom_annotation_with_invalid_group_size(self): """ Checks that an error is raised when the group size given by the custom annotation is not divisible by the channel size. """ common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) - node_names = list( - self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) - ) + node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) with pytest.raises(InvalidGroupSizeError) as exc_info: compress_weights( @@ -1176,18 +1168,15 @@ def test_custom_annotation_with_invalid_group_size(self, mocker): (nncf.GroupSizeFallbackMode.IGNORE, None, None), ], ) - def test_custom_annotation_group_size_fallback(self, mocker, fallback_mode, ref_group_size, ref_mode): + def test_custom_annotation_group_size_fallback(self, fallback_mode, ref_group_size, ref_mode): """ Checks that the group size fallback mode is applied to the group size defined by the custom annotation: ADJUST replaces it with a valid value, IGNORE keeps the node in the original precision. """ common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) - node_names = list( - self._compress_and_get_configs(mocker, model=self._get_sequential_matmul_model(), **common_kwargs) - ) + node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) configs = self._compress_and_get_configs( - mocker, model=self._get_sequential_matmul_model(), **common_kwargs, advanced_parameters=CompressionParams(group_size_fallback_mode=fallback_mode, min_adjusted_group_size=4), From bb9ffbd665bad49d3c00c477fc37c18af0624a9c Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 15 Sep 2026 13:43:25 +0400 Subject: [PATCH 07/43] update tests --- .../template_test_weights_compression.py | 168 ++++++++++++------ .../quantization/test_weights_compression.py | 75 -------- 2 files changed, 111 insertions(+), 132 deletions(-) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index da18d1ce049..8366f35cf64 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -994,6 +994,25 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa all_weight_params = captured_weight_params[-1] return {wp.node_with_weight.node_name: wp.compression_config for wp in all_weight_params} + def _compress_and_get_dumped_parameters(self, **kwargs) -> dict[str, Any]: + """ + Compresses a model and returns the parameters dumped into the meta information of the compressed model. + + :param kwargs: Arguments for the `compress_weights` function. + :return: The dumped parameters. + """ + dumped_parameters = {} + original_fn = WeightCompression.apply_with_parameters + + def apply_with_parameters(self, *args): + self._backend_entity.dump_parameters = lambda model, parameters, **_: dumped_parameters.update(parameters) + return original_fn(self, *args) + + with patch.object(WeightCompression, "apply_with_parameters", apply_with_parameters): + compress_weights(**kwargs) + + return dumped_parameters + def test_custom_annotation(self): """ Checks that the config given by the custom annotation is assigned to the matched nodes, overriding both @@ -1114,72 +1133,32 @@ def test_custom_annotation_overlap(self): warn_msg = "Several custom annotations match the same nodes" assert any(warn_msg in msg for msg in warning_messages) - def test_custom_annotation_without_weights(self): - """ - Checks that a warning is logged when the custom annotation matches only the nodes that have no weight. - """ - model = self._get_sequential_matmul_model() - input_node_name = nncf.build_graph(model).get_input_nodes()[0].node_name - - with patch.object(nncf_logger, "warning") as mock_warning: - self._compress_and_get_configs( - model=self._get_sequential_matmul_model(), - mode=CompressWeightsMode.INT4_SYM, - ratio=1.0, - group_size=-1, - custom_annotation=[ - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[input_node_name]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), - ) - ], - ) - - warning_messages = [args[0] for args, _ in mock_warning.call_args_list] - warn_msg = "are matched by the custom annotation, but have no weight to compress" - assert any(warn_msg in msg for msg in warning_messages) - - def test_custom_annotation_with_invalid_group_size(self): - """ - Checks that an error is raised when the group size given by the custom annotation is not divisible by the - channel size. - """ - common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) - node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) - - with pytest.raises(InvalidGroupSizeError) as exc_info: - compress_weights( - model=self._get_sequential_matmul_model(), - **common_kwargs, - custom_annotation=[ - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[node_names[1]]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_SYM, group_size=3), - ) - ], - ) - assert "Failed to apply group-wise quantization with group size value" in str(exc_info.value) - assert f'"{node_names[1]}" (channel size: 4, group size: 3)' in str(exc_info.value) - @pytest.mark.parametrize( - "fallback_mode, ref_mode, ref_group_size", + "fallback_mode, expected_outcome, ref_config", [ - (nncf.GroupSizeFallbackMode.ADJUST, CompressWeightsMode.INT4_ASYM, 4), - (nncf.GroupSizeFallbackMode.IGNORE, None, None), + (None, "exception", None), + (nncf.GroupSizeFallbackMode.ERROR, "exception", None), + (nncf.GroupSizeFallbackMode.IGNORE, "ignored", None), + ( + nncf.GroupSizeFallbackMode.ADJUST, + "adjusted", + WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4), + ), ], + ids=["default", "error", "ignore", "adjust"], ) - def test_custom_annotation_group_size_fallback(self, fallback_mode, ref_group_size, ref_mode): + def test_custom_annotation_with_invalid_group_size(self, fallback_mode, expected_outcome, ref_config): """ - Checks that the group size fallback mode is applied to the group size defined by the custom annotation: - ADJUST replaces it with a valid value, IGNORE keeps the node in the original precision. + Checks how a group size given by the custom annotation which is not divisible by the channel size is + handled depending on the group size fallback mode: an error is raised by default and with the ERROR mode, + IGNORE keeps the node in the original precision, ADJUST replaces the group size with a valid value. """ common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) - configs = self._compress_and_get_configs( + kwargs = dict( model=self._get_sequential_matmul_model(), **common_kwargs, - advanced_parameters=CompressionParams(group_size_fallback_mode=fallback_mode, min_adjusted_group_size=4), custom_annotation=[ nncf.CustomAnnotation( scope=nncf.CustomAnnotationScope(names=[node_names[1]]), @@ -1188,11 +1167,86 @@ def test_custom_annotation_group_size_fallback(self, fallback_mode, ref_group_si ) ], ) + if fallback_mode is not None: + kwargs["advanced_parameters"] = CompressionParams( + group_size_fallback_mode=fallback_mode, min_adjusted_group_size=4 + ) + + if expected_outcome == "exception": + with pytest.raises(InvalidGroupSizeError) as exc_info: + compress_weights(**kwargs) + assert "Failed to apply group-wise quantization with group size value" in str(exc_info.value) + assert f'"{node_names[1]}" (channel size: 4, group size: 3)' in str(exc_info.value) + return - if ref_mode is None: + configs = self._compress_and_get_configs(**kwargs) + if expected_outcome == "ignored": assert node_names[1] not in configs else: - assert configs[node_names[1]] == WeightCompressionConfig(mode=ref_mode, group_size=ref_group_size) + assert configs[node_names[1]] == ref_config + + @pytest.mark.parametrize( + "custom_annotation, error", + [ + (nncf.CustomAnnotation(), nncf.ValidationError), + (["anything"], nncf.ValidationError), + ([nncf.CustomAnnotation(scope=nncf.IgnoredScope())], nncf.ValidationError), + ([nncf.CustomAnnotation(config="anything")], nncf.ValidationError), + ( + [ + nncf.CustomAnnotation( + config=WeightCompressionConfig(mode=CompressWeightsMode.CODEBOOK, codebook_values=None) + ) + ], + nncf.ValidationError, + ), + ( + [ + nncf.CustomAnnotation( + config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=4) + ) + ], + nncf.ParameterNotSupportedError, + ), + ], + ids=[ + "not_a_list", + "not_an_annotation", + "wrong_scope", + "wrong_config", + "codebook_without_values", + "int8_with_group_size", + ], + ) + def test_invalid_custom_annotation(self, custom_annotation, error): + """ + Checks that an error is raised for an invalid custom annotation. + """ + with pytest.raises(error): + compress_weights( + model=self._get_sequential_matmul_model(), + mode=CompressWeightsMode.INT4_SYM, + custom_annotation=custom_annotation, + ) + + def test_dump_custom_annotation(self): + """ + Checks that the custom annotation is dumped into the meta information of the compressed model. + """ + common_kwargs = dict(mode=CompressWeightsMode.INT8_ASYM) + node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) + annotation = nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[node_names[1]]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + + for custom_annotation, ref_dump in [(None, []), ([annotation], [annotation])]: + dumped_parameters = self._compress_and_get_dumped_parameters( + model=self._get_sequential_matmul_model(), + **common_kwargs, + custom_annotation=custom_annotation, + ) + assert dumped_parameters["custom_annotation"] == ref_dump @pytest.mark.parametrize("is_3d_weights", [True, False]) @pytest.mark.parametrize("dataset", [None, np.ones([2, 8, 8], dtype=np.float32)]) diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 44624af5aba..9382eadac0b 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -925,81 +925,6 @@ def test_raise_error_with_unsupported_params_for_int4(mode, params): compress_weights(ov.Model([], []), mode=mode, **params) -@pytest.mark.parametrize( - "custom_annotation, error", - ( - (nncf.CustomAnnotation(), nncf.ValidationError), - (["anything"], nncf.ValidationError), - ([nncf.CustomAnnotation(scope=nncf.IgnoredScope())], nncf.ValidationError), - ([nncf.CustomAnnotation(config="anything")], nncf.ValidationError), - ( - [ - nncf.CustomAnnotation( - config=WeightCompressionConfig(mode=CompressWeightsMode.CODEBOOK, codebook_values=None) - ) - ], - nncf.ValidationError, - ), - ( - [nncf.CustomAnnotation(config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=4))], - nncf.ParameterNotSupportedError, - ), - ( - [nncf.CustomAnnotation(config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_ASYM, group_size=4))], - nncf.ParameterNotSupportedError, - ), - ), - ids=[ - "not_a_list", - "not_an_annotation", - "wrong_scope", - "wrong_config", - "codebook_without_values", - "int8_sym_with_group_size", - "int8_asym_with_group_size", - ], -) -def test_raise_error_with_invalid_custom_annotation(custom_annotation, error): - with pytest.raises(error): - compress_weights(ov.Model([], []), mode=CompressWeightsMode.INT4_SYM, custom_annotation=custom_annotation) - - -@pytest.mark.parametrize( - "custom_annotation, expected_dump", - [ - (None, {"": "[]"}), - ( - [ - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(patterns=["MatMul.*"], types=["MatMul"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), - ), - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=["MatMul_1"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), - ), - ], - { - "0/scope/patterns": "['MatMul.*']", - "0/scope/types": "['MatMul']", - "0/config": "{'mode': 'int8_sym', 'group_size': -1}", - "1/scope/names": "['MatMul_1']", - "1/config": "{'mode': 'int4_asym', 'group_size': 3}", - }, - ), - ], - ids=["without_annotation", "with_annotation"], -) -def test_dump_custom_annotation(custom_annotation, expected_dump): - model = IntegerModel().ov_model - compressed_model = compress_weights(model, mode=CompressWeightsMode.INT8_ASYM, custom_annotation=custom_annotation) - - rt_path = ["nncf", "weight_compression", "custom_annotation"] - for key, value in expected_dump.items(): - path = rt_path + key.split("/") if key else rt_path - assert compressed_model.get_rt_info(path).astype(str) == value - - @pytest.mark.parametrize("mode", [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3, CompressWeightsMode.NVFP4]) class TestUnsupportedParams: @pytest.mark.parametrize( From 1dd2c0633f3ad3ba1b583bb571ab47f81170762d Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 15 Sep 2026 14:25:43 +0400 Subject: [PATCH 08/43] upd --- .../template_test_weights_compression.py | 322 +++++++----------- .../custom_annotation_ref.json | 130 +++++++ .../quantization/test_weights_compression.py | 4 + .../custom_annotation_ref.json | 130 +++++++ .../quantization/test_weights_compression.py | 4 + .../custom_annotation_ref.json | 130 +++++++ .../custom_annotation_ref.json | 130 +++++++ .../quantization/test_weights_compression.py | 4 + tests/torch/fx/test_weights_compression.py | 4 + 9 files changed, 653 insertions(+), 205 deletions(-) create mode 100644 tests/onnx/data/reference_scales/custom_annotation_ref.json create mode 100644 tests/openvino/native/data/2026.2/reference_scales/custom_annotation_ref.json create mode 100644 tests/torch/data/function_hook/quantization/reference_scales/custom_annotation_ref.json create mode 100644 tests/torch/data/fx/reference_scales/custom_annotation_ref.json diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 8366f35cf64..f8cd04a87b5 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -968,6 +968,13 @@ def test_group_size_fallback_modes( # Test Custom Annotation + @staticmethod + @abstractmethod + def get_custom_annotation_ref_path() -> Path: + """ + Returns the path to the reference compression configs of the custom annotation test. + """ + def _get_sequential_matmul_model(self): """Returns a backend model with a sequence of MatMul layers, ready to be compressed.""" return self.wrap_model( @@ -994,228 +1001,149 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa all_weight_params = captured_weight_params[-1] return {wp.node_with_weight.node_name: wp.compression_config for wp in all_weight_params} - def _compress_and_get_dumped_parameters(self, **kwargs) -> dict[str, Any]: - """ - Compresses a model and returns the parameters dumped into the meta information of the compressed model. - - :param kwargs: Arguments for the `compress_weights` function. - :return: The dumped parameters. - """ - dumped_parameters = {} - original_fn = WeightCompression.apply_with_parameters - - def apply_with_parameters(self, *args): - self._backend_entity.dump_parameters = lambda model, parameters, **_: dumped_parameters.update(parameters) - return original_fn(self, *args) - - with patch.object(WeightCompression, "apply_with_parameters", apply_with_parameters): - compress_weights(**kwargs) - - return dumped_parameters - - def test_custom_annotation(self): - """ - Checks that the config given by the custom annotation is assigned to the matched nodes, overriding both - the primary precision and the backup precision of the last layer, and that the rest of the nodes is not - affected. - """ - common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=False) - reference_configs = self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs) - node_names = list(reference_configs) - # The last layer is compressed to the backup precision since all_layers is False - assert reference_configs[node_names[-1]].mode == CompressWeightsMode.INT8_ASYM - - annotated_config = WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1) - last_layer_config = WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4) - configs = self._compress_and_get_configs( - model=self._get_sequential_matmul_model(), - **common_kwargs, - custom_annotation=[ - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[node_names[1]]), - config=annotated_config, - ), - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[node_names[-1]]), - config=last_layer_config, - ), - ], - ) - - assert configs[node_names[1]] == annotated_config - assert configs[node_names[-1]] == last_layer_config - for node_name in node_names[2:-1]: - assert configs[node_name] == reference_configs[node_name] - - def test_custom_annotation_over_mixed_precision(self, mocker): - """ - Checks that an annotated node is excluded from the ratio-defining parameters, so that the mixed precision - algorithm can not override the config given by the custom annotation. - """ - annotated_config = WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=-1) - common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=0.5, group_size=-1, all_layers=True) - node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) - - mixed_precision_spy = mocker.spy(WeightCompression, "apply_mixed_precision") - configs = self._compress_and_get_configs( - model=self._get_sequential_matmul_model(), - **common_kwargs, - custom_annotation=[ - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[node_names[1]]), - config=annotated_config, - ) - ], - ) - - ratio_defining_params = mixed_precision_spy.call_args.args[1] - ratio_defining_names = [wp.node_with_weight.node_name for wp in ratio_defining_params] - assert node_names[1] not in ratio_defining_names - assert configs[node_names[1]] == annotated_config - - def test_custom_annotation_over_ignored_scope(self): - """ - Checks that a node from the ignored scope is compressed with the config given by the custom annotation - and that a warning is logged. - """ - annotated_config = WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1) - common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) - node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) - ignored_node_name = node_names[1] - - with patch.object(nncf_logger, "warning") as mock_warning: - configs = self._compress_and_get_configs( - model=self._get_sequential_matmul_model(), - **common_kwargs, - ignored_scope=IgnoredScope(names=[ignored_node_name]), - custom_annotation=[ - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[ignored_node_name]), - config=annotated_config, - ) + @pytest.mark.parametrize( + ("kwargs", "annotations", "ignored_indices"), + [ + pytest.param( + dict(ratio=1.0, all_layers=False), + [ + ([1], WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1)), + ([-1], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4)), ], - ) - - assert configs[ignored_node_name] == annotated_config - assert list(configs) == node_names, "The topological order of the weight parameters is not kept." - warning_messages = [args[0] for args, _ in mock_warning.call_args_list] - warn_msg = "are excluded from the compression, e.g. by the ignored scope, but are matched" - assert any(warn_msg in msg for msg in warning_messages) - - def test_custom_annotation_overlap(self): - """ - Checks that the last matched annotation takes precedence and that a warning is logged. - """ - common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) - node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) - first_config = WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1) - last_config = WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4) - - with patch.object(nncf_logger, "warning") as mock_warning: - configs = self._compress_and_get_configs( - model=self._get_sequential_matmul_model(), - **common_kwargs, - custom_annotation=[ - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[node_names[1], node_names[2]]), - config=first_config, + None, + id="primary_and_backup_precision", + ), + pytest.param( + dict(ratio=0.5, all_layers=True), + [([1], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=-1))], + None, + id="mixed_precision", + ), + pytest.param( + dict(ratio=1.0, all_layers=True), + [([1], WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1))], + [1], + id="ignored_scope", + ), + pytest.param( + dict(ratio=1.0, all_layers=True), + [ + ([1, 2], WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1)), + ([2], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4)), + ], + None, + id="overlapping_annotations", + ), + pytest.param( + dict( + ratio=1.0, + all_layers=True, + advanced_parameters=CompressionParams( + group_size_fallback_mode=nncf.GroupSizeFallbackMode.ADJUST, min_adjusted_group_size=4 ), - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[node_names[2]]), - config=last_config, + ), + # The channel size of the model is 4, so the group size of 3 is invalid + [([1], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3))], + None, + id="group_size_fallback_adjust", + ), + pytest.param( + dict( + ratio=1.0, + all_layers=True, + advanced_parameters=CompressionParams( + group_size_fallback_mode=nncf.GroupSizeFallbackMode.IGNORE, min_adjusted_group_size=4 ), - ], - ) - - assert configs[node_names[1]] == first_config - assert configs[node_names[2]] == last_config - warning_messages = [args[0] for args, _ in mock_warning.call_args_list] - warn_msg = "Several custom annotations match the same nodes" - assert any(warn_msg in msg for msg in warning_messages) - - @pytest.mark.parametrize( - "fallback_mode, expected_outcome, ref_config", - [ - (None, "exception", None), - (nncf.GroupSizeFallbackMode.ERROR, "exception", None), - (nncf.GroupSizeFallbackMode.IGNORE, "ignored", None), - ( - nncf.GroupSizeFallbackMode.ADJUST, - "adjusted", - WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4), + ), + [([1], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3))], + None, + id="group_size_fallback_ignore", ), ], - ids=["default", "error", "ignore", "adjust"], ) - def test_custom_annotation_with_invalid_group_size(self, fallback_mode, expected_outcome, ref_config): + def test_custom_annotation(self, kwargs, annotations, ignored_indices, request): """ - Checks how a group size given by the custom annotation which is not divisible by the channel size is - handled depending on the group size fallback mode: an error is raised by default and with the ERROR mode, - IGNORE keeps the node in the original precision, ADJUST replaces the group size with a valid value. + Compares the compression config assigned to each weight node with the reference one when the given custom + annotation is applied. The annotation scopes and the ignored scope are given by the indices of the weight + nodes in the topological order. A node that is not compressed has no config in the reference file. + + Set the NNCF_TEST_REGEN_DOT environment variable to regenerate the reference file. """ - common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, all_layers=True) + common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, group_size=-1, **kwargs) node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) - kwargs = dict( + if ignored_indices is not None: + common_kwargs["ignored_scope"] = IgnoredScope(names=[node_names[i] for i in ignored_indices]) + configs = self._compress_and_get_configs( model=self._get_sequential_matmul_model(), **common_kwargs, custom_annotation=[ nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[node_names[1]]), - # The channel size of the model is 4, so the group size of 3 is invalid - config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), + scope=nncf.CustomAnnotationScope(names=[node_names[i] for i in indices]), config=config ) + for indices, config in annotations ], ) - if fallback_mode is not None: - kwargs["advanced_parameters"] = CompressionParams( - group_size_fallback_mode=fallback_mode, min_adjusted_group_size=4 - ) - if expected_outcome == "exception": - with pytest.raises(InvalidGroupSizeError) as exc_info: - compress_weights(**kwargs) - assert "Failed to apply group-wise quantization with group size value" in str(exc_info.value) - assert f'"{node_names[1]}" (channel size: 4, group size: 3)' in str(exc_info.value) - return + actual_configs = { + node_name: {"mode": config.mode, "group_size": config.group_size} for node_name, config in configs.items() + } - configs = self._compress_and_get_configs(**kwargs) - if expected_outcome == "ignored": - assert node_names[1] not in configs - else: - assert configs[node_names[1]] == ref_config + ref_key = request.node.callspec.id + ref_path = self.get_custom_annotation_ref_path() + if os.getenv("NNCF_TEST_REGEN_DOT") is not None: + ref_configs = load_json(ref_path) + ref_configs[ref_key] = actual_configs + dump_to_json(ref_path, ref_configs) + + assert actual_configs == load_json(ref_path)[ref_key] @pytest.mark.parametrize( - "custom_annotation, error", + ("custom_annotation", "error"), [ - (nncf.CustomAnnotation(), nncf.ValidationError), - (["anything"], nncf.ValidationError), - ([nncf.CustomAnnotation(scope=nncf.IgnoredScope())], nncf.ValidationError), - ([nncf.CustomAnnotation(config="anything")], nncf.ValidationError), - ( + pytest.param(nncf.CustomAnnotation(), nncf.ValidationError, id="not_a_list"), + pytest.param(["anything"], nncf.ValidationError, id="not_an_annotation"), + pytest.param([nncf.CustomAnnotation(scope=nncf.IgnoredScope())], nncf.ValidationError, id="wrong_scope"), + pytest.param([nncf.CustomAnnotation(config="anything")], nncf.ValidationError, id="wrong_config"), + pytest.param( [ nncf.CustomAnnotation( config=WeightCompressionConfig(mode=CompressWeightsMode.CODEBOOK, codebook_values=None) ) ], nncf.ValidationError, + id="codebook_without_values", ), - ( + pytest.param( [ nncf.CustomAnnotation( config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=4) ) ], nncf.ParameterNotSupportedError, + id="int8_with_group_size", + ), + pytest.param( + [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=["unknown_node"]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + ], + nncf.ValidationError, + id="unmatched_scope", + ), + pytest.param( + [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*"]), + # The channel size of the model is 4, so the group size of 3 is invalid + config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), + ) + ], + InvalidGroupSizeError, + id="invalid_group_size", ), - ], - ids=[ - "not_a_list", - "not_an_annotation", - "wrong_scope", - "wrong_config", - "codebook_without_values", - "int8_with_group_size", ], ) def test_invalid_custom_annotation(self, custom_annotation, error): @@ -1226,28 +1154,12 @@ def test_invalid_custom_annotation(self, custom_annotation, error): compress_weights( model=self._get_sequential_matmul_model(), mode=CompressWeightsMode.INT4_SYM, + ratio=1.0, + group_size=-1, + all_layers=True, custom_annotation=custom_annotation, ) - def test_dump_custom_annotation(self): - """ - Checks that the custom annotation is dumped into the meta information of the compressed model. - """ - common_kwargs = dict(mode=CompressWeightsMode.INT8_ASYM) - node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) - annotation = nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[node_names[1]]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), - ) - - for custom_annotation, ref_dump in [(None, []), ([annotation], [annotation])]: - dumped_parameters = self._compress_and_get_dumped_parameters( - model=self._get_sequential_matmul_model(), - **common_kwargs, - custom_annotation=custom_annotation, - ) - assert dumped_parameters["custom_annotation"] == ref_dump - @pytest.mark.parametrize("is_3d_weights", [True, False]) @pytest.mark.parametrize("dataset", [None, np.ones([2, 8, 8], dtype=np.float32)]) @pytest.mark.parametrize("prefer_data_aware_scaling", [True, False]) diff --git a/tests/onnx/data/reference_scales/custom_annotation_ref.json b/tests/onnx/data/reference_scales/custom_annotation_ref.json new file mode 100644 index 00000000000..dc8ca539323 --- /dev/null +++ b/tests/onnx/data/reference_scales/custom_annotation_ref.json @@ -0,0 +1,130 @@ +{ + "primary_and_backup_precision": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_asym", + "group_size": 4 + } + }, + "mixed_precision": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int4_asym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int8_asym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int8_asym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int8_asym", + "group_size": -1 + } + }, + "ignored_scope": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "overlapping_annotations": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_asym", + "group_size": 4 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "group_size_fallback_adjust": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int4_asym", + "group_size": 4 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "group_size_fallback_ignore": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } + } +} \ No newline at end of file diff --git a/tests/onnx/quantization/test_weights_compression.py b/tests/onnx/quantization/test_weights_compression.py index a412e97ae58..bd1e5c70acc 100644 --- a/tests/onnx/quantization/test_weights_compression.py +++ b/tests/onnx/quantization/test_weights_compression.py @@ -726,6 +726,10 @@ def get_ignored_scope_name(is_3d_weights) -> str: def get_awq_scale_ref_path() -> Path: return REFERENCE_SCALES_DIR / "awq_scale_ref.json" + @staticmethod + def get_custom_annotation_ref_path() -> Path: + return REFERENCE_SCALES_DIR / "custom_annotation_ref.json" + @staticmethod def get_transform_func() -> Callable[..., Any] | None: def transform_func(x): diff --git a/tests/openvino/native/data/2026.2/reference_scales/custom_annotation_ref.json b/tests/openvino/native/data/2026.2/reference_scales/custom_annotation_ref.json new file mode 100644 index 00000000000..dc8ca539323 --- /dev/null +++ b/tests/openvino/native/data/2026.2/reference_scales/custom_annotation_ref.json @@ -0,0 +1,130 @@ +{ + "primary_and_backup_precision": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_asym", + "group_size": 4 + } + }, + "mixed_precision": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int4_asym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int8_asym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int8_asym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int8_asym", + "group_size": -1 + } + }, + "ignored_scope": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "overlapping_annotations": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_asym", + "group_size": 4 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "group_size_fallback_adjust": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int4_asym", + "group_size": 4 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "group_size_fallback_ignore": { + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } + } +} \ No newline at end of file diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 9382eadac0b..4f6394b579e 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -2545,6 +2545,10 @@ def get_num_multiply_from_awq(model): def get_awq_scale_ref_path() -> Path: return get_actual_reference_for_current_openvino(REFERENCE_SCALES_DIR / "awq_scale_ref.json") + @staticmethod + def get_custom_annotation_ref_path() -> Path: + return get_actual_reference_for_current_openvino(REFERENCE_SCALES_DIR / "custom_annotation_ref.json") + @pytest.fixture def transpose_a_supported(self) -> bool: return True diff --git a/tests/torch/data/function_hook/quantization/reference_scales/custom_annotation_ref.json b/tests/torch/data/function_hook/quantization/reference_scales/custom_annotation_ref.json new file mode 100644 index 00000000000..302e18b4d36 --- /dev/null +++ b/tests/torch/data/function_hook/quantization/reference_scales/custom_annotation_ref.json @@ -0,0 +1,130 @@ +{ + "primary_and_backup_precision": { + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int8_sym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_asym", + "group_size": 4 + } + }, + "mixed_precision": { + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int4_asym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int8_asym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int8_asym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int8_asym", + "group_size": -1 + } + }, + "ignored_scope": { + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int8_sym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "overlapping_annotations": { + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int8_sym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int4_asym", + "group_size": 4 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "group_size_fallback_adjust": { + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int4_asym", + "group_size": 4 + }, + "/linear/2": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "group_size_fallback_ignore": { + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_sym", + "group_size": -1 + } + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/reference_scales/custom_annotation_ref.json b/tests/torch/data/fx/reference_scales/custom_annotation_ref.json new file mode 100644 index 00000000000..ab5e548fb23 --- /dev/null +++ b/tests/torch/data/fx/reference_scales/custom_annotation_ref.json @@ -0,0 +1,130 @@ +{ + "primary_and_backup_precision": { + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "linear_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_asym", + "group_size": 4 + } + }, + "mixed_precision": { + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int4_asym", + "group_size": -1 + }, + "linear_2": { + "mode": "int8_asym", + "group_size": -1 + }, + "linear_3": { + "mode": "int8_asym", + "group_size": -1 + }, + "linear_4": { + "mode": "int8_asym", + "group_size": -1 + } + }, + "ignored_scope": { + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "linear_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "overlapping_annotations": { + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "linear_2": { + "mode": "int4_asym", + "group_size": 4 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "group_size_fallback_adjust": { + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int4_asym", + "group_size": 4 + }, + "linear_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_sym", + "group_size": -1 + } + }, + "group_size_fallback_ignore": { + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_sym", + "group_size": -1 + } + } +} \ No newline at end of file diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index c845009b75b..b605d10aab9 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -723,6 +723,10 @@ def get_num_multiply_from_awq(model): def get_awq_scale_ref_path() -> Path: return REFERENCE_SCALES_DIR / "awq_scale_ref.json" + @staticmethod + def get_custom_annotation_ref_path() -> Path: + return REFERENCE_SCALES_DIR / "custom_annotation_ref.json" + @staticmethod def get_transposable_awq_model(transpose_a: bool, transpose_b: bool, input_shape=None, is_3d_weights: bool = False): pytest.skip("Transposable models are not supported") diff --git a/tests/torch/fx/test_weights_compression.py b/tests/torch/fx/test_weights_compression.py index 0b29bb144eb..5f50de8ae29 100644 --- a/tests/torch/fx/test_weights_compression.py +++ b/tests/torch/fx/test_weights_compression.py @@ -513,6 +513,10 @@ def get_num_multiply_from_awq(model): def get_awq_scale_ref_path() -> Path: return REFERENCE_SCALES_DIR / "awq_scale_ref.json" + @staticmethod + def get_custom_annotation_ref_path() -> Path: + return REFERENCE_SCALES_DIR / "custom_annotation_ref.json" + @staticmethod def get_transposable_awq_model(transpose_a: bool, transpose_b: bool, input_shape=None, is_3d_weights: bool = False): pytest.skip("Transposable models are not supported") From dc94fe3723168167fca2de1d86ed82b0c53d4367 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 15 Sep 2026 14:32:02 +0400 Subject: [PATCH 09/43] update refs --- .../template_test_weights_compression.py | 13 +- .../custom_annotation_ref.json | 130 ------------------ .../group_size_fallback_adjust.json | 22 +++ .../group_size_fallback_ignore.json | 18 +++ .../data/reference_scales/ignored_scope.json | 22 +++ .../reference_scales/mixed_precision.json | 22 +++ .../overlapping_annotations.json | 22 +++ .../primary_and_backup_precision.json | 22 +++ .../quantization/test_weights_compression.py | 4 +- .../custom_annotation_ref.json | 130 ------------------ .../group_size_fallback_adjust.json | 22 +++ .../group_size_fallback_ignore.json | 18 +++ .../reference_scales/ignored_scope.json | 22 +++ .../reference_scales/mixed_precision.json | 22 +++ .../overlapping_annotations.json | 22 +++ .../primary_and_backup_precision.json | 22 +++ .../quantization/test_weights_compression.py | 4 +- .../custom_annotation_ref.json | 130 ------------------ .../group_size_fallback_adjust.json | 22 +++ .../group_size_fallback_ignore.json | 18 +++ .../reference_scales/ignored_scope.json | 22 +++ .../reference_scales/mixed_precision.json | 22 +++ .../overlapping_annotations.json | 22 +++ .../primary_and_backup_precision.json | 22 +++ .../custom_annotation_ref.json | 130 ------------------ .../group_size_fallback_adjust.json | 22 +++ .../group_size_fallback_ignore.json | 18 +++ .../fx/reference_scales/ignored_scope.json | 22 +++ .../fx/reference_scales/mixed_precision.json | 22 +++ .../overlapping_annotations.json | 22 +++ .../primary_and_backup_precision.json | 22 +++ .../quantization/test_weights_compression.py | 4 +- tests/torch/fx/test_weights_compression.py | 4 +- 33 files changed, 525 insertions(+), 536 deletions(-) delete mode 100644 tests/onnx/data/reference_scales/custom_annotation_ref.json create mode 100644 tests/onnx/data/reference_scales/group_size_fallback_adjust.json create mode 100644 tests/onnx/data/reference_scales/group_size_fallback_ignore.json create mode 100644 tests/onnx/data/reference_scales/ignored_scope.json create mode 100644 tests/onnx/data/reference_scales/mixed_precision.json create mode 100644 tests/onnx/data/reference_scales/overlapping_annotations.json create mode 100644 tests/onnx/data/reference_scales/primary_and_backup_precision.json delete mode 100644 tests/openvino/native/data/2026.2/reference_scales/custom_annotation_ref.json create mode 100644 tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json create mode 100644 tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json create mode 100644 tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json create mode 100644 tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json create mode 100644 tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json create mode 100644 tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json delete mode 100644 tests/torch/data/function_hook/quantization/reference_scales/custom_annotation_ref.json create mode 100644 tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json create mode 100644 tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json create mode 100644 tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json create mode 100644 tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json create mode 100644 tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json create mode 100644 tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json delete mode 100644 tests/torch/data/fx/reference_scales/custom_annotation_ref.json create mode 100644 tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json create mode 100644 tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json create mode 100644 tests/torch/data/fx/reference_scales/ignored_scope.json create mode 100644 tests/torch/data/fx/reference_scales/mixed_precision.json create mode 100644 tests/torch/data/fx/reference_scales/overlapping_annotations.json create mode 100644 tests/torch/data/fx/reference_scales/primary_and_backup_precision.json diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index f8cd04a87b5..02a323eff49 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -970,9 +970,9 @@ def test_group_size_fallback_modes( @staticmethod @abstractmethod - def get_custom_annotation_ref_path() -> Path: + def get_custom_annotation_ref_path(ref_name: str) -> Path: """ - Returns the path to the reference compression configs of the custom annotation test. + Returns the path to the reference compression configs of the given custom annotation test case. """ def _get_sequential_matmul_model(self): @@ -1089,14 +1089,11 @@ def test_custom_annotation(self, kwargs, annotations, ignored_indices, request): node_name: {"mode": config.mode, "group_size": config.group_size} for node_name, config in configs.items() } - ref_key = request.node.callspec.id - ref_path = self.get_custom_annotation_ref_path() + ref_path = self.get_custom_annotation_ref_path(request.node.callspec.id) if os.getenv("NNCF_TEST_REGEN_DOT") is not None: - ref_configs = load_json(ref_path) - ref_configs[ref_key] = actual_configs - dump_to_json(ref_path, ref_configs) + dump_to_json(ref_path, actual_configs) - assert actual_configs == load_json(ref_path)[ref_key] + assert actual_configs == load_json(ref_path) @pytest.mark.parametrize( ("custom_annotation", "error"), diff --git a/tests/onnx/data/reference_scales/custom_annotation_ref.json b/tests/onnx/data/reference_scales/custom_annotation_ref.json deleted file mode 100644 index dc8ca539323..00000000000 --- a/tests/onnx/data/reference_scales/custom_annotation_ref.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "primary_and_backup_precision": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int8_sym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_asym", - "group_size": 4 - } - }, - "mixed_precision": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int4_asym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int8_asym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int8_asym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int8_asym", - "group_size": -1 - } - }, - "ignored_scope": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int8_sym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "overlapping_annotations": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int8_sym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int4_asym", - "group_size": 4 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "group_size_fallback_adjust": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int4_asym", - "group_size": 4 - }, - "MatMul_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "group_size_fallback_ignore": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_sym", - "group_size": -1 - } - } -} \ No newline at end of file diff --git a/tests/onnx/data/reference_scales/group_size_fallback_adjust.json b/tests/onnx/data/reference_scales/group_size_fallback_adjust.json new file mode 100644 index 00000000000..78030303d7b --- /dev/null +++ b/tests/onnx/data/reference_scales/group_size_fallback_adjust.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int4_asym", + "group_size": 4 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/onnx/data/reference_scales/group_size_fallback_ignore.json b/tests/onnx/data/reference_scales/group_size_fallback_ignore.json new file mode 100644 index 00000000000..1c5df1151f3 --- /dev/null +++ b/tests/onnx/data/reference_scales/group_size_fallback_ignore.json @@ -0,0 +1,18 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/onnx/data/reference_scales/ignored_scope.json b/tests/onnx/data/reference_scales/ignored_scope.json new file mode 100644 index 00000000000..cbb89a90857 --- /dev/null +++ b/tests/onnx/data/reference_scales/ignored_scope.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/onnx/data/reference_scales/mixed_precision.json b/tests/onnx/data/reference_scales/mixed_precision.json new file mode 100644 index 00000000000..3c15eae243e --- /dev/null +++ b/tests/onnx/data/reference_scales/mixed_precision.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int4_asym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int8_asym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int8_asym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int8_asym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/onnx/data/reference_scales/overlapping_annotations.json b/tests/onnx/data/reference_scales/overlapping_annotations.json new file mode 100644 index 00000000000..7ccda47f024 --- /dev/null +++ b/tests/onnx/data/reference_scales/overlapping_annotations.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_asym", + "group_size": 4 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/onnx/data/reference_scales/primary_and_backup_precision.json b/tests/onnx/data/reference_scales/primary_and_backup_precision.json new file mode 100644 index 00000000000..b8ba4aa62a6 --- /dev/null +++ b/tests/onnx/data/reference_scales/primary_and_backup_precision.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_asym", + "group_size": 4 + } +} \ No newline at end of file diff --git a/tests/onnx/quantization/test_weights_compression.py b/tests/onnx/quantization/test_weights_compression.py index bd1e5c70acc..fdeb2f53a60 100644 --- a/tests/onnx/quantization/test_weights_compression.py +++ b/tests/onnx/quantization/test_weights_compression.py @@ -727,8 +727,8 @@ def get_awq_scale_ref_path() -> Path: return REFERENCE_SCALES_DIR / "awq_scale_ref.json" @staticmethod - def get_custom_annotation_ref_path() -> Path: - return REFERENCE_SCALES_DIR / "custom_annotation_ref.json" + def get_custom_annotation_ref_path(ref_name: str) -> Path: + return REFERENCE_SCALES_DIR / f"{ref_name}.json" @staticmethod def get_transform_func() -> Callable[..., Any] | None: diff --git a/tests/openvino/native/data/2026.2/reference_scales/custom_annotation_ref.json b/tests/openvino/native/data/2026.2/reference_scales/custom_annotation_ref.json deleted file mode 100644 index dc8ca539323..00000000000 --- a/tests/openvino/native/data/2026.2/reference_scales/custom_annotation_ref.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "primary_and_backup_precision": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int8_sym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_asym", - "group_size": 4 - } - }, - "mixed_precision": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int4_asym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int8_asym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int8_asym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int8_asym", - "group_size": -1 - } - }, - "ignored_scope": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int8_sym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "overlapping_annotations": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int8_sym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int4_asym", - "group_size": 4 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "group_size_fallback_adjust": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_1": { - "mode": "int4_asym", - "group_size": 4 - }, - "MatMul_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "group_size_fallback_ignore": { - "MatMul_0": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "MatMul_4": { - "mode": "int4_sym", - "group_size": -1 - } - } -} \ No newline at end of file diff --git a/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json b/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json new file mode 100644 index 00000000000..78030303d7b --- /dev/null +++ b/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int4_asym", + "group_size": 4 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json b/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json new file mode 100644 index 00000000000..1c5df1151f3 --- /dev/null +++ b/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json @@ -0,0 +1,18 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json b/tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json new file mode 100644 index 00000000000..cbb89a90857 --- /dev/null +++ b/tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json b/tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json new file mode 100644 index 00000000000..3c15eae243e --- /dev/null +++ b/tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int4_asym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int8_asym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int8_asym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int8_asym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json b/tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json new file mode 100644 index 00000000000..7ccda47f024 --- /dev/null +++ b/tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_asym", + "group_size": 4 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json b/tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json new file mode 100644 index 00000000000..b8ba4aa62a6 --- /dev/null +++ b/tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json @@ -0,0 +1,22 @@ +{ + "MatMul_0": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "MatMul_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "MatMul_4": { + "mode": "int4_asym", + "group_size": 4 + } +} \ No newline at end of file diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 4f6394b579e..316128512e2 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -2546,8 +2546,8 @@ def get_awq_scale_ref_path() -> Path: return get_actual_reference_for_current_openvino(REFERENCE_SCALES_DIR / "awq_scale_ref.json") @staticmethod - def get_custom_annotation_ref_path() -> Path: - return get_actual_reference_for_current_openvino(REFERENCE_SCALES_DIR / "custom_annotation_ref.json") + def get_custom_annotation_ref_path(ref_name: str) -> Path: + return get_actual_reference_for_current_openvino(REFERENCE_SCALES_DIR / f"{ref_name}.json") @pytest.fixture def transpose_a_supported(self) -> bool: diff --git a/tests/torch/data/function_hook/quantization/reference_scales/custom_annotation_ref.json b/tests/torch/data/function_hook/quantization/reference_scales/custom_annotation_ref.json deleted file mode 100644 index 302e18b4d36..00000000000 --- a/tests/torch/data/function_hook/quantization/reference_scales/custom_annotation_ref.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "primary_and_backup_precision": { - "/linear/0": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/1": { - "mode": "int8_sym", - "group_size": -1 - }, - "/linear/2": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/3": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/4": { - "mode": "int4_asym", - "group_size": 4 - } - }, - "mixed_precision": { - "/linear/0": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/1": { - "mode": "int4_asym", - "group_size": -1 - }, - "/linear/2": { - "mode": "int8_asym", - "group_size": -1 - }, - "/linear/3": { - "mode": "int8_asym", - "group_size": -1 - }, - "/linear/4": { - "mode": "int8_asym", - "group_size": -1 - } - }, - "ignored_scope": { - "/linear/0": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/1": { - "mode": "int8_sym", - "group_size": -1 - }, - "/linear/2": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/3": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "overlapping_annotations": { - "/linear/0": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/1": { - "mode": "int8_sym", - "group_size": -1 - }, - "/linear/2": { - "mode": "int4_asym", - "group_size": 4 - }, - "/linear/3": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "group_size_fallback_adjust": { - "/linear/0": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/1": { - "mode": "int4_asym", - "group_size": 4 - }, - "/linear/2": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/3": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "group_size_fallback_ignore": { - "/linear/0": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/2": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/3": { - "mode": "int4_sym", - "group_size": -1 - }, - "/linear/4": { - "mode": "int4_sym", - "group_size": -1 - } - } -} \ No newline at end of file diff --git a/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json b/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json new file mode 100644 index 00000000000..4b28a616884 --- /dev/null +++ b/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json @@ -0,0 +1,22 @@ +{ + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int4_asym", + "group_size": 4 + }, + "/linear/2": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json b/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json new file mode 100644 index 00000000000..2057f16cac1 --- /dev/null +++ b/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json @@ -0,0 +1,18 @@ +{ + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json b/tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json new file mode 100644 index 00000000000..2a062a89ba3 --- /dev/null +++ b/tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json @@ -0,0 +1,22 @@ +{ + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int8_sym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json b/tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json new file mode 100644 index 00000000000..4e8dfcd7b72 --- /dev/null +++ b/tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json @@ -0,0 +1,22 @@ +{ + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int4_asym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int8_asym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int8_asym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int8_asym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json b/tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json new file mode 100644 index 00000000000..5e5b0abb41f --- /dev/null +++ b/tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json @@ -0,0 +1,22 @@ +{ + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int8_sym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int4_asym", + "group_size": 4 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json b/tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json new file mode 100644 index 00000000000..a415b54e145 --- /dev/null +++ b/tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json @@ -0,0 +1,22 @@ +{ + "/linear/0": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/1": { + "mode": "int8_sym", + "group_size": -1 + }, + "/linear/2": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/3": { + "mode": "int4_sym", + "group_size": -1 + }, + "/linear/4": { + "mode": "int4_asym", + "group_size": 4 + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/reference_scales/custom_annotation_ref.json b/tests/torch/data/fx/reference_scales/custom_annotation_ref.json deleted file mode 100644 index ab5e548fb23..00000000000 --- a/tests/torch/data/fx/reference_scales/custom_annotation_ref.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "primary_and_backup_precision": { - "linear": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_1": { - "mode": "int8_sym", - "group_size": -1 - }, - "linear_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_4": { - "mode": "int4_asym", - "group_size": 4 - } - }, - "mixed_precision": { - "linear": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_1": { - "mode": "int4_asym", - "group_size": -1 - }, - "linear_2": { - "mode": "int8_asym", - "group_size": -1 - }, - "linear_3": { - "mode": "int8_asym", - "group_size": -1 - }, - "linear_4": { - "mode": "int8_asym", - "group_size": -1 - } - }, - "ignored_scope": { - "linear": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_1": { - "mode": "int8_sym", - "group_size": -1 - }, - "linear_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "overlapping_annotations": { - "linear": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_1": { - "mode": "int8_sym", - "group_size": -1 - }, - "linear_2": { - "mode": "int4_asym", - "group_size": 4 - }, - "linear_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "group_size_fallback_adjust": { - "linear": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_1": { - "mode": "int4_asym", - "group_size": 4 - }, - "linear_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_4": { - "mode": "int4_sym", - "group_size": -1 - } - }, - "group_size_fallback_ignore": { - "linear": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_2": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_3": { - "mode": "int4_sym", - "group_size": -1 - }, - "linear_4": { - "mode": "int4_sym", - "group_size": -1 - } - } -} \ No newline at end of file diff --git a/tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json b/tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json new file mode 100644 index 00000000000..6740b56c68d --- /dev/null +++ b/tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json @@ -0,0 +1,22 @@ +{ + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int4_asym", + "group_size": 4 + }, + "linear_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json b/tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json new file mode 100644 index 00000000000..f154364552a --- /dev/null +++ b/tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json @@ -0,0 +1,18 @@ +{ + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/reference_scales/ignored_scope.json b/tests/torch/data/fx/reference_scales/ignored_scope.json new file mode 100644 index 00000000000..e3f5f12ea78 --- /dev/null +++ b/tests/torch/data/fx/reference_scales/ignored_scope.json @@ -0,0 +1,22 @@ +{ + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "linear_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/reference_scales/mixed_precision.json b/tests/torch/data/fx/reference_scales/mixed_precision.json new file mode 100644 index 00000000000..c6d88127bba --- /dev/null +++ b/tests/torch/data/fx/reference_scales/mixed_precision.json @@ -0,0 +1,22 @@ +{ + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int4_asym", + "group_size": -1 + }, + "linear_2": { + "mode": "int8_asym", + "group_size": -1 + }, + "linear_3": { + "mode": "int8_asym", + "group_size": -1 + }, + "linear_4": { + "mode": "int8_asym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/reference_scales/overlapping_annotations.json b/tests/torch/data/fx/reference_scales/overlapping_annotations.json new file mode 100644 index 00000000000..8313516f7f5 --- /dev/null +++ b/tests/torch/data/fx/reference_scales/overlapping_annotations.json @@ -0,0 +1,22 @@ +{ + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "linear_2": { + "mode": "int4_asym", + "group_size": 4 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_sym", + "group_size": -1 + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/reference_scales/primary_and_backup_precision.json b/tests/torch/data/fx/reference_scales/primary_and_backup_precision.json new file mode 100644 index 00000000000..682f57e2f14 --- /dev/null +++ b/tests/torch/data/fx/reference_scales/primary_and_backup_precision.json @@ -0,0 +1,22 @@ +{ + "linear": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_1": { + "mode": "int8_sym", + "group_size": -1 + }, + "linear_2": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_3": { + "mode": "int4_sym", + "group_size": -1 + }, + "linear_4": { + "mode": "int4_asym", + "group_size": 4 + } +} \ No newline at end of file diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index b605d10aab9..e60e7014c48 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -724,8 +724,8 @@ def get_awq_scale_ref_path() -> Path: return REFERENCE_SCALES_DIR / "awq_scale_ref.json" @staticmethod - def get_custom_annotation_ref_path() -> Path: - return REFERENCE_SCALES_DIR / "custom_annotation_ref.json" + def get_custom_annotation_ref_path(ref_name: str) -> Path: + return REFERENCE_SCALES_DIR / f"{ref_name}.json" @staticmethod def get_transposable_awq_model(transpose_a: bool, transpose_b: bool, input_shape=None, is_3d_weights: bool = False): diff --git a/tests/torch/fx/test_weights_compression.py b/tests/torch/fx/test_weights_compression.py index 5f50de8ae29..f00a609d7fc 100644 --- a/tests/torch/fx/test_weights_compression.py +++ b/tests/torch/fx/test_weights_compression.py @@ -514,8 +514,8 @@ def get_awq_scale_ref_path() -> Path: return REFERENCE_SCALES_DIR / "awq_scale_ref.json" @staticmethod - def get_custom_annotation_ref_path() -> Path: - return REFERENCE_SCALES_DIR / "custom_annotation_ref.json" + def get_custom_annotation_ref_path(ref_name: str) -> Path: + return REFERENCE_SCALES_DIR / f"{ref_name}.json" @staticmethod def get_transposable_awq_model(transpose_a: bool, transpose_b: bool, input_shape=None, is_3d_weights: bool = False): From fffc8cf01c1beec4ccec6452060c388bdf103003 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 15 Sep 2026 14:45:48 +0400 Subject: [PATCH 10/43] newline --- .../onnx/data/reference_scales/group_size_fallback_adjust.json | 2 +- .../onnx/data/reference_scales/group_size_fallback_ignore.json | 2 +- tests/onnx/data/reference_scales/ignored_scope.json | 2 +- tests/onnx/data/reference_scales/mixed_precision.json | 2 +- tests/onnx/data/reference_scales/overlapping_annotations.json | 2 +- .../data/reference_scales/primary_and_backup_precision.json | 2 +- .../2026.2/reference_scales/group_size_fallback_adjust.json | 2 +- .../2026.2/reference_scales/group_size_fallback_ignore.json | 2 +- .../native/data/2026.2/reference_scales/ignored_scope.json | 2 +- .../native/data/2026.2/reference_scales/mixed_precision.json | 2 +- .../data/2026.2/reference_scales/overlapping_annotations.json | 2 +- .../2026.2/reference_scales/primary_and_backup_precision.json | 2 +- .../reference_scales/group_size_fallback_adjust.json | 2 +- .../reference_scales/group_size_fallback_ignore.json | 2 +- .../quantization/reference_scales/ignored_scope.json | 2 +- .../quantization/reference_scales/mixed_precision.json | 2 +- .../quantization/reference_scales/overlapping_annotations.json | 2 +- .../reference_scales/primary_and_backup_precision.json | 2 +- .../data/fx/reference_scales/group_size_fallback_adjust.json | 2 +- .../data/fx/reference_scales/group_size_fallback_ignore.json | 2 +- tests/torch/data/fx/reference_scales/ignored_scope.json | 2 +- tests/torch/data/fx/reference_scales/mixed_precision.json | 2 +- .../torch/data/fx/reference_scales/overlapping_annotations.json | 2 +- .../data/fx/reference_scales/primary_and_backup_precision.json | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/onnx/data/reference_scales/group_size_fallback_adjust.json b/tests/onnx/data/reference_scales/group_size_fallback_adjust.json index 78030303d7b..f68f05eac1f 100644 --- a/tests/onnx/data/reference_scales/group_size_fallback_adjust.json +++ b/tests/onnx/data/reference_scales/group_size_fallback_adjust.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/onnx/data/reference_scales/group_size_fallback_ignore.json b/tests/onnx/data/reference_scales/group_size_fallback_ignore.json index 1c5df1151f3..b0e4e802a9b 100644 --- a/tests/onnx/data/reference_scales/group_size_fallback_ignore.json +++ b/tests/onnx/data/reference_scales/group_size_fallback_ignore.json @@ -15,4 +15,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/onnx/data/reference_scales/ignored_scope.json b/tests/onnx/data/reference_scales/ignored_scope.json index cbb89a90857..d19414720f5 100644 --- a/tests/onnx/data/reference_scales/ignored_scope.json +++ b/tests/onnx/data/reference_scales/ignored_scope.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/onnx/data/reference_scales/mixed_precision.json b/tests/onnx/data/reference_scales/mixed_precision.json index 3c15eae243e..8c7101fd8f2 100644 --- a/tests/onnx/data/reference_scales/mixed_precision.json +++ b/tests/onnx/data/reference_scales/mixed_precision.json @@ -19,4 +19,4 @@ "mode": "int8_asym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/onnx/data/reference_scales/overlapping_annotations.json b/tests/onnx/data/reference_scales/overlapping_annotations.json index 7ccda47f024..dfb1f734ec7 100644 --- a/tests/onnx/data/reference_scales/overlapping_annotations.json +++ b/tests/onnx/data/reference_scales/overlapping_annotations.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/onnx/data/reference_scales/primary_and_backup_precision.json b/tests/onnx/data/reference_scales/primary_and_backup_precision.json index b8ba4aa62a6..60d3061df24 100644 --- a/tests/onnx/data/reference_scales/primary_and_backup_precision.json +++ b/tests/onnx/data/reference_scales/primary_and_backup_precision.json @@ -19,4 +19,4 @@ "mode": "int4_asym", "group_size": 4 } -} \ No newline at end of file +} diff --git a/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json b/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json index 78030303d7b..f68f05eac1f 100644 --- a/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json +++ b/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json b/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json index 1c5df1151f3..b0e4e802a9b 100644 --- a/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json +++ b/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json @@ -15,4 +15,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json b/tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json index cbb89a90857..d19414720f5 100644 --- a/tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json +++ b/tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json b/tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json index 3c15eae243e..8c7101fd8f2 100644 --- a/tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json +++ b/tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json @@ -19,4 +19,4 @@ "mode": "int8_asym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json b/tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json index 7ccda47f024..dfb1f734ec7 100644 --- a/tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json +++ b/tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json b/tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json index b8ba4aa62a6..60d3061df24 100644 --- a/tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json +++ b/tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json @@ -19,4 +19,4 @@ "mode": "int4_asym", "group_size": 4 } -} \ No newline at end of file +} diff --git a/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json b/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json index 4b28a616884..2b446595a2e 100644 --- a/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json +++ b/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json b/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json index 2057f16cac1..a0f01bc6a54 100644 --- a/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json +++ b/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json @@ -15,4 +15,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json b/tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json index 2a062a89ba3..dd5b2b2aceb 100644 --- a/tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json +++ b/tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json b/tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json index 4e8dfcd7b72..b1bf72f23aa 100644 --- a/tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json +++ b/tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json @@ -19,4 +19,4 @@ "mode": "int8_asym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json b/tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json index 5e5b0abb41f..23b9b647a02 100644 --- a/tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json +++ b/tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json b/tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json index a415b54e145..c25d2ae3532 100644 --- a/tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json +++ b/tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json @@ -19,4 +19,4 @@ "mode": "int4_asym", "group_size": 4 } -} \ No newline at end of file +} diff --git a/tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json b/tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json index 6740b56c68d..b0717d74c14 100644 --- a/tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json +++ b/tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json b/tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json index f154364552a..eb67736e217 100644 --- a/tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json +++ b/tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json @@ -15,4 +15,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/fx/reference_scales/ignored_scope.json b/tests/torch/data/fx/reference_scales/ignored_scope.json index e3f5f12ea78..dae3b7b0ba3 100644 --- a/tests/torch/data/fx/reference_scales/ignored_scope.json +++ b/tests/torch/data/fx/reference_scales/ignored_scope.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/fx/reference_scales/mixed_precision.json b/tests/torch/data/fx/reference_scales/mixed_precision.json index c6d88127bba..7206a76df08 100644 --- a/tests/torch/data/fx/reference_scales/mixed_precision.json +++ b/tests/torch/data/fx/reference_scales/mixed_precision.json @@ -19,4 +19,4 @@ "mode": "int8_asym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/fx/reference_scales/overlapping_annotations.json b/tests/torch/data/fx/reference_scales/overlapping_annotations.json index 8313516f7f5..7d9e8c5043c 100644 --- a/tests/torch/data/fx/reference_scales/overlapping_annotations.json +++ b/tests/torch/data/fx/reference_scales/overlapping_annotations.json @@ -19,4 +19,4 @@ "mode": "int4_sym", "group_size": -1 } -} \ No newline at end of file +} diff --git a/tests/torch/data/fx/reference_scales/primary_and_backup_precision.json b/tests/torch/data/fx/reference_scales/primary_and_backup_precision.json index 682f57e2f14..5c083176c3f 100644 --- a/tests/torch/data/fx/reference_scales/primary_and_backup_precision.json +++ b/tests/torch/data/fx/reference_scales/primary_and_backup_precision.json @@ -19,4 +19,4 @@ "mode": "int4_asym", "group_size": 4 } -} \ No newline at end of file +} From b2da0b40021edfdb42ac58fa5c71d363fa72d908 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 15 Sep 2026 15:23:30 +0400 Subject: [PATCH 11/43] ref --- .../group_size_fallback_adjust.json | 0 .../group_size_fallback_ignore.json | 0 .../ignored_scope.json | 0 .../mixed_precision.json | 0 .../overlapping_annotations.json | 0 .../primary_and_backup_precision.json | 0 tests/onnx/quantization/test_weights_compression.py | 3 ++- .../group_size_fallback_adjust.json | 0 .../group_size_fallback_ignore.json | 0 .../references_custom_annotation}/ignored_scope.json | 0 .../references_custom_annotation}/mixed_precision.json | 0 .../overlapping_annotations.json | 0 .../primary_and_backup_precision.json | 0 .../openvino/native/quantization/test_weights_compression.py | 4 +++- .../group_size_fallback_adjust.json | 0 .../group_size_fallback_ignore.json | 0 .../ignored_scope.json | 0 .../mixed_precision.json | 0 .../overlapping_annotations.json | 0 .../primary_and_backup_precision.json | 0 .../group_size_fallback_adjust.json | 0 .../group_size_fallback_ignore.json | 0 .../ignored_scope.json | 0 .../mixed_precision.json | 0 .../overlapping_annotations.json | 0 .../primary_and_backup_precision.json | 0 .../function_hook/quantization/test_weights_compression.py | 5 ++++- tests/torch/fx/test_weights_compression.py | 3 ++- 28 files changed, 11 insertions(+), 4 deletions(-) rename tests/onnx/data/{reference_scales => references_custom_annotation}/group_size_fallback_adjust.json (100%) rename tests/onnx/data/{reference_scales => references_custom_annotation}/group_size_fallback_ignore.json (100%) rename tests/onnx/data/{reference_scales => references_custom_annotation}/ignored_scope.json (100%) rename tests/onnx/data/{reference_scales => references_custom_annotation}/mixed_precision.json (100%) rename tests/onnx/data/{reference_scales => references_custom_annotation}/overlapping_annotations.json (100%) rename tests/onnx/data/{reference_scales => references_custom_annotation}/primary_and_backup_precision.json (100%) rename tests/openvino/native/data/{2026.2/reference_scales => 2026.4/references_custom_annotation}/group_size_fallback_adjust.json (100%) rename tests/openvino/native/data/{2026.2/reference_scales => 2026.4/references_custom_annotation}/group_size_fallback_ignore.json (100%) rename tests/openvino/native/data/{2026.2/reference_scales => 2026.4/references_custom_annotation}/ignored_scope.json (100%) rename tests/openvino/native/data/{2026.2/reference_scales => 2026.4/references_custom_annotation}/mixed_precision.json (100%) rename tests/openvino/native/data/{2026.2/reference_scales => 2026.4/references_custom_annotation}/overlapping_annotations.json (100%) rename tests/openvino/native/data/{2026.2/reference_scales => 2026.4/references_custom_annotation}/primary_and_backup_precision.json (100%) rename tests/torch/data/function_hook/quantization/{reference_scales => references_custom_annotation}/group_size_fallback_adjust.json (100%) rename tests/torch/data/function_hook/quantization/{reference_scales => references_custom_annotation}/group_size_fallback_ignore.json (100%) rename tests/torch/data/function_hook/quantization/{reference_scales => references_custom_annotation}/ignored_scope.json (100%) rename tests/torch/data/function_hook/quantization/{reference_scales => references_custom_annotation}/mixed_precision.json (100%) rename tests/torch/data/function_hook/quantization/{reference_scales => references_custom_annotation}/overlapping_annotations.json (100%) rename tests/torch/data/function_hook/quantization/{reference_scales => references_custom_annotation}/primary_and_backup_precision.json (100%) rename tests/torch/data/fx/{reference_scales => references_custom_annotation}/group_size_fallback_adjust.json (100%) rename tests/torch/data/fx/{reference_scales => references_custom_annotation}/group_size_fallback_ignore.json (100%) rename tests/torch/data/fx/{reference_scales => references_custom_annotation}/ignored_scope.json (100%) rename tests/torch/data/fx/{reference_scales => references_custom_annotation}/mixed_precision.json (100%) rename tests/torch/data/fx/{reference_scales => references_custom_annotation}/overlapping_annotations.json (100%) rename tests/torch/data/fx/{reference_scales => references_custom_annotation}/primary_and_backup_precision.json (100%) diff --git a/tests/onnx/data/reference_scales/group_size_fallback_adjust.json b/tests/onnx/data/references_custom_annotation/group_size_fallback_adjust.json similarity index 100% rename from tests/onnx/data/reference_scales/group_size_fallback_adjust.json rename to tests/onnx/data/references_custom_annotation/group_size_fallback_adjust.json diff --git a/tests/onnx/data/reference_scales/group_size_fallback_ignore.json b/tests/onnx/data/references_custom_annotation/group_size_fallback_ignore.json similarity index 100% rename from tests/onnx/data/reference_scales/group_size_fallback_ignore.json rename to tests/onnx/data/references_custom_annotation/group_size_fallback_ignore.json diff --git a/tests/onnx/data/reference_scales/ignored_scope.json b/tests/onnx/data/references_custom_annotation/ignored_scope.json similarity index 100% rename from tests/onnx/data/reference_scales/ignored_scope.json rename to tests/onnx/data/references_custom_annotation/ignored_scope.json diff --git a/tests/onnx/data/reference_scales/mixed_precision.json b/tests/onnx/data/references_custom_annotation/mixed_precision.json similarity index 100% rename from tests/onnx/data/reference_scales/mixed_precision.json rename to tests/onnx/data/references_custom_annotation/mixed_precision.json diff --git a/tests/onnx/data/reference_scales/overlapping_annotations.json b/tests/onnx/data/references_custom_annotation/overlapping_annotations.json similarity index 100% rename from tests/onnx/data/reference_scales/overlapping_annotations.json rename to tests/onnx/data/references_custom_annotation/overlapping_annotations.json diff --git a/tests/onnx/data/reference_scales/primary_and_backup_precision.json b/tests/onnx/data/references_custom_annotation/primary_and_backup_precision.json similarity index 100% rename from tests/onnx/data/reference_scales/primary_and_backup_precision.json rename to tests/onnx/data/references_custom_annotation/primary_and_backup_precision.json diff --git a/tests/onnx/quantization/test_weights_compression.py b/tests/onnx/quantization/test_weights_compression.py index fdeb2f53a60..9ac60351fbb 100644 --- a/tests/onnx/quantization/test_weights_compression.py +++ b/tests/onnx/quantization/test_weights_compression.py @@ -50,6 +50,7 @@ from tests.onnx.conftest import ONNX_TEST_ROOT REFERENCE_SCALES_DIR = ONNX_TEST_ROOT / "data" / "reference_scales" +CUSTOM_ANNOTATION_REFERENCES_DIR = ONNX_TEST_ROOT / "data" / "references_custom_annotation" UNSUPPORTED_MODES = ( CompressWeightsMode.NF4, @@ -728,7 +729,7 @@ def get_awq_scale_ref_path() -> Path: @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: - return REFERENCE_SCALES_DIR / f"{ref_name}.json" + return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" @staticmethod def get_transform_func() -> Callable[..., Any] | None: diff --git a/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json b/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_adjust.json similarity index 100% rename from tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_adjust.json rename to tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_adjust.json diff --git a/tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json b/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_ignore.json similarity index 100% rename from tests/openvino/native/data/2026.2/reference_scales/group_size_fallback_ignore.json rename to tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_ignore.json diff --git a/tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json b/tests/openvino/native/data/2026.4/references_custom_annotation/ignored_scope.json similarity index 100% rename from tests/openvino/native/data/2026.2/reference_scales/ignored_scope.json rename to tests/openvino/native/data/2026.4/references_custom_annotation/ignored_scope.json diff --git a/tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json b/tests/openvino/native/data/2026.4/references_custom_annotation/mixed_precision.json similarity index 100% rename from tests/openvino/native/data/2026.2/reference_scales/mixed_precision.json rename to tests/openvino/native/data/2026.4/references_custom_annotation/mixed_precision.json diff --git a/tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json b/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json similarity index 100% rename from tests/openvino/native/data/2026.2/reference_scales/overlapping_annotations.json rename to tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json diff --git a/tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json b/tests/openvino/native/data/2026.4/references_custom_annotation/primary_and_backup_precision.json similarity index 100% rename from tests/openvino/native/data/2026.2/reference_scales/primary_and_backup_precision.json rename to tests/openvino/native/data/2026.4/references_custom_annotation/primary_and_backup_precision.json diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 316128512e2..6b10de733fb 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -88,6 +88,8 @@ from tests.openvino.native.models import WeightsModel from tests.openvino.native.quantization.test_fq_params_calculation import REFERENCE_SCALES_DIR +CUSTOM_ANNOTATION_REFERENCES_DIR = Path("references_custom_annotation") + TEST_MODELS = { IntegerModel: ["matmul_2_data", "gather_2_data", "matmul_1_data"], WeightsModel: ["weights_0", "weights_1"], @@ -2547,7 +2549,7 @@ def get_awq_scale_ref_path() -> Path: @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: - return get_actual_reference_for_current_openvino(REFERENCE_SCALES_DIR / f"{ref_name}.json") + return get_actual_reference_for_current_openvino(CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json") @pytest.fixture def transpose_a_supported(self) -> bool: diff --git a/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_adjust.json similarity index 100% rename from tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_adjust.json rename to tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_adjust.json diff --git a/tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_ignore.json similarity index 100% rename from tests/torch/data/function_hook/quantization/reference_scales/group_size_fallback_ignore.json rename to tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_ignore.json diff --git a/tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/ignored_scope.json similarity index 100% rename from tests/torch/data/function_hook/quantization/reference_scales/ignored_scope.json rename to tests/torch/data/function_hook/quantization/references_custom_annotation/ignored_scope.json diff --git a/tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/mixed_precision.json similarity index 100% rename from tests/torch/data/function_hook/quantization/reference_scales/mixed_precision.json rename to tests/torch/data/function_hook/quantization/references_custom_annotation/mixed_precision.json diff --git a/tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json similarity index 100% rename from tests/torch/data/function_hook/quantization/reference_scales/overlapping_annotations.json rename to tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json diff --git a/tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/primary_and_backup_precision.json similarity index 100% rename from tests/torch/data/function_hook/quantization/reference_scales/primary_and_backup_precision.json rename to tests/torch/data/function_hook/quantization/references_custom_annotation/primary_and_backup_precision.json diff --git a/tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json b/tests/torch/data/fx/references_custom_annotation/group_size_fallback_adjust.json similarity index 100% rename from tests/torch/data/fx/reference_scales/group_size_fallback_adjust.json rename to tests/torch/data/fx/references_custom_annotation/group_size_fallback_adjust.json diff --git a/tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json b/tests/torch/data/fx/references_custom_annotation/group_size_fallback_ignore.json similarity index 100% rename from tests/torch/data/fx/reference_scales/group_size_fallback_ignore.json rename to tests/torch/data/fx/references_custom_annotation/group_size_fallback_ignore.json diff --git a/tests/torch/data/fx/reference_scales/ignored_scope.json b/tests/torch/data/fx/references_custom_annotation/ignored_scope.json similarity index 100% rename from tests/torch/data/fx/reference_scales/ignored_scope.json rename to tests/torch/data/fx/references_custom_annotation/ignored_scope.json diff --git a/tests/torch/data/fx/reference_scales/mixed_precision.json b/tests/torch/data/fx/references_custom_annotation/mixed_precision.json similarity index 100% rename from tests/torch/data/fx/reference_scales/mixed_precision.json rename to tests/torch/data/fx/references_custom_annotation/mixed_precision.json diff --git a/tests/torch/data/fx/reference_scales/overlapping_annotations.json b/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json similarity index 100% rename from tests/torch/data/fx/reference_scales/overlapping_annotations.json rename to tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json diff --git a/tests/torch/data/fx/reference_scales/primary_and_backup_precision.json b/tests/torch/data/fx/references_custom_annotation/primary_and_backup_precision.json similarity index 100% rename from tests/torch/data/fx/reference_scales/primary_and_backup_precision.json rename to tests/torch/data/fx/references_custom_annotation/primary_and_backup_precision.json diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index e60e7014c48..fb747262cb6 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -52,6 +52,9 @@ from tests.torch.test_tensor import cast_to REFERENCE_SCALES_DIR = TEST_ROOT / "torch" / "data" / "function_hook" / "quantization" / "reference_scales" +CUSTOM_ANNOTATION_REFERENCES_DIR = ( + TEST_ROOT / "torch" / "data" / "function_hook" / "quantization" / "references_custom_annotation" +) ALL_SENSITIVITY_METRICS = list(SensitivityMetric) @@ -725,7 +728,7 @@ def get_awq_scale_ref_path() -> Path: @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: - return REFERENCE_SCALES_DIR / f"{ref_name}.json" + return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" @staticmethod def get_transposable_awq_model(transpose_a: bool, transpose_b: bool, input_shape=None, is_3d_weights: bool = False): diff --git a/tests/torch/fx/test_weights_compression.py b/tests/torch/fx/test_weights_compression.py index f00a609d7fc..650279833c3 100644 --- a/tests/torch/fx/test_weights_compression.py +++ b/tests/torch/fx/test_weights_compression.py @@ -60,6 +60,7 @@ from tests.torch.test_tensor import cast_to REFERENCE_SCALES_DIR = TEST_ROOT / "torch" / "data" / "fx" / "reference_scales" +CUSTOM_ANNOTATION_REFERENCES_DIR = TEST_ROOT / "torch" / "data" / "fx" / "references_custom_annotation" DATA_BASED_SENSITIVITY_METRICS = ( SensitivityMetric.HESSIAN_INPUT_ACTIVATION, @@ -515,7 +516,7 @@ def get_awq_scale_ref_path() -> Path: @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: - return REFERENCE_SCALES_DIR / f"{ref_name}.json" + return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" @staticmethod def get_transposable_awq_model(transpose_a: bool, transpose_b: bool, input_shape=None, is_3d_weights: bool = False): From cb96a10ac0571a6592834cc207794cc93a7670d2 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 15 Sep 2026 15:46:50 +0400 Subject: [PATCH 12/43] test --- .../template_test_weights_compression.py | 76 +++++++++++++------ 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 02a323eff49..5397b251732 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -1001,35 +1001,59 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa all_weight_params = captured_weight_params[-1] return {wp.node_with_weight.node_name: wp.compression_config for wp in all_weight_params} + # The weight nodes of the sequential MatMul model are named "MatMul_" / "linear_" / "/linear/" + # depending on the backend, so they are referred to by a pattern that matches the node index. @pytest.mark.parametrize( - ("kwargs", "annotations", "ignored_indices"), + ("kwargs", "annotations", "ignored_scope"), [ pytest.param( dict(ratio=1.0, all_layers=False), [ - ([1], WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1)), - ([-1], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4)), + ( + nncf.CustomAnnotationScope(patterns=[".*1$"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ), + ( + nncf.CustomAnnotationScope(patterns=[".*4$"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4), + ), ], None, id="primary_and_backup_precision", ), pytest.param( dict(ratio=0.5, all_layers=True), - [([1], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=-1))], + [ + ( + nncf.CustomAnnotationScope(patterns=[".*1$"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=-1), + ) + ], None, id="mixed_precision", ), pytest.param( dict(ratio=1.0, all_layers=True), - [([1], WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1))], - [1], + [ + ( + nncf.CustomAnnotationScope(patterns=[".*1$"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + ], + IgnoredScope(patterns=[".*1$"]), id="ignored_scope", ), pytest.param( dict(ratio=1.0, all_layers=True), [ - ([1, 2], WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1)), - ([2], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4)), + ( + nncf.CustomAnnotationScope(patterns=[".*1$", ".*2$"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ), + ( + nncf.CustomAnnotationScope(patterns=[".*2$"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4), + ), ], None, id="overlapping_annotations", @@ -1043,7 +1067,12 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa ), ), # The channel size of the model is 4, so the group size of 3 is invalid - [([1], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3))], + [ + ( + nncf.CustomAnnotationScope(patterns=[".*1$"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), + ) + ], None, id="group_size_fallback_adjust", ), @@ -1055,34 +1084,31 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa group_size_fallback_mode=nncf.GroupSizeFallbackMode.IGNORE, min_adjusted_group_size=4 ), ), - [([1], WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3))], + [ + ( + nncf.CustomAnnotationScope(patterns=[".*1$"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), + ) + ], None, id="group_size_fallback_ignore", ), ], ) - def test_custom_annotation(self, kwargs, annotations, ignored_indices, request): + def test_custom_annotation(self, kwargs, annotations, ignored_scope, request): """ Compares the compression config assigned to each weight node with the reference one when the given custom - annotation is applied. The annotation scopes and the ignored scope are given by the indices of the weight - nodes in the topological order. A node that is not compressed has no config in the reference file. + annotation is applied. A node that is not compressed has no config in the reference file. Set the NNCF_TEST_REGEN_DOT environment variable to regenerate the reference file. """ - common_kwargs = dict(mode=CompressWeightsMode.INT4_SYM, group_size=-1, **kwargs) - node_names = list(self._compress_and_get_configs(model=self._get_sequential_matmul_model(), **common_kwargs)) - - if ignored_indices is not None: - common_kwargs["ignored_scope"] = IgnoredScope(names=[node_names[i] for i in ignored_indices]) configs = self._compress_and_get_configs( model=self._get_sequential_matmul_model(), - **common_kwargs, - custom_annotation=[ - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(names=[node_names[i] for i in indices]), config=config - ) - for indices, config in annotations - ], + mode=CompressWeightsMode.INT4_SYM, + group_size=-1, + ignored_scope=ignored_scope, + **kwargs, + custom_annotation=[nncf.CustomAnnotation(scope=scope, config=config) for scope, config in annotations], ) actual_configs = { From dc1f7808e2be48a7353b32cf1f5841397889c8f4 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 15 Sep 2026 16:00:07 +0400 Subject: [PATCH 13/43] use full ref --- .../template_test_weights_compression.py | 14 +++++--------- .../group_size_fallback_adjust.json | 15 ++++++++++----- .../group_size_fallback_ignore.json | 12 ++++++++---- .../ignored_scope.json | 15 ++++++++++----- .../mixed_precision.json | 15 ++++++++++----- .../overlapping_annotations.json | 15 ++++++++++----- .../primary_and_backup_precision.json | 15 ++++++++++----- .../group_size_fallback_adjust.json | 15 ++++++++++----- .../group_size_fallback_ignore.json | 12 ++++++++---- .../ignored_scope.json | 15 ++++++++++----- .../mixed_precision.json | 15 ++++++++++----- .../overlapping_annotations.json | 15 ++++++++++----- .../primary_and_backup_precision.json | 15 ++++++++++----- .../group_size_fallback_adjust.json | 15 ++++++++++----- .../group_size_fallback_ignore.json | 12 ++++++++---- .../ignored_scope.json | 15 ++++++++++----- .../mixed_precision.json | 15 ++++++++++----- .../overlapping_annotations.json | 15 ++++++++++----- .../primary_and_backup_precision.json | 15 ++++++++++----- .../group_size_fallback_adjust.json | 15 ++++++++++----- .../group_size_fallback_ignore.json | 12 ++++++++---- .../ignored_scope.json | 15 ++++++++++----- .../mixed_precision.json | 15 ++++++++++----- .../overlapping_annotations.json | 15 ++++++++++----- .../primary_and_backup_precision.json | 15 ++++++++++----- 25 files changed, 237 insertions(+), 125 deletions(-) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 5397b251732..5c50e867376 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -12,6 +12,7 @@ import os from abc import ABC from abc import abstractmethod +from dataclasses import asdict from dataclasses import dataclass from functools import reduce from operator import mul @@ -981,7 +982,7 @@ def _get_sequential_matmul_model(self): self.get_sequential_matmul_model(transpose_a=False), np.ones([1, 4, 4], dtype=np.float32) ) - def _compress_and_get_configs(self, **kwargs) -> dict[str, WeightCompressionConfig]: + def _compress_and_get_configs(self, **kwargs) -> dict[str, dict[str, Any]]: """ Compresses a model and returns the compression config assigned to each compressed weight node. @@ -999,7 +1000,7 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa compress_weights(**kwargs) all_weight_params = captured_weight_params[-1] - return {wp.node_with_weight.node_name: wp.compression_config for wp in all_weight_params} + return {wp.node_with_weight.node_name: asdict(wp.compression_config) for wp in all_weight_params} # The weight nodes of the sequential MatMul model are named "MatMul_" / "linear_" / "/linear/" # depending on the backend, so they are referred to by a pattern that matches the node index. @@ -1111,15 +1112,11 @@ def test_custom_annotation(self, kwargs, annotations, ignored_scope, request): custom_annotation=[nncf.CustomAnnotation(scope=scope, config=config) for scope, config in annotations], ) - actual_configs = { - node_name: {"mode": config.mode, "group_size": config.group_size} for node_name, config in configs.items() - } - ref_path = self.get_custom_annotation_ref_path(request.node.callspec.id) if os.getenv("NNCF_TEST_REGEN_DOT") is not None: - dump_to_json(ref_path, actual_configs) + dump_to_json(ref_path, configs) - assert actual_configs == load_json(ref_path) + assert configs == load_json(ref_path) @pytest.mark.parametrize( ("custom_annotation", "error"), @@ -1160,7 +1157,6 @@ def test_custom_annotation(self, kwargs, annotations, ignored_scope, request): [ nncf.CustomAnnotation( scope=nncf.CustomAnnotationScope(patterns=[".*"]), - # The channel size of the model is 4, so the group size of 3 is invalid config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), ) ], diff --git a/tests/onnx/data/references_custom_annotation/group_size_fallback_adjust.json b/tests/onnx/data/references_custom_annotation/group_size_fallback_adjust.json index f68f05eac1f..c798b116c5b 100644 --- a/tests/onnx/data/references_custom_annotation/group_size_fallback_adjust.json +++ b/tests/onnx/data/references_custom_annotation/group_size_fallback_adjust.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null }, "MatMul_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/onnx/data/references_custom_annotation/group_size_fallback_ignore.json b/tests/onnx/data/references_custom_annotation/group_size_fallback_ignore.json index b0e4e802a9b..e8976e5f595 100644 --- a/tests/onnx/data/references_custom_annotation/group_size_fallback_ignore.json +++ b/tests/onnx/data/references_custom_annotation/group_size_fallback_ignore.json @@ -1,18 +1,22 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/onnx/data/references_custom_annotation/ignored_scope.json b/tests/onnx/data/references_custom_annotation/ignored_scope.json index d19414720f5..1cda8feed3e 100644 --- a/tests/onnx/data/references_custom_annotation/ignored_scope.json +++ b/tests/onnx/data/references_custom_annotation/ignored_scope.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/onnx/data/references_custom_annotation/mixed_precision.json b/tests/onnx/data/references_custom_annotation/mixed_precision.json index 8c7101fd8f2..5931e0a1eff 100644 --- a/tests/onnx/data/references_custom_annotation/mixed_precision.json +++ b/tests/onnx/data/references_custom_annotation/mixed_precision.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int4_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/onnx/data/references_custom_annotation/overlapping_annotations.json b/tests/onnx/data/references_custom_annotation/overlapping_annotations.json index dfb1f734ec7..a7cd32e850a 100644 --- a/tests/onnx/data/references_custom_annotation/overlapping_annotations.json +++ b/tests/onnx/data/references_custom_annotation/overlapping_annotations.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/onnx/data/references_custom_annotation/primary_and_backup_precision.json b/tests/onnx/data/references_custom_annotation/primary_and_backup_precision.json index 60d3061df24..e74ed5f39db 100644 --- a/tests/onnx/data/references_custom_annotation/primary_and_backup_precision.json +++ b/tests/onnx/data/references_custom_annotation/primary_and_backup_precision.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null } } diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_adjust.json b/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_adjust.json index f68f05eac1f..c798b116c5b 100644 --- a/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_adjust.json +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_adjust.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null }, "MatMul_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_ignore.json b/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_ignore.json index b0e4e802a9b..e8976e5f595 100644 --- a/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_ignore.json +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_ignore.json @@ -1,18 +1,22 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/ignored_scope.json b/tests/openvino/native/data/2026.4/references_custom_annotation/ignored_scope.json index d19414720f5..1cda8feed3e 100644 --- a/tests/openvino/native/data/2026.4/references_custom_annotation/ignored_scope.json +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/ignored_scope.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/mixed_precision.json b/tests/openvino/native/data/2026.4/references_custom_annotation/mixed_precision.json index 8c7101fd8f2..5931e0a1eff 100644 --- a/tests/openvino/native/data/2026.4/references_custom_annotation/mixed_precision.json +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/mixed_precision.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int4_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json b/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json index dfb1f734ec7..a7cd32e850a 100644 --- a/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/primary_and_backup_precision.json b/tests/openvino/native/data/2026.4/references_custom_annotation/primary_and_backup_precision.json index 60d3061df24..e74ed5f39db 100644 --- a/tests/openvino/native/data/2026.4/references_custom_annotation/primary_and_backup_precision.json +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/primary_and_backup_precision.json @@ -1,22 +1,27 @@ { "MatMul_0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "MatMul_4": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null } } diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_adjust.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_adjust.json index 2b446595a2e..6d69baf2aa8 100644 --- a/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_adjust.json +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_adjust.json @@ -1,22 +1,27 @@ { "/linear/0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/1": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null }, "/linear/2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_ignore.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_ignore.json index a0f01bc6a54..b89e9b7ec18 100644 --- a/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_ignore.json +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_ignore.json @@ -1,18 +1,22 @@ { "/linear/0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/ignored_scope.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/ignored_scope.json index dd5b2b2aceb..91cb4e0f36c 100644 --- a/tests/torch/data/function_hook/quantization/references_custom_annotation/ignored_scope.json +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/ignored_scope.json @@ -1,22 +1,27 @@ { "/linear/0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/mixed_precision.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/mixed_precision.json index b1bf72f23aa..14bac0165e0 100644 --- a/tests/torch/data/function_hook/quantization/references_custom_annotation/mixed_precision.json +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/mixed_precision.json @@ -1,22 +1,27 @@ { "/linear/0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/1": { "mode": "int4_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/2": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/3": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/4": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json index 23b9b647a02..15359265f4d 100644 --- a/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json @@ -1,22 +1,27 @@ { "/linear/0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/2": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null }, "/linear/3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/primary_and_backup_precision.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/primary_and_backup_precision.json index c25d2ae3532..c1b48809e7d 100644 --- a/tests/torch/data/function_hook/quantization/references_custom_annotation/primary_and_backup_precision.json +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/primary_and_backup_precision.json @@ -1,22 +1,27 @@ { "/linear/0": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "/linear/4": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null } } diff --git a/tests/torch/data/fx/references_custom_annotation/group_size_fallback_adjust.json b/tests/torch/data/fx/references_custom_annotation/group_size_fallback_adjust.json index b0717d74c14..f03c94e0bb8 100644 --- a/tests/torch/data/fx/references_custom_annotation/group_size_fallback_adjust.json +++ b/tests/torch/data/fx/references_custom_annotation/group_size_fallback_adjust.json @@ -1,22 +1,27 @@ { "linear": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_1": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null }, "linear_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/fx/references_custom_annotation/group_size_fallback_ignore.json b/tests/torch/data/fx/references_custom_annotation/group_size_fallback_ignore.json index eb67736e217..047cd81e4ed 100644 --- a/tests/torch/data/fx/references_custom_annotation/group_size_fallback_ignore.json +++ b/tests/torch/data/fx/references_custom_annotation/group_size_fallback_ignore.json @@ -1,18 +1,22 @@ { "linear": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/fx/references_custom_annotation/ignored_scope.json b/tests/torch/data/fx/references_custom_annotation/ignored_scope.json index dae3b7b0ba3..abe319e732e 100644 --- a/tests/torch/data/fx/references_custom_annotation/ignored_scope.json +++ b/tests/torch/data/fx/references_custom_annotation/ignored_scope.json @@ -1,22 +1,27 @@ { "linear": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/fx/references_custom_annotation/mixed_precision.json b/tests/torch/data/fx/references_custom_annotation/mixed_precision.json index 7206a76df08..6332214ee21 100644 --- a/tests/torch/data/fx/references_custom_annotation/mixed_precision.json +++ b/tests/torch/data/fx/references_custom_annotation/mixed_precision.json @@ -1,22 +1,27 @@ { "linear": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_1": { "mode": "int4_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_2": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_3": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_4": { "mode": "int8_asym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json b/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json index 7d9e8c5043c..7936d407b6e 100644 --- a/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json +++ b/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json @@ -1,22 +1,27 @@ { "linear": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_2": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null }, "linear_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_4": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null } } diff --git a/tests/torch/data/fx/references_custom_annotation/primary_and_backup_precision.json b/tests/torch/data/fx/references_custom_annotation/primary_and_backup_precision.json index 5c083176c3f..d71960e6269 100644 --- a/tests/torch/data/fx/references_custom_annotation/primary_and_backup_precision.json +++ b/tests/torch/data/fx/references_custom_annotation/primary_and_backup_precision.json @@ -1,22 +1,27 @@ { "linear": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_1": { "mode": "int8_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_2": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_3": { "mode": "int4_sym", - "group_size": -1 + "group_size": -1, + "codebook_values": null }, "linear_4": { "mode": "int4_asym", - "group_size": 4 + "group_size": 4, + "codebook_values": null } } From f2c990535347a4eade8e978547be9b11c961858b Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 16 Sep 2026 00:32:25 +0400 Subject: [PATCH 14/43] refactor --- .../weight_compression/algorithm.py | 212 +++++++++++------- 1 file changed, 127 insertions(+), 85 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 3858f23d060..2b6fc9a1290 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -1203,6 +1203,109 @@ def _get_custom_annotation_configs(self, graph: NNCFGraph) -> dict[str, WeightCo node_name_to_config[node_name] = annotation.config return node_name_to_config + def _get_applicable_annotation_configs( + self, graph: NNCFGraph, weight_params: list[WeightCompressionParameters] + ) -> dict[str, WeightCompressionConfig]: + """ + Returns the annotated configurations that can be applied to the given weight parameters. + """ + node_name_to_config = self._get_custom_annotation_configs(graph) + + unsupported_node_names = set() + for w_params in weight_params: + node_name = w_params.node_with_weight.node_name + config = node_name_to_config.get(node_name) + if config is not None and not self.is_weight_compression_supported(w_params.weight_dtype, config.mode): + unsupported_node_names.add(node_name) + if unsupported_node_names: + nncf_logger.warning( + "The compression mode defined by the custom annotation is not supported for the data type of the " + "following nodes. They are kept with the configuration assigned by the algorithm:\n\t" + + "\n\t".join(sorted(unsupported_node_names)) + ) + + weight_node_names = set(w_params.node_with_weight.node_name for w_params in weight_params) + node_names_without_weight = node_name_to_config.keys() - weight_node_names + if node_names_without_weight: + nncf_logger.warning( + "The following nodes are matched by the custom annotation, but have no weight to compress. " + "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(node_names_without_weight)) + ) + + dropped_node_names = unsupported_node_names | node_names_without_weight + return { + node_name: config + for node_name, config in node_name_to_config.items() + if node_name not in dropped_node_names + } + + @staticmethod + def _get_restored_weight_params( + node_name_to_config: dict[str, WeightCompressionConfig], + all_weight_params: list[WeightCompressionParameters], + skipped_weight_params: list[WeightCompressionParameters], + ) -> list[WeightCompressionParameters]: + """ + Returns the skipped weight parameters that have to be compressed since they are annotated. + """ + compressed_weight_names = set(w_params.weight_name for w_params in all_weight_params) + restored_weight_params = [] + for w_params in skipped_weight_params: + # A weight that is already compressed under another node, e.g. a shared weight, must not be restored + # to avoid processing the same weight more than once. + is_annotated = w_params.node_with_weight.node_name in node_name_to_config + if is_annotated and w_params.weight_name not in compressed_weight_names: + compressed_weight_names.add(w_params.weight_name) + restored_weight_params.append(w_params) + + if restored_weight_params: + nncf_logger.warning( + "The following nodes are excluded from the compression, e.g. by the ignored scope, but are matched " + "by the custom annotation. They will be compressed with the user-defined configuration:\n\t" + + "\n\t".join(sorted(w_params.node_with_weight.node_name for w_params in restored_weight_params)) + ) + return restored_weight_params + + @staticmethod + def _assign_annotation_configs( + node_name_to_config: dict[str, WeightCompressionConfig], + weight_params: list[WeightCompressionParameters], + ) -> list[WeightCompressionParameters]: + """ + Assigns the annotated configurations to the matched weight parameters and returns them. + """ + annotated_weight_params = [] + for w_params in weight_params: + config = node_name_to_config.get(w_params.node_with_weight.node_name) + if config is not None: + # A deep copy is required since some algorithms, e.g. codebook estimation, update the compression + # config of a weight parameter in place. + w_params.compression_config = copy.deepcopy(config) + annotated_weight_params.append(w_params) + return annotated_weight_params + + @staticmethod + def _restore_weight_params( + graph: NNCFGraph, + all_weight_params: list[WeightCompressionParameters], + skipped_weight_params: list[WeightCompressionParameters], + restored_weight_params: list[WeightCompressionParameters], + ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters]]: + """ + Moves the restored weight parameters from the skipped ones to the compressed ones. + """ + restored_weight_names = set(w_params.weight_name for w_params in restored_weight_params) + skipped_weight_params = [ + w_params for w_params in skipped_weight_params if w_params.weight_name not in restored_weight_names + ] + # Keep the topological order of the weight parameters + node_name_to_position = {node.node_name: i for i, node in enumerate(graph.topological_sort())} + all_weight_params = sorted( + all_weight_params + restored_weight_params, + key=lambda wp: node_name_to_position[wp.node_with_weight.node_name], + ) + return all_weight_params, skipped_weight_params + def apply_custom_annotation( self, graph: NNCFGraph, @@ -1213,112 +1316,47 @@ def apply_custom_annotation( list[WeightCompressionParameters], list[WeightCompressionParameters], list[WeightCompressionParameters], + list[WeightCompressionParameters], ]: """ - Assigns the user-defined weight compression configurations to the weight parameters matched by the - custom annotation. - - The annotated parameters are excluded from the ratio-defining ones, so that neither the mixed precision - algorithm nor the `all_layers` and the last layer rules can override the user-defined configuration. - The parameters that were skipped, e.g. by the ignored scope, are returned back to the compressed ones - if they are matched by the custom annotation. + Applies the user-defined weight compression configurations to the parameters matched by the annotation. :param graph: NNCFGraph instance. :param all_weight_params: List of all weight compression parameters that can be compressed. :param ratio_defining_params: List of ratio-defining parameters. :param skipped_weight_params: List of weight compression parameters that are not compressed. - :return: The updated tuple of all, ratio-defining and skipped weight compression parameters. + :return: The updated tuple of all, ratio-defining, skipped and annotated weight compression parameters. """ if not self._custom_annotation: - return all_weight_params, ratio_defining_params, skipped_weight_params + return all_weight_params, ratio_defining_params, skipped_weight_params, [] - node_name_to_config = self._get_custom_annotation_configs(graph) - compressed_weight_names = set(w_params.weight_name for w_params in all_weight_params) - annotated_weight_params = [] - restored_weight_params = [] - unsupported_node_names = [] - - def annotate(w_params: WeightCompressionParameters) -> bool: - config = node_name_to_config.get(w_params.node_with_weight.node_name) - if config is None: - return False - if not self.is_weight_compression_supported(w_params.weight_dtype, config.mode): - unsupported_node_names.append(w_params.node_with_weight.node_name) - return False - # A deep copy is required since some algorithms, e.g. codebook estimation, update the compression - # config of a weight parameter in place. - w_params.compression_config = copy.deepcopy(config) - annotated_weight_params.append(w_params) - return True - - for w_params in all_weight_params: - annotate(w_params) - - for w_params in skipped_weight_params: - # A weight that is already compressed under another node, e.g. a shared weight, must not be restored - # to avoid processing the same weight more than once. - if w_params.weight_name in compressed_weight_names: - continue - if annotate(w_params): - compressed_weight_names.add(w_params.weight_name) - restored_weight_params.append(w_params) - - if unsupported_node_names: - nncf_logger.warning( - "The compression mode defined by the custom annotation is not supported for the data type of the " - "following nodes. They are kept with the configuration assigned by the algorithm:\n\t" - + "\n\t".join(sorted(unsupported_node_names)) - ) - - matched_node_names = set( - w_params.node_with_weight.node_name - for w_params in all_weight_params + skipped_weight_params - if w_params.node_with_weight.node_name in node_name_to_config + node_name_to_config = self._get_applicable_annotation_configs(graph, all_weight_params + skipped_weight_params) + restored_weight_params = self._get_restored_weight_params( + node_name_to_config, all_weight_params, skipped_weight_params + ) + annotated_weight_params = self._assign_annotation_configs( + node_name_to_config, all_weight_params + restored_weight_params ) - not_applied_node_names = node_name_to_config.keys() - matched_node_names - if not_applied_node_names: - nncf_logger.warning( - "The following nodes are matched by the custom annotation, but have no weight to compress. " - "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(not_applied_node_names)) - ) - if restored_weight_params: - nncf_logger.warning( - "The following nodes are excluded from the compression, e.g. by the ignored scope, but are matched " - "by the custom annotation. They will be compressed with the user-defined configuration:\n\t" - + "\n\t".join(sorted(w_params.node_with_weight.node_name for w_params in restored_weight_params)) - ) - restored_weight_names = set(w_params.weight_name for w_params in restored_weight_params) - skipped_weight_params = [ - w_params for w_params in skipped_weight_params if w_params.weight_name not in restored_weight_names - ] - # Keep the topological order of the weight parameters - node_name_to_position = {node.node_name: i for i, node in enumerate(graph.topological_sort())} - all_weight_params = sorted( - all_weight_params + restored_weight_params, - key=lambda wp: node_name_to_position[wp.node_with_weight.node_name], + all_weight_params, skipped_weight_params = self._restore_weight_params( + graph, all_weight_params, skipped_weight_params, restored_weight_params ) - # The user-defined configuration takes precedence over the mixed precision assignment + # Update the ratio defining parameters because the user-defined configuration takes precedence over the + # mixed precision assignment annotated_weight_names = set(w_params.weight_name for w_params in annotated_weight_params) ratio_defining_params = [ w_params for w_params in ratio_defining_params if w_params.weight_name not in annotated_weight_names ] - # The group size fallback modes are applied to the annotated weights as well. The resulting group size is - # validated together with the rest of the weights in the apply() method. - all_weight_params, annotated_weight_params, skipped_weight_params = self._handle_group_size_fallback( - all_weight_params, annotated_weight_params, skipped_weight_params - ) - - return all_weight_params, ratio_defining_params, skipped_weight_params + return all_weight_params, ratio_defining_params, skipped_weight_params, annotated_weight_params def _handle_group_size_fallback( self, all_weight_params: list[WeightCompressionParameters], annotated_weight_params: list[WeightCompressionParameters], skipped_weight_params: list[WeightCompressionParameters], - ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters], list[WeightCompressionParameters]]: + ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters]]: """ Applies the configured group size fallback mode to the weights annotated by the custom annotation. @@ -1328,7 +1366,7 @@ def _handle_group_size_fallback( :param all_weight_params: List of all weight compression parameters that can be compressed. :param annotated_weight_params: List of annotated weight compression parameters. :param skipped_weight_params: List of weight compression parameters that are not compressed. - :return: The updated tuple of all, annotated and skipped weight compression parameters. + :return: The updated tuple of all and skipped weight compression parameters. """ group_sizes = self._get_requested_group_sizes(annotated_weight_params) if self._group_size_fallback_mode == GroupSizeFallbackMode.IGNORE: @@ -1350,7 +1388,7 @@ def _handle_group_size_fallback( w_params.compression_config = self._get_backup_config(w_params.weight_dtype) annotated_weight_params = valid_weight_params - return all_weight_params, annotated_weight_params, skipped_weight_params + return all_weight_params, skipped_weight_params def apply( self, @@ -1366,8 +1404,12 @@ def apply( model, graph ) # Assign the user-defined weight compression configurations - all_weight_params, ratio_defining_params, skipped_weight_params = self.apply_custom_annotation( - graph, all_weight_params, ratio_defining_params, skipped_weight_params + all_weight_params, ratio_defining_params, skipped_weight_params, annotated_weight_params = ( + self.apply_custom_annotation(graph, all_weight_params, ratio_defining_params, skipped_weight_params) + ) + # The group size fallback modes are applied to the annotated weights as well + all_weight_params, skipped_weight_params = self._handle_group_size_fallback( + all_weight_params, annotated_weight_params, skipped_weight_params ) # Collect statistics for the weights compression statistics, statistic_points = self.collect_statistics_and_statistic_points( From c9c483b3167e462fe95b72a92e9cf5d335f28180 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 16 Sep 2026 15:26:34 +0400 Subject: [PATCH 15/43] refactor ordering --- .../weight_compression/algorithm.py | 150 +++++++++--------- 1 file changed, 78 insertions(+), 72 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 2b6fc9a1290..b1b1c1812fc 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -1179,17 +1179,20 @@ def get_weight_compression_parameters( return all_weight_params, ratio_defining_params, skipped_weight_params - def _get_custom_annotation_configs(self, graph: NNCFGraph) -> dict[str, WeightCompressionConfig]: + @staticmethod + def _get_custom_annotation_configs( + custom_annotation: list[CustomAnnotation], graph: NNCFGraph + ) -> dict[str, WeightCompressionConfig]: """ Maps node names matched by the custom annotation to the user-defined weight compression configuration. - If several annotations match the same node, the last one takes precedence. + :param custom_annotation: List of user-defined custom annotations. :param graph: NNCFGraph instance. :return: A mapping from a node name to the user-defined weight compression configuration. """ node_name_to_config: dict[str, WeightCompressionConfig] = {} - for annotation in self._custom_annotation: + for annotation in custom_annotation: annotated_names = get_node_names_from_scope( annotation.scope, graph, strict=annotation.scope.validate, scope_kind="Annotated" ) @@ -1203,14 +1206,15 @@ def _get_custom_annotation_configs(self, graph: NNCFGraph) -> dict[str, WeightCo node_name_to_config[node_name] = annotation.config return node_name_to_config - def _get_applicable_annotation_configs( - self, graph: NNCFGraph, weight_params: list[WeightCompressionParameters] + def _drop_annotation_configs_that_can_not_be_assigned( + self, + node_name_to_config: dict[str, WeightCompressionConfig], + weight_params: list[WeightCompressionParameters], ) -> dict[str, WeightCompressionConfig]: """ - Returns the annotated configurations that can be applied to the given weight parameters. + Drops the annotated configurations that can not be assigned to a weight of the given parameters, i.e. the + ones matching a node without a weight or a weight with a data type that the annotated mode can not compress. """ - node_name_to_config = self._get_custom_annotation_configs(graph) - unsupported_node_names = set() for w_params in weight_params: node_name = w_params.node_with_weight.node_name @@ -1239,61 +1243,62 @@ def _get_applicable_annotation_configs( if node_name not in dropped_node_names } - @staticmethod - def _get_restored_weight_params( + def _assign_annotation_configs( + self, node_name_to_config: dict[str, WeightCompressionConfig], all_weight_params: list[WeightCompressionParameters], - skipped_weight_params: list[WeightCompressionParameters], + ratio_defining_params: list[WeightCompressionParameters], ) -> list[WeightCompressionParameters]: """ - Returns the skipped weight parameters that have to be compressed since they are annotated. - """ - compressed_weight_names = set(w_params.weight_name for w_params in all_weight_params) - restored_weight_params = [] - for w_params in skipped_weight_params: - # A weight that is already compressed under another node, e.g. a shared weight, must not be restored - # to avoid processing the same weight more than once. - is_annotated = w_params.node_with_weight.node_name in node_name_to_config - if is_annotated and w_params.weight_name not in compressed_weight_names: - compressed_weight_names.add(w_params.weight_name) - restored_weight_params.append(w_params) + Assigns the annotated configurations to the matched weight parameters. - if restored_weight_params: - nncf_logger.warning( - "The following nodes are excluded from the compression, e.g. by the ignored scope, but are matched " - "by the custom annotation. They will be compressed with the user-defined configuration:\n\t" - + "\n\t".join(sorted(w_params.node_with_weight.node_name for w_params in restored_weight_params)) - ) - return restored_weight_params - - @staticmethod - def _assign_annotation_configs( - node_name_to_config: dict[str, WeightCompressionConfig], - weight_params: list[WeightCompressionParameters], - ) -> list[WeightCompressionParameters]: - """ - Assigns the annotated configurations to the matched weight parameters and returns them. + :param node_name_to_config: A mapping from a node name to the annotated compression configuration. + :param all_weight_params: List of all weight compression parameters that can be compressed. + :param ratio_defining_params: List of ratio-defining parameters. + :return: The updated list of ratio-defining parameters. """ - annotated_weight_params = [] - for w_params in weight_params: + annotated_weight_names = set() + for w_params in all_weight_params: config = node_name_to_config.get(w_params.node_with_weight.node_name) if config is not None: # A deep copy is required since some algorithms, e.g. codebook estimation, update the compression # config of a weight parameter in place. w_params.compression_config = copy.deepcopy(config) - annotated_weight_params.append(w_params) - return annotated_weight_params + annotated_weight_names.add(w_params.weight_name) + + # The annotated parameters are excluded from the ratio-defining ones, so that the mixed precision + # algorithm, the `all_layers` and the last layer rules can not override the annotated configuration + return [w_params for w_params in ratio_defining_params if w_params.weight_name not in annotated_weight_names] @staticmethod - def _restore_weight_params( + def _move_annotated_ignored_scope_params_from_skipped_to_compressed( + node_name_to_config: dict[str, WeightCompressionConfig], graph: NNCFGraph, all_weight_params: list[WeightCompressionParameters], skipped_weight_params: list[WeightCompressionParameters], - restored_weight_params: list[WeightCompressionParameters], ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters]]: """ - Moves the restored weight parameters from the skipped ones to the compressed ones. + Moves the annotated weight parameters that are not compressed, e.g. the ignored ones, from the skipped + parameters to the compressed ones, so that the annotation takes precedence over the ignored scope. """ + compressed_weight_names = set(w_params.weight_name for w_params in all_weight_params) + restored_weight_params = [] + for w_params in skipped_weight_params: + # A weight that is already compressed under another node, e.g. a shared weight, must not be restored + # to avoid processing the same weight more than once. + is_annotated = w_params.node_with_weight.node_name in node_name_to_config + if is_annotated and w_params.weight_name not in compressed_weight_names: + compressed_weight_names.add(w_params.weight_name) + restored_weight_params.append(w_params) + + if not restored_weight_params: + return all_weight_params, skipped_weight_params + + nncf_logger.warning( + "The following nodes are excluded from the compression, e.g. by the ignored scope, but are matched " + "by the custom annotation. They will be compressed with the user-defined configuration:\n\t" + + "\n\t".join(sorted(w_params.node_with_weight.node_name for w_params in restored_weight_params)) + ) restored_weight_names = set(w_params.weight_name for w_params in restored_weight_params) skipped_weight_params = [ w_params for w_params in skipped_weight_params if w_params.weight_name not in restored_weight_names @@ -1316,45 +1321,47 @@ def apply_custom_annotation( list[WeightCompressionParameters], list[WeightCompressionParameters], list[WeightCompressionParameters], - list[WeightCompressionParameters], ]: """ Applies the user-defined weight compression configurations to the parameters matched by the annotation. + The flow of this function is as follows: + 1. Convert the annotation to a mapping from a node name to the configuration, the last match wins. + 2. Drop the annotated configurations that can not be assigned to a weight. + 3. Move the annotated weight parameters that are not compressed, e.g. the ignored ones, to the + compressed ones. + 4. Assign the remaining configurations to the matched weight parameters. + 5. Apply the group size fallback mode to the annotated group size values. :param graph: NNCFGraph instance. :param all_weight_params: List of all weight compression parameters that can be compressed. :param ratio_defining_params: List of ratio-defining parameters. :param skipped_weight_params: List of weight compression parameters that are not compressed. - :return: The updated tuple of all, ratio-defining, skipped and annotated weight compression parameters. + :return: The updated tuple of all, ratio-defining and skipped weight compression parameters. """ if not self._custom_annotation: - return all_weight_params, ratio_defining_params, skipped_weight_params, [] + return all_weight_params, ratio_defining_params, skipped_weight_params - node_name_to_config = self._get_applicable_annotation_configs(graph, all_weight_params + skipped_weight_params) - restored_weight_params = self._get_restored_weight_params( - node_name_to_config, all_weight_params, skipped_weight_params + node_name_to_config = self._get_custom_annotation_configs(self._custom_annotation, graph) + node_name_to_config = self._drop_annotation_configs_that_can_not_be_assigned( + node_name_to_config, all_weight_params + skipped_weight_params ) - annotated_weight_params = self._assign_annotation_configs( - node_name_to_config, all_weight_params + restored_weight_params + all_weight_params, skipped_weight_params = self._move_annotated_ignored_scope_params_from_skipped_to_compressed( + node_name_to_config, graph, all_weight_params, skipped_weight_params + ) + ratio_defining_params = self._assign_annotation_configs( + node_name_to_config, all_weight_params, ratio_defining_params + ) + # The resulting group size values are validated together with the rest of the weights in the apply() method + all_weight_params, skipped_weight_params = self._handle_group_size_fallback( + node_name_to_config, all_weight_params, skipped_weight_params ) - if restored_weight_params: - all_weight_params, skipped_weight_params = self._restore_weight_params( - graph, all_weight_params, skipped_weight_params, restored_weight_params - ) - - # Update the ratio defining parameters because the user-defined configuration takes precedence over the - # mixed precision assignment - annotated_weight_names = set(w_params.weight_name for w_params in annotated_weight_params) - ratio_defining_params = [ - w_params for w_params in ratio_defining_params if w_params.weight_name not in annotated_weight_names - ] - return all_weight_params, ratio_defining_params, skipped_weight_params, annotated_weight_params + return all_weight_params, ratio_defining_params, skipped_weight_params def _handle_group_size_fallback( self, + node_name_to_config: dict[str, WeightCompressionConfig], all_weight_params: list[WeightCompressionParameters], - annotated_weight_params: list[WeightCompressionParameters], skipped_weight_params: list[WeightCompressionParameters], ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters]]: """ @@ -1363,11 +1370,14 @@ def _handle_group_size_fallback( The group size of an annotated weight is defined by the user, so the fallback mode is applied to that value instead of the group size configured for the algorithm. + :param node_name_to_config: A mapping from a node name to the applied annotation configuration. :param all_weight_params: List of all weight compression parameters that can be compressed. - :param annotated_weight_params: List of annotated weight compression parameters. :param skipped_weight_params: List of weight compression parameters that are not compressed. :return: The updated tuple of all and skipped weight compression parameters. """ + annotated_weight_params = [ + w_params for w_params in all_weight_params if w_params.node_with_weight.node_name in node_name_to_config + ] group_sizes = self._get_requested_group_sizes(annotated_weight_params) if self._group_size_fallback_mode == GroupSizeFallbackMode.IGNORE: all_weight_params, annotated_weight_params, skipped_weight_params = self._handle_ignore_group_size_fallback( @@ -1404,12 +1414,8 @@ def apply( model, graph ) # Assign the user-defined weight compression configurations - all_weight_params, ratio_defining_params, skipped_weight_params, annotated_weight_params = ( - self.apply_custom_annotation(graph, all_weight_params, ratio_defining_params, skipped_weight_params) - ) - # The group size fallback modes are applied to the annotated weights as well - all_weight_params, skipped_weight_params = self._handle_group_size_fallback( - all_weight_params, annotated_weight_params, skipped_weight_params + all_weight_params, ratio_defining_params, skipped_weight_params = self.apply_custom_annotation( + graph, all_weight_params, ratio_defining_params, skipped_weight_params ) # Collect statistics for the weights compression statistics, statistic_points = self.collect_statistics_and_statistic_points( From 47fdcd5e5ef592524db7efd690596b6fc13780c8 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 16 Sep 2026 18:26:39 +0400 Subject: [PATCH 16/43] ref --- src/nncf/quantization/advanced_parameters.py | 2 + .../weight_compression/algorithm.py | 51 ++++++++++++++----- src/nncf/quantization/quantize_model.py | 5 +- src/nncf/scopes.py | 43 +++++++++++----- 4 files changed, 73 insertions(+), 28 deletions(-) diff --git a/src/nncf/quantization/advanced_parameters.py b/src/nncf/quantization/advanced_parameters.py index 67d9486dced..5c35ea07a46 100644 --- a/src/nncf/quantization/advanced_parameters.py +++ b/src/nncf/quantization/advanced_parameters.py @@ -463,6 +463,8 @@ class CustomAnnotation: matched nodes, including the assignment made by the mixed precision algorithm and the `ignored_scope`, `all_layers` and `backup_mode` options. + If several annotations match the same node, the last one in the `custom_annotation` list takes precedence. + Example: .. code-block:: python diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index b1b1c1812fc..3d654d796eb 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -53,6 +53,7 @@ from nncf.quantization.algorithms.weight_compression.weight_lowering import get_reduction_channel_size from nncf.scopes import CustomAnnotationScope from nncf.scopes import IgnoredScope +from nncf.scopes import ScopeType from nncf.scopes import get_node_names_from_scope from nncf.tensor import Tensor from nncf.tensor import functions as fns @@ -428,7 +429,8 @@ def __init__( :param advanced_parameters: advanced parameters for algorithms in compression pipeline. :param custom_annotation: A list of user-defined weight compression configurations bound to portions of the model. The configuration of a custom annotation takes precedence over the one assigned by the - algorithm for all the matched nodes. + algorithm for all the matched nodes. If several annotations match the same node, the last one in the + list takes precedence. """ super().__init__() self._mode = mode @@ -1192,15 +1194,18 @@ def _get_custom_annotation_configs( :return: A mapping from a node name to the user-defined weight compression configuration. """ node_name_to_config: dict[str, WeightCompressionConfig] = {} - for annotation in custom_annotation: + for index, annotation in enumerate(custom_annotation): annotated_names = get_node_names_from_scope( - annotation.scope, graph, strict=annotation.scope.validate, scope_kind="Annotated" + annotation.scope, graph, strict=annotation.scope.validate, scope_type=ScopeType.ANNOTATED ) overlapped_names = sorted(annotated_names & node_name_to_config.keys()) if overlapped_names: + config = annotation.config + group_size = "per-channel" if config.group_size == -1 else f"group size {config.group_size}" nncf_logger.warning( - "Several custom annotations match the same nodes. The last matched annotation takes " - "precedence for the following nodes:\n\t" + "\n\t".join(overlapped_names) + f"Several custom annotations match the same nodes. The configuration of the custom annotation " + f"with index {index} ({config.mode.value}, {group_size}) takes precedence, since it is the last " + "matched one, and is used for the following nodes:\n\t" + "\n\t".join(overlapped_names) ) for node_name in annotated_names: node_name_to_config[node_name] = annotation.config @@ -1209,14 +1214,17 @@ def _get_custom_annotation_configs( def _drop_annotation_configs_that_can_not_be_assigned( self, node_name_to_config: dict[str, WeightCompressionConfig], - weight_params: list[WeightCompressionParameters], + graph: NNCFGraph, + all_weight_params: list[WeightCompressionParameters], + skipped_weight_params: list[WeightCompressionParameters], ) -> dict[str, WeightCompressionConfig]: """ - Drops the annotated configurations that can not be assigned to a weight of the given parameters, i.e. the - ones matching a node without a weight or a weight with a data type that the annotated mode can not compress. + Drops the annotated configurations that can not be assigned to a weight, i.e. the ones matching a node + without a weight, a weight with a data type that the annotated mode can not compress, or a weight that is + compressed under another node. """ unsupported_node_names = set() - for w_params in weight_params: + for w_params in all_weight_params + skipped_weight_params: node_name = w_params.node_with_weight.node_name config = node_name_to_config.get(node_name) if config is not None and not self.is_weight_compression_supported(w_params.weight_dtype, config.mode): @@ -1228,15 +1236,32 @@ def _drop_annotation_configs_that_can_not_be_assigned( + "\n\t".join(sorted(unsupported_node_names)) ) - weight_node_names = set(w_params.node_with_weight.node_name for w_params in weight_params) - node_names_without_weight = node_name_to_config.keys() - weight_node_names + node_names_with_weight = set(node.node_name for node in self.get_nodes_to_compress(graph)) + node_names_without_weight = node_name_to_config.keys() - node_names_with_weight if node_names_without_weight: nncf_logger.warning( "The following nodes are matched by the custom annotation, but have no weight to compress. " "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(node_names_without_weight)) ) - dropped_node_names = unsupported_node_names | node_names_without_weight + # A weight is compressed under a single node, so the annotation of the other nodes that share this weight, + # e.g. the ones matched by the annotation only, can not be assigned to it. + compressed_weight_names = set(w_params.weight_name for w_params in all_weight_params) + compressed_node_names = set(w_params.node_with_weight.node_name for w_params in all_weight_params) + shared_weight_node_names = set() + for node_name in (node_name_to_config.keys() & node_names_with_weight) - compressed_node_names: + node = graph.get_node_by_name(node_name) + weight_names = (name for name, _ in self._backend_entity.get_weight_names_and_port_ids(node, graph)) + if any(weight_name in compressed_weight_names for weight_name in weight_names): + shared_weight_node_names.add(node_name) + if shared_weight_node_names: + nncf_logger.warning( + "The weight of the following nodes matched by the custom annotation is compressed under another " + "node, e.g. a shared weight. The annotation has no effect for them:\n\t" + + "\n\t".join(sorted(shared_weight_node_names)) + ) + + dropped_node_names = unsupported_node_names | node_names_without_weight | shared_weight_node_names return { node_name: config for node_name, config in node_name_to_config.items() @@ -1343,7 +1368,7 @@ def apply_custom_annotation( node_name_to_config = self._get_custom_annotation_configs(self._custom_annotation, graph) node_name_to_config = self._drop_annotation_configs_that_can_not_be_assigned( - node_name_to_config, all_weight_params + skipped_weight_params + node_name_to_config, graph, all_weight_params, skipped_weight_params ) all_weight_params, skipped_weight_params = self._move_annotated_ignored_scope_params_from_skipped_to_compressed( node_name_to_config, graph, all_weight_params, skipped_weight_params diff --git a/src/nncf/quantization/quantize_model.py b/src/nncf/quantization/quantize_model.py index 568ff9e39b4..af2fe483fa2 100644 --- a/src/nncf/quantization/quantize_model.py +++ b/src/nncf/quantization/quantize_model.py @@ -499,7 +499,10 @@ def compress_weights( algorithm, which makes it possible to compress certain layers, e.g. attention or MoE router layers, to a precision different from the one defined by `mode`, `ratio` and `backup_mode`. The custom annotation takes precedence over the mixed precision assignment, the `ignored_scope` and the `all_layers` options. - If several annotations match the same node, the last one takes precedence. + If several annotations match the same node, the last one in the list takes precedence. For example, an + annotation of a whole block followed by an annotation of a single layer of this block compresses the block + with the first configuration, except for this layer, while the opposite order compresses the whole block + with the second configuration. :type custom_annotation: list[nncf.CustomAnnotation] :return: The non-trainable model with compressed weights. """ diff --git a/src/nncf/scopes.py b/src/nncf/scopes.py index cda80dbb9c7..5f51c8922b1 100644 --- a/src/nncf/scopes.py +++ b/src/nncf/scopes.py @@ -17,6 +17,7 @@ from nncf.common.graph.graph import NNCFGraph from nncf.common.logging import nncf_logger from nncf.common.utils.api_marker import api +from nncf.parameters import StrEnum @api(canonical_alias="nncf.Subgraph") @@ -256,24 +257,35 @@ def get_matched_scope_info(scope: BaseScope, nncf_graphs: list[NNCFGraph]) -> tu return matched_scope, matches -def _info_matched_scope(matches: dict[str, set[str]], scope_kind: str = "Ignored") -> None: +class ScopeType(StrEnum): + """ + Type of a scope, used in the messages about the matched and unmatched nodes. + """ + + IGNORED = "Ignored" + ANNOTATED = "Annotated" + + +def _info_matched_scope(matches: dict[str, set[str]], scope_type: ScopeType = ScopeType.IGNORED) -> None: """ Log matches. :param matches: Matches. - :param scope_kind: Human-readable kind of the scope to mention in the message. + :param scope_type: Type of the scope to mention in the message. """ for rule_type, rules in matches.items(): if rules: - nncf_logger.info(f"{len(rules)} {scope_kind.lower()} nodes were found by {rule_type} in the NNCFGraph") + nncf_logger.info( + f"{len(rules)} {scope_type.value.lower()} nodes were found by {rule_type} in the NNCFGraph" + ) -def _error_unmatched_scope(unmatched_scope: BaseScope, scope_kind: str = "Ignored") -> str: +def _error_unmatched_scope(unmatched_scope: BaseScope, scope_type: ScopeType = ScopeType.IGNORED) -> str: """ Returns an error message for unmatched scope. :param unmatched_scope: Unmatched scope. - :param scope_kind: Human-readable kind of the scope to mention in the message. + :param scope_type: Type of the scope to mention in the message. :return str: Error message. """ err_msg = "\n" @@ -281,31 +293,34 @@ def _error_unmatched_scope(unmatched_scope: BaseScope, scope_kind: str = "Ignore unmatched_rules = getattr(unmatched_scope, rule_type) if unmatched_rules: err_msg += ( - f"{scope_kind} nodes that matches {rule_type} {unmatched_rules} were not found in the NNCFGraph.\n" + f"{scope_type.value} nodes that matches {rule_type} {unmatched_rules} " + "were not found in the NNCFGraph.\n" ) for subgraph in unmatched_scope.subgraphs: err_msg += ( - f"{scope_kind} nodes that matches subgraph with input names {subgraph.inputs} " + f"{scope_type.value} nodes that matches subgraph with input names {subgraph.inputs} " f"and output names {subgraph.outputs} were not found in the NNCFGraph.\n" ) return err_msg -def _check_scope_strictly_matched(scope: BaseScope, matched_scope: BaseScope, scope_kind: str = "Ignored") -> None: +def _check_scope_strictly_matched( + scope: BaseScope, matched_scope: BaseScope, scope_type: ScopeType = ScopeType.IGNORED +) -> None: """ Passes when scope and matched_scope are equal, otherwise - raises ValidationError. :param scope: Scope. :param matched_scope: Matched scope. - :param scope_kind: Human-readable kind of the scope to mention in the error message. + :param scope_type: Type of the scope to mention in the error message. """ unmatched_scope = get_difference_scope(scope, matched_scope) if unmatched_scope.names or unmatched_scope.types or unmatched_scope.patterns or unmatched_scope.subgraphs: - raise nncf.ValidationError(_error_unmatched_scope(unmatched_scope, scope_kind)) + raise nncf.ValidationError(_error_unmatched_scope(unmatched_scope, scope_type)) def get_node_names_from_scope( - scope: BaseScope, nncf_graph: NNCFGraph, strict: bool = True, scope_kind: str = "Ignored" + scope: BaseScope, nncf_graph: NNCFGraph, strict: bool = True, scope_type: ScopeType = ScopeType.IGNORED ) -> set[str]: """ Returns matched names according to the scope and NNCFGraph. @@ -315,13 +330,13 @@ def get_node_names_from_scope( :param scope: Scope. :param nncf_graph: Graph. :param strict: Whether all scope rules must match at least one node or not. - :param scope_kind: Human-readable kind of the scope to mention in the error message. + :param scope_type: Type of the scope to mention in the error message. :return: NNCF node names from given NNCFGraph specified in given scope. """ matched_scope, matches = get_matched_scope_info(scope, [nncf_graph]) if strict: - _check_scope_strictly_matched(scope, matched_scope, scope_kind) - _info_matched_scope(matches, scope_kind) + _check_scope_strictly_matched(scope, matched_scope, scope_type) + _info_matched_scope(matches, scope_type) return {name for match in matches.values() for name in match} From a9545cc524ac6b3c6d4c6f96b020e30ceb0acd31 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Fri, 18 Sep 2026 14:13:23 +0400 Subject: [PATCH 17/43] remove old scope kind --- src/nncf/scopes.py | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/nncf/scopes.py b/src/nncf/scopes.py index 5f51c8922b1..6b5c5b315ea 100644 --- a/src/nncf/scopes.py +++ b/src/nncf/scopes.py @@ -17,7 +17,6 @@ from nncf.common.graph.graph import NNCFGraph from nncf.common.logging import nncf_logger from nncf.common.utils.api_marker import api -from nncf.parameters import StrEnum @api(canonical_alias="nncf.Subgraph") @@ -257,71 +256,65 @@ def get_matched_scope_info(scope: BaseScope, nncf_graphs: list[NNCFGraph]) -> tu return matched_scope, matches -class ScopeType(StrEnum): - """ - Type of a scope, used in the messages about the matched and unmatched nodes. +def _get_scope_name(scope: BaseScope) -> str: """ + Returns the name of the given scope, used in the messages about the matched and unmatched nodes. - IGNORED = "Ignored" - ANNOTATED = "Annotated" + :param scope: Scope. + :return: Name of the scope. + """ + return "Annotated" if isinstance(scope, CustomAnnotationScope) else "Ignored" -def _info_matched_scope(matches: dict[str, set[str]], scope_type: ScopeType = ScopeType.IGNORED) -> None: +def _info_matched_scope(matches: dict[str, set[str]], scope: BaseScope) -> None: """ Log matches. :param matches: Matches. - :param scope_type: Type of the scope to mention in the message. + :param scope: Scope the matches are found by. """ + scope_name = _get_scope_name(scope) for rule_type, rules in matches.items(): if rules: - nncf_logger.info( - f"{len(rules)} {scope_type.value.lower()} nodes were found by {rule_type} in the NNCFGraph" - ) + nncf_logger.info(f"{len(rules)} {scope_name} nodes were found by {rule_type} in the NNCFGraph") -def _error_unmatched_scope(unmatched_scope: BaseScope, scope_type: ScopeType = ScopeType.IGNORED) -> str: +def _error_unmatched_scope(unmatched_scope: BaseScope) -> str: """ Returns an error message for unmatched scope. :param unmatched_scope: Unmatched scope. - :param scope_type: Type of the scope to mention in the message. :return str: Error message. """ + scope_name = _get_scope_name(unmatched_scope) err_msg = "\n" for rule_type in ("names", "types", "patterns"): unmatched_rules = getattr(unmatched_scope, rule_type) if unmatched_rules: err_msg += ( - f"{scope_type.value} nodes that matches {rule_type} {unmatched_rules} " - "were not found in the NNCFGraph.\n" + f"{scope_name} nodes that matches {rule_type} {unmatched_rules} were not found in the NNCFGraph.\n" ) for subgraph in unmatched_scope.subgraphs: err_msg += ( - f"{scope_type.value} nodes that matches subgraph with input names {subgraph.inputs} " + f"{scope_name} nodes that matches subgraph with input names {subgraph.inputs} " f"and output names {subgraph.outputs} were not found in the NNCFGraph.\n" ) return err_msg -def _check_scope_strictly_matched( - scope: BaseScope, matched_scope: BaseScope, scope_type: ScopeType = ScopeType.IGNORED -) -> None: +def _check_scope_strictly_matched(scope: BaseScope, matched_scope: BaseScope) -> None: """ Passes when scope and matched_scope are equal, otherwise - raises ValidationError. :param scope: Scope. :param matched_scope: Matched scope. - :param scope_type: Type of the scope to mention in the error message. """ unmatched_scope = get_difference_scope(scope, matched_scope) if unmatched_scope.names or unmatched_scope.types or unmatched_scope.patterns or unmatched_scope.subgraphs: - raise nncf.ValidationError(_error_unmatched_scope(unmatched_scope, scope_type)) + raise nncf.ValidationError(_error_unmatched_scope(unmatched_scope)) -def get_node_names_from_scope( - scope: BaseScope, nncf_graph: NNCFGraph, strict: bool = True, scope_type: ScopeType = ScopeType.IGNORED -) -> set[str]: +def get_node_names_from_scope(scope: BaseScope, nncf_graph: NNCFGraph, strict: bool = True) -> set[str]: """ Returns matched names according to the scope and NNCFGraph. If strict is True, raises nncf.ValidationError if any rule was not matched. @@ -330,13 +323,12 @@ def get_node_names_from_scope( :param scope: Scope. :param nncf_graph: Graph. :param strict: Whether all scope rules must match at least one node or not. - :param scope_type: Type of the scope to mention in the error message. :return: NNCF node names from given NNCFGraph specified in given scope. """ matched_scope, matches = get_matched_scope_info(scope, [nncf_graph]) if strict: - _check_scope_strictly_matched(scope, matched_scope, scope_type) - _info_matched_scope(matches, scope_type) + _check_scope_strictly_matched(scope, matched_scope) + _info_matched_scope(matches, scope) return {name for match in matches.values() for name in match} From 1f3b3273a232efdb9fc93375da06dc7d79a730c5 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Fri, 18 Sep 2026 14:15:00 +0400 Subject: [PATCH 18/43] add shared weight edge case --- .../weight_compression/algorithm.py | 15 ++---- .../template_test_weights_compression.py | 46 +++++++++++++++++-- .../shared_weight.json | 7 +++ .../shared_weight_with_ignored_scope.json | 7 +++ .../quantization/test_weights_compression.py | 22 +++++++++ .../shared_weight.json | 17 +++++++ .../shared_weight_with_ignored_scope.json | 17 +++++++ .../quantization/test_weights_compression.py | 4 ++ .../shared_weight.json | 12 +++++ .../shared_weight_with_ignored_scope.json | 12 +++++ .../shared_weight.json | 12 +++++ .../shared_weight_with_ignored_scope.json | 12 +++++ .../quantization/test_weights_compression.py | 5 ++ tests/torch/fx/test_weights_compression.py | 5 ++ 14 files changed, 180 insertions(+), 13 deletions(-) create mode 100644 tests/onnx/data/references_custom_annotation/shared_weight.json create mode 100644 tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_scope.json create mode 100644 tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight.json create mode 100644 tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_with_ignored_scope.json create mode 100644 tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight.json create mode 100644 tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_scope.json create mode 100644 tests/torch/data/fx/references_custom_annotation/shared_weight.json create mode 100644 tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_scope.json diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 3d654d796eb..8caeb325296 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -53,7 +53,6 @@ from nncf.quantization.algorithms.weight_compression.weight_lowering import get_reduction_channel_size from nncf.scopes import CustomAnnotationScope from nncf.scopes import IgnoredScope -from nncf.scopes import ScopeType from nncf.scopes import get_node_names_from_scope from nncf.tensor import Tensor from nncf.tensor import functions as fns @@ -1195,9 +1194,7 @@ def _get_custom_annotation_configs( """ node_name_to_config: dict[str, WeightCompressionConfig] = {} for index, annotation in enumerate(custom_annotation): - annotated_names = get_node_names_from_scope( - annotation.scope, graph, strict=annotation.scope.validate, scope_type=ScopeType.ANNOTATED - ) + annotated_names = get_node_names_from_scope(annotation.scope, graph, strict=annotation.scope.validate) overlapped_names = sorted(annotated_names & node_name_to_config.keys()) if overlapped_names: config = annotation.config @@ -1306,14 +1303,13 @@ def _move_annotated_ignored_scope_params_from_skipped_to_compressed( Moves the annotated weight parameters that are not compressed, e.g. the ignored ones, from the skipped parameters to the compressed ones, so that the annotation takes precedence over the ignored scope. """ - compressed_weight_names = set(w_params.weight_name for w_params in all_weight_params) + restored_weight_names = set() restored_weight_params = [] for w_params in skipped_weight_params: - # A weight that is already compressed under another node, e.g. a shared weight, must not be restored - # to avoid processing the same weight more than once. + # A weight is compressed under a single node, so a shared weight is restored only once is_annotated = w_params.node_with_weight.node_name in node_name_to_config - if is_annotated and w_params.weight_name not in compressed_weight_names: - compressed_weight_names.add(w_params.weight_name) + if is_annotated and w_params.weight_name not in restored_weight_names: + restored_weight_names.add(w_params.weight_name) restored_weight_params.append(w_params) if not restored_weight_params: @@ -1324,7 +1320,6 @@ def _move_annotated_ignored_scope_params_from_skipped_to_compressed( "by the custom annotation. They will be compressed with the user-defined configuration:\n\t" + "\n\t".join(sorted(w_params.node_with_weight.node_name for w_params in restored_weight_params)) ) - restored_weight_names = set(w_params.weight_name for w_params in restored_weight_params) skipped_weight_params = [ w_params for w_params in skipped_weight_params if w_params.weight_name not in restored_weight_names ] diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 5c50e867376..5f69120acf7 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -976,6 +976,12 @@ def get_custom_annotation_ref_path(ref_name: str) -> Path: Returns the path to the reference compression configs of the given custom annotation test case. """ + @abstractmethod + def get_shared_weight_model(self) -> TModel: + """ + Returns a backend model where two nodes share the same weight, ready to be compressed. + """ + def _get_sequential_matmul_model(self): """Returns a backend model with a sequence of MatMul layers, ready to be compressed.""" return self.wrap_model( @@ -1005,9 +1011,10 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa # The weight nodes of the sequential MatMul model are named "MatMul_" / "linear_" / "/linear/" # depending on the backend, so they are referred to by a pattern that matches the node index. @pytest.mark.parametrize( - ("kwargs", "annotations", "ignored_scope"), + ("model_name", "kwargs", "annotations", "ignored_scope"), [ pytest.param( + "sequential_matmul", dict(ratio=1.0, all_layers=False), [ ( @@ -1023,6 +1030,7 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa id="primary_and_backup_precision", ), pytest.param( + "sequential_matmul", dict(ratio=0.5, all_layers=True), [ ( @@ -1034,6 +1042,7 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa id="mixed_precision", ), pytest.param( + "sequential_matmul", dict(ratio=1.0, all_layers=True), [ ( @@ -1045,6 +1054,7 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa id="ignored_scope", ), pytest.param( + "sequential_matmul", dict(ratio=1.0, all_layers=True), [ ( @@ -1060,6 +1070,7 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa id="overlapping_annotations", ), pytest.param( + "sequential_matmul", dict( ratio=1.0, all_layers=True, @@ -1078,6 +1089,7 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa id="group_size_fallback_adjust", ), pytest.param( + "sequential_matmul", dict( ratio=1.0, all_layers=True, @@ -1094,17 +1106,45 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa None, id="group_size_fallback_ignore", ), + pytest.param( + "shared_weight", + dict(ratio=1.0, all_layers=True), + [ + ( + nncf.CustomAnnotationScope(patterns=[".*"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + ], + None, + id="shared_weight", + ), + pytest.param( + "shared_weight", + dict(ratio=1.0, all_layers=True), + [ + ( + nncf.CustomAnnotationScope(patterns=[".*"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + ], + IgnoredScope(patterns=[".*"]), + id="shared_weight_with_ignored_scope", + ), ], ) - def test_custom_annotation(self, kwargs, annotations, ignored_scope, request): + def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, request): """ Compares the compression config assigned to each weight node with the reference one when the given custom annotation is applied. A node that is not compressed has no config in the reference file. Set the NNCF_TEST_REGEN_DOT environment variable to regenerate the reference file. """ + models = { + "sequential_matmul": self._get_sequential_matmul_model, + "shared_weight": self.get_shared_weight_model, + } configs = self._compress_and_get_configs( - model=self._get_sequential_matmul_model(), + model=models[model_name](), mode=CompressWeightsMode.INT4_SYM, group_size=-1, ignored_scope=ignored_scope, diff --git a/tests/onnx/data/references_custom_annotation/shared_weight.json b/tests/onnx/data/references_custom_annotation/shared_weight.json new file mode 100644 index 00000000000..e16fb2f3d37 --- /dev/null +++ b/tests/onnx/data/references_custom_annotation/shared_weight.json @@ -0,0 +1,7 @@ +{ + "MatMul_0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_scope.json b/tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_scope.json new file mode 100644 index 00000000000..e16fb2f3d37 --- /dev/null +++ b/tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_scope.json @@ -0,0 +1,7 @@ +{ + "MatMul_0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/onnx/quantization/test_weights_compression.py b/tests/onnx/quantization/test_weights_compression.py index 9ac60351fbb..f3b06ed22c4 100644 --- a/tests/onnx/quantization/test_weights_compression.py +++ b/tests/onnx/quantization/test_weights_compression.py @@ -727,6 +727,28 @@ def get_ignored_scope_name(is_3d_weights) -> str: def get_awq_scale_ref_path() -> Path: return REFERENCE_SCALES_DIR / "awq_scale_ref.json" + @staticmethod + def get_shared_weight_model() -> onnx.ModelProto: + """ + Builds a model with two MatMul nodes that share the same weight. + """ + weight = np.arange(0, 16).reshape(4, 4).astype(np.float32) + initializer = onnx.numpy_helper.from_array(weight, name="shared_weight") + nodes = [ + onnx.helper.make_node("MatMul", ["input", "shared_weight"], ["MatMul_0_output"], name="MatMul_0"), + onnx.helper.make_node("MatMul", ["MatMul_0_output", "shared_weight"], ["output"], name="MatMul_1"), + ] + graph = onnx.helper.make_graph( + nodes, + "shared-weight-graph", + inputs=[onnx.helper.make_tensor_value_info("input", onnx.TensorProto.FLOAT, (1, 4, 4))], + outputs=[onnx.helper.make_tensor_value_info("output", onnx.TensorProto.FLOAT, (1, 4, 4))], + initializer=[initializer], + ) + model = onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_operatorsetid("", 21)]) + onnx.checker.check_model(model) + return model + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight.json b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight.json new file mode 100644 index 00000000000..9af92af8a7a --- /dev/null +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight.json @@ -0,0 +1,17 @@ +{ + "Gather_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "Gather_2": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_with_ignored_scope.json b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_with_ignored_scope.json new file mode 100644 index 00000000000..9af92af8a7a --- /dev/null +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_with_ignored_scope.json @@ -0,0 +1,17 @@ +{ + "Gather_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "Gather_2": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 6b10de733fb..f752aab187f 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -2547,6 +2547,10 @@ def get_num_multiply_from_awq(model): def get_awq_scale_ref_path() -> Path: return get_actual_reference_for_current_openvino(REFERENCE_SCALES_DIR / "awq_scale_ref.json") + @staticmethod + def get_shared_weight_model() -> ov.Model: + return GatherAndMatmulShareData().ov_model + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return get_actual_reference_for_current_openvino(CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json") diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight.json new file mode 100644 index 00000000000..b1af06693ed --- /dev/null +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight.json @@ -0,0 +1,12 @@ +{ + "wte/embedding/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "linear/linear/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_scope.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_scope.json new file mode 100644 index 00000000000..b1af06693ed --- /dev/null +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_scope.json @@ -0,0 +1,12 @@ +{ + "wte/embedding/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "linear/linear/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight.json b/tests/torch/data/fx/references_custom_annotation/shared_weight.json new file mode 100644 index 00000000000..e191f7b1803 --- /dev/null +++ b/tests/torch/data/fx/references_custom_annotation/shared_weight.json @@ -0,0 +1,12 @@ +{ + "embedding": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "linear": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_scope.json b/tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_scope.json new file mode 100644 index 00000000000..e191f7b1803 --- /dev/null +++ b/tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_scope.json @@ -0,0 +1,12 @@ +{ + "embedding": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "linear": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index fb747262cb6..e4a755f102a 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -726,6 +726,11 @@ def get_num_multiply_from_awq(model): def get_awq_scale_ref_path() -> Path: return REFERENCE_SCALES_DIR / "awq_scale_ref.json" + @staticmethod + def get_shared_weight_model() -> GraphModelWrapper: + model = wrap_model(ShortTransformer(8, 16, share_weights=True)) + return GraphModelWrapper(model, example_input=torch.randint(0, 10, (8,))) + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" diff --git a/tests/torch/fx/test_weights_compression.py b/tests/torch/fx/test_weights_compression.py index 650279833c3..480588eb306 100644 --- a/tests/torch/fx/test_weights_compression.py +++ b/tests/torch/fx/test_weights_compression.py @@ -514,6 +514,11 @@ def get_num_multiply_from_awq(model): def get_awq_scale_ref_path() -> Path: return REFERENCE_SCALES_DIR / "awq_scale_ref.json" + @staticmethod + def get_shared_weight_model() -> torch.fx.GraphModule: + model = ShortTransformer(8, 16, share_weights=True) + return get_torch_fx_model(model, torch.randint(0, 10, (8,))) + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" From 85719b53318654be76ea217eae699730bd88a648 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 22 Sep 2026 11:50:56 +0400 Subject: [PATCH 19/43] fix --- src/nncf/openvino/rt_info.py | 5 +++-- src/nncf/quantization/advanced_parameters.py | 9 +++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/nncf/openvino/rt_info.py b/src/nncf/openvino/rt_info.py index 2f5ebc3ccd8..cb1f81dbcdf 100644 --- a/src/nncf/openvino/rt_info.py +++ b/src/nncf/openvino/rt_info.py @@ -38,8 +38,7 @@ def exclude_empty_fields(value: dict[str, Any]) -> dict[str, Any]: def compression_config_to_dict(config: WeightCompressionConfig) -> dict[str, Any]: """ Converts a weight compression config into a dictionary suitable for dumping into Model's meta section. - - Codebook values are reduced to their number since the values themselves are too large to be dumped. + Codebook values are saved as their size since the values themselves are too large to be dumped. :param config: Weight compression config. :return: Dictionary with the compression config parameters. @@ -75,6 +74,8 @@ def dump_parameters( # Special condition for the list of custom annotations elif isinstance(value, (list, tuple)) and any(isinstance(item, CustomAnnotation) for item in value): for i, annotation in enumerate(value): + # Index each annotation so its scope and config stay paired and annotations + # do not overwrite each other, e.g. custom_annotation/0/{scope/patterns,config}. dump_parameters( model, {"scope": annotation.scope, "config": compression_config_to_dict(annotation.config)}, diff --git a/src/nncf/quantization/advanced_parameters.py b/src/nncf/quantization/advanced_parameters.py index 5c35ea07a46..f4c68d8bd5c 100644 --- a/src/nncf/quantization/advanced_parameters.py +++ b/src/nncf/quantization/advanced_parameters.py @@ -458,12 +458,9 @@ class AdvancedCompressionParameters: class CustomAnnotation: """ Binds a user-defined weight compression configuration to a portion of a model. - - The configuration takes precedence over the one assigned by the weight compression algorithm for all - matched nodes, including the assignment made by the mixed precision algorithm and the `ignored_scope`, - `all_layers` and `backup_mode` options. - - If several annotations match the same node, the last one in the `custom_annotation` list takes precedence. + It overrides the configuration assigned by the weight compression algorithm, including the mixed precision + algorithm and the `ignored_scope`, `all_layers` and `backup_mode` options. + If several annotations match the same node, the last one in the `custom_annotation` list wins. Example: From 75e822519ef7e7de9f0cfd1f3ec8dd4da6f53137 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 22 Sep 2026 12:48:38 +0400 Subject: [PATCH 20/43] clean --- .../template_test_weights_compression.py | 23 ++++++++----------- .../quantization/test_weights_compression.py | 3 ++- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 5f69120acf7..e49f08fd129 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -184,7 +184,7 @@ def test_data_based_criterion(self, mode, ref_score, ref_act_score, mocker): @staticmethod @abstractmethod def get_sequential_matmul_model(transpose_a: bool) -> TModel: - """Returns a backend model for test_mixed_precision.""" + """Returns a backend model with a sequence of MatMul layers.""" @staticmethod @abstractmethod @@ -979,15 +979,9 @@ def get_custom_annotation_ref_path(ref_name: str) -> Path: @abstractmethod def get_shared_weight_model(self) -> TModel: """ - Returns a backend model where two nodes share the same weight, ready to be compressed. + Returns a backend model where two nodes share the same weight. """ - def _get_sequential_matmul_model(self): - """Returns a backend model with a sequence of MatMul layers, ready to be compressed.""" - return self.wrap_model( - self.get_sequential_matmul_model(transpose_a=False), np.ones([1, 4, 4], dtype=np.float32) - ) - def _compress_and_get_configs(self, **kwargs) -> dict[str, dict[str, Any]]: """ Compresses a model and returns the compression config assigned to each compressed weight node. @@ -1139,12 +1133,13 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, Set the NNCF_TEST_REGEN_DOT environment variable to regenerate the reference file. """ - models = { - "sequential_matmul": self._get_sequential_matmul_model, - "shared_weight": self.get_shared_weight_model, - } + if model_name == "sequential_matmul": + model = self.get_sequential_matmul_model(transpose_a=False) + else: + model = self.get_shared_weight_model() + configs = self._compress_and_get_configs( - model=models[model_name](), + model=model, mode=CompressWeightsMode.INT4_SYM, group_size=-1, ignored_scope=ignored_scope, @@ -1211,7 +1206,7 @@ def test_invalid_custom_annotation(self, custom_annotation, error): """ with pytest.raises(error): compress_weights( - model=self._get_sequential_matmul_model(), + model=self.get_sequential_matmul_model(transpose_a=False), mode=CompressWeightsMode.INT4_SYM, ratio=1.0, group_size=-1, diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index e4a755f102a..ed4a0d6cb11 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -596,7 +596,8 @@ def get_SAM_PE_model() -> torch.nn.Module: def get_sequential_matmul_model(transpose_a: bool) -> torch.nn.Module: if transpose_a: pytest.skip("transpose_a=True is not supported for PT backend") - return SequentialMatmulModel() + model = wrap_model(SequentialMatmulModel()) + return GraphModelWrapper(model, example_input=torch.ones([1, 4, 4], dtype=torch.float32)) @staticmethod def get_model_for_test_scale_estimation(transpose_a: bool): From d510074f24e740e6efc9774bf31867e950616720 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 22 Sep 2026 12:53:47 +0400 Subject: [PATCH 21/43] fix --- .../function_hook/quantization/test_weights_compression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index ed4a0d6cb11..2805b504ded 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -596,8 +596,8 @@ def get_SAM_PE_model() -> torch.nn.Module: def get_sequential_matmul_model(transpose_a: bool) -> torch.nn.Module: if transpose_a: pytest.skip("transpose_a=True is not supported for PT backend") - model = wrap_model(SequentialMatmulModel()) - return GraphModelWrapper(model, example_input=torch.ones([1, 4, 4], dtype=torch.float32)) + wrapped_model = GraphModelWrapper(wrap_model(SequentialMatmulModel()), example_input=torch.ones([1, 4, 4])) + return wrapped_model @staticmethod def get_model_for_test_scale_estimation(transpose_a: bool): From c090461922b44eeb5d94d12b29c4a7789f8a78ae Mon Sep 17 00:00:00 2001 From: anzr299 Date: Tue, 22 Sep 2026 22:33:11 +0400 Subject: [PATCH 22/43] shared weights also give precendence to custom annotation --- .../weights_compression/Usage.md | 3 +- .../weight_compression/algorithm.py | 61 +++++++++---------- .../template_test_weights_compression.py | 25 +++++++- ...shared_weight_annotated_by_other_node.json | 7 +++ .../quantization/test_weights_compression.py | 4 ++ ...shared_weight_annotated_by_other_node.json | 17 ++++++ .../quantization/test_weights_compression.py | 4 ++ ...shared_weight_annotated_by_other_node.json | 12 ++++ ...shared_weight_annotated_by_other_node.json | 12 ++++ .../quantization/test_weights_compression.py | 4 ++ tests/torch/fx/test_weights_compression.py | 4 ++ 11 files changed, 119 insertions(+), 34 deletions(-) create mode 100644 tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json create mode 100644 tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json create mode 100644 tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json create mode 100644 tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json diff --git a/docs/usage/post_training_compression/weights_compression/Usage.md b/docs/usage/post_training_compression/weights_compression/Usage.md index 29420508e4b..2178101a34a 100644 --- a/docs/usage/post_training_compression/weights_compression/Usage.md +++ b/docs/usage/post_training_compression/weights_compression/Usage.md @@ -140,7 +140,8 @@ compressed_model = compress_weights( mixed-precision assignment, the `all_layers` option and the backup precision of the embeddings and the last linear layer. A layer that is matched by both the `ignored_scope` and a custom annotation is compressed with the user-defined configuration and a warning is logged. If several annotations match the same layer, the last one takes - precedence. + precedence. A weight shared by several layers is compressed once, so annotating any of them applies the + configuration to this weight. #### Data-aware methods diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 8caeb325296..9c6cfc6213f 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -160,7 +160,7 @@ def validate_custom_annotation(custom_annotation: list[CustomAnnotation] | None) for annotation in custom_annotation: if not isinstance(annotation, CustomAnnotation): msg = ( - "The custom_annotation option expects a list of nncf.CustomAnnotation instances, " + "The custom_annotation argument expects a list of nncf.CustomAnnotation instances, " f"but an element of type {type(annotation).__name__} is given." ) raise nncf.ValidationError(msg) @@ -1198,11 +1198,10 @@ def _get_custom_annotation_configs( overlapped_names = sorted(annotated_names & node_name_to_config.keys()) if overlapped_names: config = annotation.config - group_size = "per-channel" if config.group_size == -1 else f"group size {config.group_size}" nncf_logger.warning( f"Several custom annotations match the same nodes. The configuration of the custom annotation " - f"with index {index} ({config.mode.value}, {group_size}) takes precedence, since it is the last " - "matched one, and is used for the following nodes:\n\t" + "\n\t".join(overlapped_names) + f"with index {index} ({config.mode.value}, {config.group_size}) takes precedence, since it is " + "the last matched one, and is used for the following nodes:\n\t" + "\n\t".join(overlapped_names) ) for node_name in annotated_names: node_name_to_config[node_name] = annotation.config @@ -1216,23 +1215,12 @@ def _drop_annotation_configs_that_can_not_be_assigned( skipped_weight_params: list[WeightCompressionParameters], ) -> dict[str, WeightCompressionConfig]: """ - Drops the annotated configurations that can not be assigned to a weight, i.e. the ones matching a node - without a weight, a weight with a data type that the annotated mode can not compress, or a weight that is - compressed under another node. + Validate and drop the annotated configurations that can not be assigned to a weight, i.e. the ones + matching a node without a weight or a weight with a data type that the annotated mode can not compress. + The configuration of a node whose weight is compressed under another node, e.g. a shared weight, is moved + to that node, so that the annotation takes effect no matter which of the nodes sharing a weight is + matched by it. """ - unsupported_node_names = set() - for w_params in all_weight_params + skipped_weight_params: - node_name = w_params.node_with_weight.node_name - config = node_name_to_config.get(node_name) - if config is not None and not self.is_weight_compression_supported(w_params.weight_dtype, config.mode): - unsupported_node_names.add(node_name) - if unsupported_node_names: - nncf_logger.warning( - "The compression mode defined by the custom annotation is not supported for the data type of the " - "following nodes. They are kept with the configuration assigned by the algorithm:\n\t" - + "\n\t".join(sorted(unsupported_node_names)) - ) - node_names_with_weight = set(node.node_name for node in self.get_nodes_to_compress(graph)) node_names_without_weight = node_name_to_config.keys() - node_names_with_weight if node_names_without_weight: @@ -1241,21 +1229,32 @@ def _drop_annotation_configs_that_can_not_be_assigned( "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(node_names_without_weight)) ) - # A weight is compressed under a single node, so the annotation of the other nodes that share this weight, - # e.g. the ones matched by the annotation only, can not be assigned to it. - compressed_weight_names = set(w_params.weight_name for w_params in all_weight_params) - compressed_node_names = set(w_params.node_with_weight.node_name for w_params in all_weight_params) + # A weight is compressed under a single node, so the annotation of the other nodes sharing this weight, + # e.g. the ones matched by the annotation only, is moved to the node under which it is compressed. + node_name_by_weight_name = {wp.weight_name: wp.node_with_weight.node_name for wp in all_weight_params} + compressed_node_names = set(node_name_by_weight_name.values()) + moved_configs = {} shared_weight_node_names = set() for node_name in (node_name_to_config.keys() & node_names_with_weight) - compressed_node_names: node = graph.get_node_by_name(node_name) - weight_names = (name for name, _ in self._backend_entity.get_weight_names_and_port_ids(node, graph)) - if any(weight_name in compressed_weight_names for weight_name in weight_names): - shared_weight_node_names.add(node_name) - if shared_weight_node_names: + for weight_name, _ in self._backend_entity.get_weight_names_and_port_ids(node, graph): + if weight_name in node_name_by_weight_name: + moved_configs[node_name_by_weight_name[weight_name]] = node_name_to_config[node_name] + shared_weight_node_names.add(node_name) + # The annotation of the node under which a weight is compressed takes precedence over the moved one + node_name_to_config = moved_configs | node_name_to_config + + unsupported_node_names = set() + for w_params in all_weight_params + skipped_weight_params: + node_name = w_params.node_with_weight.node_name + config = node_name_to_config.get(node_name) + if config is not None and not self.is_weight_compression_supported(w_params.weight_dtype, config.mode): + unsupported_node_names.add(node_name) + if unsupported_node_names: nncf_logger.warning( - "The weight of the following nodes matched by the custom annotation is compressed under another " - "node, e.g. a shared weight. The annotation has no effect for them:\n\t" - + "\n\t".join(sorted(shared_weight_node_names)) + "The compression mode defined by the custom annotation is not supported for the data type of the " + "following nodes. They are kept with the configuration assigned by the algorithm:\n\t" + + "\n\t".join(sorted(unsupported_node_names)) ) dropped_node_names = unsupported_node_names | node_names_without_weight | shared_weight_node_names diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index e49f08fd129..698d6917242 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -982,6 +982,14 @@ def get_shared_weight_model(self) -> TModel: Returns a backend model where two nodes share the same weight. """ + @staticmethod + @abstractmethod + def get_shared_weight_node_name() -> str: + """ + Returns the name of the node from `get_shared_weight_model` whose weight is compressed under the other + node sharing it. + """ + def _compress_and_get_configs(self, **kwargs) -> dict[str, dict[str, Any]]: """ Compresses a model and returns the compression config assigned to each compressed weight node. @@ -1072,7 +1080,6 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa group_size_fallback_mode=nncf.GroupSizeFallbackMode.ADJUST, min_adjusted_group_size=4 ), ), - # The channel size of the model is 4, so the group size of 3 is invalid [ ( nncf.CustomAnnotationScope(patterns=[".*1$"]), @@ -1124,6 +1131,14 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa IgnoredScope(patterns=[".*"]), id="shared_weight_with_ignored_scope", ), + pytest.param( + "shared_weight", + dict(ratio=1.0, all_layers=True), + # The scope is None, since the annotated node name is backend-specific + [(None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2))], + None, + id="shared_weight_annotated_by_other_node", + ), ], ) def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, request): @@ -1138,13 +1153,19 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, else: model = self.get_shared_weight_model() + shared_weight_scope = nncf.CustomAnnotationScope(names=[self.get_shared_weight_node_name()]) + custom_annotation = [ + nncf.CustomAnnotation(scope=shared_weight_scope if scope is None else scope, config=config) + for scope, config in annotations + ] + configs = self._compress_and_get_configs( model=model, mode=CompressWeightsMode.INT4_SYM, group_size=-1, ignored_scope=ignored_scope, **kwargs, - custom_annotation=[nncf.CustomAnnotation(scope=scope, config=config) for scope, config in annotations], + custom_annotation=custom_annotation, ) ref_path = self.get_custom_annotation_ref_path(request.node.callspec.id) diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json new file mode 100644 index 00000000000..22e136ab8aa --- /dev/null +++ b/tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json @@ -0,0 +1,7 @@ +{ + "MatMul_0": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/onnx/quantization/test_weights_compression.py b/tests/onnx/quantization/test_weights_compression.py index f3b06ed22c4..d2e5b8e9ee7 100644 --- a/tests/onnx/quantization/test_weights_compression.py +++ b/tests/onnx/quantization/test_weights_compression.py @@ -749,6 +749,10 @@ def get_shared_weight_model() -> onnx.ModelProto: onnx.checker.check_model(model) return model + @staticmethod + def get_shared_weight_node_name() -> str: + return "MatMul_1" + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json new file mode 100644 index 00000000000..311bf243324 --- /dev/null +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json @@ -0,0 +1,17 @@ +{ + "Gather_1": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + }, + "Gather_2": { + "mode": "int4_sym", + "group_size": -1, + "codebook_values": null + }, + "MatMul_1": { + "mode": "int4_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index f752aab187f..b76a24726c9 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -2551,6 +2551,10 @@ def get_awq_scale_ref_path() -> Path: def get_shared_weight_model() -> ov.Model: return GatherAndMatmulShareData().ov_model + @staticmethod + def get_shared_weight_node_name() -> str: + return "MatMul_2" + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return get_actual_reference_for_current_openvino(CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json") diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json new file mode 100644 index 00000000000..a9f49288bf9 --- /dev/null +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json @@ -0,0 +1,12 @@ +{ + "wte/embedding/0": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + }, + "linear/linear/0": { + "mode": "int4_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json new file mode 100644 index 00000000000..d6f5be0df5e --- /dev/null +++ b/tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json @@ -0,0 +1,12 @@ +{ + "embedding": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + }, + "linear": { + "mode": "int4_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index 2805b504ded..31bab6c3e14 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -732,6 +732,10 @@ def get_shared_weight_model() -> GraphModelWrapper: model = wrap_model(ShortTransformer(8, 16, share_weights=True)) return GraphModelWrapper(model, example_input=torch.randint(0, 10, (8,))) + @staticmethod + def get_shared_weight_node_name() -> str: + return "lm_head/linear/0" + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" diff --git a/tests/torch/fx/test_weights_compression.py b/tests/torch/fx/test_weights_compression.py index 480588eb306..dc022d5e298 100644 --- a/tests/torch/fx/test_weights_compression.py +++ b/tests/torch/fx/test_weights_compression.py @@ -519,6 +519,10 @@ def get_shared_weight_model() -> torch.fx.GraphModule: model = ShortTransformer(8, 16, share_weights=True) return get_torch_fx_model(model, torch.randint(0, 10, (8,))) + @staticmethod + def get_shared_weight_node_name() -> str: + return "linear_1" + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" From e87a62e69f10d8c6dfe5ecc45c8e029569acd711 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 23 Sep 2026 13:56:10 +0400 Subject: [PATCH 23/43] fix multiple warnings; third shared node case --- .../weight_compression/algorithm.py | 92 ++++++++++-------- .../template_test_weights_compression.py | 96 +++++++++++++++++++ .../overlapping_annotations.json | 4 +- ...shared_weight_annotated_by_other_node.json | 2 +- ...red_weight_other_node_annotated_first.json | 7 ++ ...ared_weight_other_node_annotated_last.json | 7 ++ .../overlapping_annotations.json | 4 +- ...shared_weight_annotated_by_other_node.json | 2 +- ...red_weight_other_node_annotated_first.json | 17 ++++ ...ared_weight_other_node_annotated_last.json | 17 ++++ .../overlapping_annotations.json | 4 +- ...shared_weight_annotated_by_other_node.json | 2 +- ...red_weight_other_node_annotated_first.json | 12 +++ ...ared_weight_other_node_annotated_last.json | 12 +++ .../overlapping_annotations.json | 4 +- ...shared_weight_annotated_by_other_node.json | 2 +- ...red_weight_other_node_annotated_first.json | 12 +++ ...ared_weight_other_node_annotated_last.json | 12 +++ 18 files changed, 258 insertions(+), 50 deletions(-) create mode 100644 tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_first.json create mode 100644 tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_last.json create mode 100644 tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_first.json create mode 100644 tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_last.json create mode 100644 tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_first.json create mode 100644 tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_last.json create mode 100644 tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_first.json create mode 100644 tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_last.json diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 9c6cfc6213f..28e9aa233fb 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -1180,31 +1180,61 @@ def get_weight_compression_parameters( return all_weight_params, ratio_defining_params, skipped_weight_params + def _get_node_name_to_owner_name( + self, graph: NNCFGraph, all_weight_params: list[WeightCompressionParameters] + ) -> dict[str, str]: + """ + Maps the name of a node whose weight is compressed under another node, e.g. a shared weight, to the name + of that node. It allows the custom annotation to take effect no matter which of the nodes sharing a weight + is matched by it. + + :param graph: NNCFGraph instance. + :param all_weight_params: List of all weight compression parameters that can be compressed. + :return: A mapping from a node name to the name of the node under which its weight is compressed. + """ + owner_name_by_weight_name = {wp.weight_name: wp.node_with_weight.node_name for wp in all_weight_params} + node_name_to_owner_name = {} + for node in self.get_nodes_to_compress(graph): + for weight_name, _ in self._backend_entity.get_weight_names_and_port_ids(node, graph): + owner_name = owner_name_by_weight_name.get(weight_name) + if owner_name is not None and owner_name != node.node_name: + node_name_to_owner_name[node.node_name] = owner_name + return node_name_to_owner_name + @staticmethod def _get_custom_annotation_configs( - custom_annotation: list[CustomAnnotation], graph: NNCFGraph + custom_annotation: list[CustomAnnotation], graph: NNCFGraph, node_name_to_owner_name: dict[str, str] ) -> dict[str, WeightCompressionConfig]: """ Maps node names matched by the custom annotation to the user-defined weight compression configuration. - If several annotations match the same node, the last one takes precedence. + If several annotations match the same node, the last one takes precedence. The name of a node whose weight + is compressed under another node is replaced with the name of that node, so that the nodes sharing a weight + are annotated as a single one. :param custom_annotation: List of user-defined custom annotations. :param graph: NNCFGraph instance. + :param node_name_to_owner_name: A mapping from a node name to the name of the node under which its weight + is compressed. :return: A mapping from a node name to the user-defined weight compression configuration. """ node_name_to_config: dict[str, WeightCompressionConfig] = {} - for index, annotation in enumerate(custom_annotation): + overlapped_names = set() + for annotation in custom_annotation: annotated_names = get_node_names_from_scope(annotation.scope, graph, strict=annotation.scope.validate) - overlapped_names = sorted(annotated_names & node_name_to_config.keys()) - if overlapped_names: - config = annotation.config - nncf_logger.warning( - f"Several custom annotations match the same nodes. The configuration of the custom annotation " - f"with index {index} ({config.mode.value}, {config.group_size}) takes precedence, since it is " - "the last matched one, and is used for the following nodes:\n\t" + "\n\t".join(overlapped_names) - ) - for node_name in annotated_names: + owner_names = set(node_name_to_owner_name.get(name, name) for name in annotated_names) + overlapped_names.update(name for name in owner_names if name in node_name_to_config) + for node_name in owner_names: node_name_to_config[node_name] = annotation.config + + if overlapped_names: + overlapped_configs = [] + for node_name in sorted(overlapped_names): + config = node_name_to_config[node_name] + overlapped_configs.append(f"{node_name} ({config.mode.value}, {config.group_size})") + nncf_logger.warning( + "Several custom annotations match the same nodes. The configuration of the last matched custom " + "annotation takes precedence and is used for the following nodes:\n\t" + "\n\t".join(overlapped_configs) + ) return node_name_to_config def _drop_annotation_configs_that_can_not_be_assigned( @@ -1217,35 +1247,18 @@ def _drop_annotation_configs_that_can_not_be_assigned( """ Validate and drop the annotated configurations that can not be assigned to a weight, i.e. the ones matching a node without a weight or a weight with a data type that the annotated mode can not compress. - The configuration of a node whose weight is compressed under another node, e.g. a shared weight, is moved - to that node, so that the annotation takes effect no matter which of the nodes sharing a weight is - matched by it. """ node_names_with_weight = set(node.node_name for node in self.get_nodes_to_compress(graph)) - node_names_without_weight = node_name_to_config.keys() - node_names_with_weight + node_names_without_weight = set(name for name in node_name_to_config if name not in node_names_with_weight) if node_names_without_weight: nncf_logger.warning( "The following nodes are matched by the custom annotation, but have no weight to compress. " "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(node_names_without_weight)) ) - # A weight is compressed under a single node, so the annotation of the other nodes sharing this weight, - # e.g. the ones matched by the annotation only, is moved to the node under which it is compressed. - node_name_by_weight_name = {wp.weight_name: wp.node_with_weight.node_name for wp in all_weight_params} - compressed_node_names = set(node_name_by_weight_name.values()) - moved_configs = {} - shared_weight_node_names = set() - for node_name in (node_name_to_config.keys() & node_names_with_weight) - compressed_node_names: - node = graph.get_node_by_name(node_name) - for weight_name, _ in self._backend_entity.get_weight_names_and_port_ids(node, graph): - if weight_name in node_name_by_weight_name: - moved_configs[node_name_by_weight_name[weight_name]] = node_name_to_config[node_name] - shared_weight_node_names.add(node_name) - # The annotation of the node under which a weight is compressed takes precedence over the moved one - node_name_to_config = moved_configs | node_name_to_config - unsupported_node_names = set() - for w_params in all_weight_params + skipped_weight_params: + weight_params = all_weight_params + skipped_weight_params + for w_params in weight_params: node_name = w_params.node_with_weight.node_name config = node_name_to_config.get(node_name) if config is not None and not self.is_weight_compression_supported(w_params.weight_dtype, config.mode): @@ -1257,7 +1270,8 @@ def _drop_annotation_configs_that_can_not_be_assigned( + "\n\t".join(sorted(unsupported_node_names)) ) - dropped_node_names = unsupported_node_names | node_names_without_weight | shared_weight_node_names + dropped_node_names = set() + dropped_node_names.update(node_names_without_weight, unsupported_node_names) return { node_name: config for node_name, config in node_name_to_config.items() @@ -1324,10 +1338,8 @@ def _move_annotated_ignored_scope_params_from_skipped_to_compressed( ] # Keep the topological order of the weight parameters node_name_to_position = {node.node_name: i for i, node in enumerate(graph.topological_sort())} - all_weight_params = sorted( - all_weight_params + restored_weight_params, - key=lambda wp: node_name_to_position[wp.node_with_weight.node_name], - ) + all_weight_params = all_weight_params + restored_weight_params + all_weight_params.sort(key=lambda wp: node_name_to_position[wp.node_with_weight.node_name]) return all_weight_params, skipped_weight_params def apply_custom_annotation( @@ -1345,6 +1357,7 @@ def apply_custom_annotation( Applies the user-defined weight compression configurations to the parameters matched by the annotation. The flow of this function is as follows: 1. Convert the annotation to a mapping from a node name to the configuration, the last match wins. + The nodes sharing a weight are annotated as a single one, since the weight is compressed only once. 2. Drop the annotated configurations that can not be assigned to a weight. 3. Move the annotated weight parameters that are not compressed, e.g. the ignored ones, to the compressed ones. @@ -1360,7 +1373,10 @@ def apply_custom_annotation( if not self._custom_annotation: return all_weight_params, ratio_defining_params, skipped_weight_params - node_name_to_config = self._get_custom_annotation_configs(self._custom_annotation, graph) + node_name_to_owner_name = self._get_node_name_to_owner_name(graph, all_weight_params) + node_name_to_config = self._get_custom_annotation_configs( + self._custom_annotation, graph, node_name_to_owner_name + ) node_name_to_config = self._drop_annotation_configs_that_can_not_be_assigned( node_name_to_config, graph, all_weight_params, skipped_weight_params ) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 698d6917242..51729d8da78 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -1067,6 +1067,10 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa nncf.CustomAnnotationScope(patterns=[".*2$"]), WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4), ), + ( + nncf.CustomAnnotationScope(patterns=[".*2$"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT4_SYM, group_size=2), + ), ], None, id="overlapping_annotations", @@ -1139,6 +1143,32 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa None, id="shared_weight_annotated_by_other_node", ), + pytest.param( + "shared_weight", + dict(ratio=1.0, all_layers=True), + [ + ( + nncf.CustomAnnotationScope(patterns=[".*"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ), + (None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2)), + ], + None, + id="shared_weight_other_node_annotated_last", + ), + pytest.param( + "shared_weight", + dict(ratio=1.0, all_layers=True), + [ + (None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2)), + ( + nncf.CustomAnnotationScope(patterns=[".*"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ), + ], + None, + id="shared_weight_other_node_annotated_first", + ), ], ) def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, request): @@ -1174,6 +1204,72 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, assert configs == load_json(ref_path) + @pytest.mark.parametrize( + ("custom_annotation", "ignored_scope", "warning"), + [ + pytest.param( + [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*1$"]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ), + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*1$"]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4), + ), + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*1$"]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_SYM, group_size=2), + ), + ], + None, + "Several custom annotations match the same nodes", + id="overlapping_annotations", + ), + pytest.param( + [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*"]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + ], + None, + "have no weight to compress", + id="node_without_weight", + ), + pytest.param( + [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*1$"]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + ], + IgnoredScope(patterns=[".*1$"]), + "excluded from the compression", + id="ignored_scope", + ), + ], + ) + def test_custom_annotation_warning(self, custom_annotation, ignored_scope, warning): + """ + Checks that appropriate warnings are logged when the custom annotation is overridden or has no effect. + """ + with patch.object(nncf_logger, "warning") as mock_warning: + compress_weights( + model=self.get_sequential_matmul_model(transpose_a=False), + mode=CompressWeightsMode.INT4_SYM, + ratio=1.0, + group_size=-1, + all_layers=True, + ignored_scope=ignored_scope, + custom_annotation=custom_annotation, + ) + + messages = [args[0] for args, _ in mock_warning.call_args_list] + # Check if warning is logged + only 1 warning is logged for a single incident + matched = [message for message in messages if warning in message] + assert len(matched) == 1, f"Expected a single '{warning}' warning, but got: {messages}" + @pytest.mark.parametrize( ("custom_annotation", "error"), [ diff --git a/tests/onnx/data/references_custom_annotation/overlapping_annotations.json b/tests/onnx/data/references_custom_annotation/overlapping_annotations.json index a7cd32e850a..3f252175790 100644 --- a/tests/onnx/data/references_custom_annotation/overlapping_annotations.json +++ b/tests/onnx/data/references_custom_annotation/overlapping_annotations.json @@ -10,8 +10,8 @@ "codebook_values": null }, "MatMul_2": { - "mode": "int4_asym", - "group_size": 4, + "mode": "int4_sym", + "group_size": 2, "codebook_values": null }, "MatMul_3": { diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json index 22e136ab8aa..0a042d47e6e 100644 --- a/tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json +++ b/tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json @@ -4,4 +4,4 @@ "group_size": 2, "codebook_values": null } -} \ No newline at end of file +} diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_first.json b/tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_first.json new file mode 100644 index 00000000000..e16fb2f3d37 --- /dev/null +++ b/tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_first.json @@ -0,0 +1,7 @@ +{ + "MatMul_0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_last.json b/tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_last.json new file mode 100644 index 00000000000..0a042d47e6e --- /dev/null +++ b/tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_last.json @@ -0,0 +1,7 @@ +{ + "MatMul_0": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + } +} diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json b/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json index a7cd32e850a..3f252175790 100644 --- a/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json @@ -10,8 +10,8 @@ "codebook_values": null }, "MatMul_2": { - "mode": "int4_asym", - "group_size": 4, + "mode": "int4_sym", + "group_size": 2, "codebook_values": null }, "MatMul_3": { diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json index 311bf243324..6a38e9d3bad 100644 --- a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json @@ -14,4 +14,4 @@ "group_size": -1, "codebook_values": null } -} \ No newline at end of file +} diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_first.json b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_first.json new file mode 100644 index 00000000000..9af92af8a7a --- /dev/null +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_first.json @@ -0,0 +1,17 @@ +{ + "Gather_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "Gather_2": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_last.json b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_last.json new file mode 100644 index 00000000000..afe01be6569 --- /dev/null +++ b/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_last.json @@ -0,0 +1,17 @@ +{ + "Gather_1": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + }, + "Gather_2": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json index 15359265f4d..2897dbf007c 100644 --- a/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json @@ -10,8 +10,8 @@ "codebook_values": null }, "/linear/2": { - "mode": "int4_asym", - "group_size": 4, + "mode": "int4_sym", + "group_size": 2, "codebook_values": null }, "/linear/3": { diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json index a9f49288bf9..b399e1b5e7a 100644 --- a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json @@ -9,4 +9,4 @@ "group_size": -1, "codebook_values": null } -} \ No newline at end of file +} diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_first.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_first.json new file mode 100644 index 00000000000..b1af06693ed --- /dev/null +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_first.json @@ -0,0 +1,12 @@ +{ + "wte/embedding/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "linear/linear/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_last.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_last.json new file mode 100644 index 00000000000..a059faf1e4c --- /dev/null +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_last.json @@ -0,0 +1,12 @@ +{ + "wte/embedding/0": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + }, + "linear/linear/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json b/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json index 7936d407b6e..316c4eba326 100644 --- a/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json +++ b/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json @@ -10,8 +10,8 @@ "codebook_values": null }, "linear_2": { - "mode": "int4_asym", - "group_size": 4, + "mode": "int4_sym", + "group_size": 2, "codebook_values": null }, "linear_3": { diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json index d6f5be0df5e..ecaca76f5e1 100644 --- a/tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json +++ b/tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json @@ -9,4 +9,4 @@ "group_size": -1, "codebook_values": null } -} \ No newline at end of file +} diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_first.json b/tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_first.json new file mode 100644 index 00000000000..e191f7b1803 --- /dev/null +++ b/tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_first.json @@ -0,0 +1,12 @@ +{ + "embedding": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "linear": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_last.json b/tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_last.json new file mode 100644 index 00000000000..d87b890d346 --- /dev/null +++ b/tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_last.json @@ -0,0 +1,12 @@ +{ + "embedding": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + }, + "linear": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} From dd6587c87d9c99b395d95db142ce85b5e2cd4d6a Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 23 Sep 2026 14:51:25 +0400 Subject: [PATCH 24/43] extend custom annotation to awq and scale estimation tests --- .../template_test_weights_compression.py | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 51729d8da78..ef834f8e3b8 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -346,9 +346,13 @@ def get_orig_weight(model: TModel) -> Tensor: def get_decompressed_weight(compressed_model: TModel, input: TTensor) -> Tensor: """Returns decompressed weight""" + @pytest.mark.parametrize("annotated_mode", [None, CompressWeightsMode.INT4_SYM], ids=["default", "annotated"]) @pytest.mark.parametrize("transpose_a", [False, True], ids=["no_tr_a", "tr_a"]) - def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a): - """Checks that act_ch_axis parameter is passed to calculate_quantization_params.""" + def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a, annotated_mode): + """ + Checks that act_ch_axis parameter and the config defined by the custom annotation are passed to + calculate_quantization_params. + """ calc_q_params_spy = mocker.spy(ScaleEstimation, "calculate_quantization_params") model = self.get_model_for_test_scale_estimation(transpose_a=transpose_a) @@ -356,6 +360,15 @@ def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a): input = self.to_tensor(input) dataset = Dataset([input], self.get_transform_func()) + custom_annotation = None + if annotated_mode is not None: + custom_annotation = [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*"]), + config=WeightCompressionConfig(mode=annotated_mode, group_size=8), + ) + ] + with SpyWeightCompressionStatisticsContext(mocker): _ = compress_weights( model, @@ -365,6 +378,7 @@ def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a): scale_estimation=True, all_layers=True, dataset=dataset, + custom_annotation=custom_annotation, ) # Verify calculate_quantization_params was called @@ -375,6 +389,10 @@ def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a): call_args = calc_q_params_spy.call_args[0] assert len(call_args) >= 5, "calculate_quantization_params should have at least 5 positional arguments" + # Verify the config is the annotated one, if the custom annotation is given + config = call_args[3] + assert config.mode == (annotated_mode or CompressWeightsMode.INT4_ASYM) + # Verify act_ch_axis has a valid value (should be an integer) act_ch_axis = call_args[4] if transpose_a: @@ -698,10 +716,15 @@ def get_num_int4_group_sizes(model: TModel) -> dict[int, int]: @staticmethod @abstractmethod def get_ignored_scope_name(is_3d_weights) -> str: - "Returns ignored scope name for test_awq_with_ignored_scope." + "Returns the name of the node that is excluded from the 4 bit compression in test_awq_with_excluded_node." @pytest.mark.parametrize("is_3d_weights", [True, False]) - def test_awq_with_ignored_scope(self, mocker, is_3d_weights): + @pytest.mark.parametrize("exclude_by_annotation", [False, True], ids=["ignored_scope", "custom_annotation"]) + def test_awq_with_excluded_node(self, mocker, is_3d_weights, exclude_by_annotation): + """ + Checks that a node is kept out of the 4 bit compression when AWQ is enabled, either by the ignored scope + or by a custom annotation with an 8-bit configuration. + """ model = self.get_awq_model(non_mergable_pattern=False, is_3d_weights=is_3d_weights) sz = 8 n_samples = 10 @@ -713,6 +736,19 @@ def test_awq_with_ignored_scope(self, mocker, is_3d_weights): self.get_transform_func(), ) + excluded_node_name = self.get_ignored_scope_name(is_3d_weights) + ignored_scope = None + custom_annotation = None + if exclude_by_annotation: + custom_annotation = [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=[excluded_node_name]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + ] + else: + ignored_scope = IgnoredScope(names=[excluded_node_name]) + with SpyWeightCompressionStatisticsContext(mocker): compressed_model = compress_weights( model, @@ -721,10 +757,11 @@ def test_awq_with_ignored_scope(self, mocker, is_3d_weights): group_size=-1, dataset=dataset, awq=True, - ignored_scope=IgnoredScope(names=[self.get_ignored_scope_name(is_3d_weights)]), + ignored_scope=ignored_scope, + custom_annotation=custom_annotation, ) - int4_ref_num_compressed = 4 # last MatMul is always int8; one - is ignored; total 6 matmuls + int4_ref_num_compressed = 4 # last MatMul is always int8; one - is excluded; total 6 matmuls int4_num_nodes = self.get_num_int4_nodes(compressed_model) assert int4_num_nodes == int4_ref_num_compressed, int4_num_nodes From cd62da1bb8f28ad6884c60464ee432695e1ab752 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 23 Sep 2026 14:51:45 +0400 Subject: [PATCH 25/43] fix docstring --- tests/onnx/quantization/test_weights_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/onnx/quantization/test_weights_compression.py b/tests/onnx/quantization/test_weights_compression.py index d2e5b8e9ee7..b0ba45b0278 100644 --- a/tests/onnx/quantization/test_weights_compression.py +++ b/tests/onnx/quantization/test_weights_compression.py @@ -637,7 +637,7 @@ def get_awq_model(non_mergable_pattern: bool, is_3d_weights: bool, grouped_mm: b raise NotImplementedError(msg) """ Builds a model to be used in the following tests: - - TemplateWeightCompression.test_awq_with_ignored_scope() + - TemplateWeightCompression.test_awq_with_excluded_node() - TemplateWeightCompression.test_awq_scale_reference() - TemplateWeightCompression.test_error_message_for_invalid_group_size() tests. From a1777281cec6ad1bd2f4d964bbe4c40f0e63409c Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 23 Sep 2026 21:10:05 +0400 Subject: [PATCH 26/43] rework --- .../weight_compression/algorithm.py | 204 ++++++++++-------- 1 file changed, 117 insertions(+), 87 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 28e9aa233fb..49d8860fb90 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -1180,89 +1180,115 @@ def get_weight_compression_parameters( return all_weight_params, ratio_defining_params, skipped_weight_params - def _get_node_name_to_owner_name( - self, graph: NNCFGraph, all_weight_params: list[WeightCompressionParameters] - ) -> dict[str, str]: + def _get_node_name_to_weight_names_mapping(self, graph: NNCFGraph) -> dict[str, list[str]]: """ - Maps the name of a node whose weight is compressed under another node, e.g. a shared weight, to the name - of that node. It allows the custom annotation to take effect no matter which of the nodes sharing a weight - is matched by it. + Maps the name of each node that can be compressed to the names of its weights. :param graph: NNCFGraph instance. - :param all_weight_params: List of all weight compression parameters that can be compressed. - :return: A mapping from a node name to the name of the node under which its weight is compressed. + :return: A mapping from a node name to the names of the weights of that node. """ - owner_name_by_weight_name = {wp.weight_name: wp.node_with_weight.node_name for wp in all_weight_params} - node_name_to_owner_name = {} - for node in self.get_nodes_to_compress(graph): - for weight_name, _ in self._backend_entity.get_weight_names_and_port_ids(node, graph): - owner_name = owner_name_by_weight_name.get(weight_name) - if owner_name is not None and owner_name != node.node_name: - node_name_to_owner_name[node.node_name] = owner_name - return node_name_to_owner_name + return { + node.node_name: [name for name, _ in self._backend_entity.get_weight_names_and_port_ids(node, graph)] + for node in self.get_nodes_to_compress(graph) + } @staticmethod - def _get_custom_annotation_configs( - custom_annotation: list[CustomAnnotation], graph: NNCFGraph, node_name_to_owner_name: dict[str, str] - ) -> dict[str, WeightCompressionConfig]: + def _get_weight_name_to_config_mapping( + custom_annotation: list[CustomAnnotation], + graph: NNCFGraph, + node_name_to_weight_names_mapping: dict[str, list[str]], + ) -> tuple[dict[str, WeightCompressionConfig], set[str], set[str]]: """ - Maps node names matched by the custom annotation to the user-defined weight compression configuration. - If several annotations match the same node, the last one takes precedence. The name of a node whose weight - is compressed under another node is replaced with the name of that node, so that the nodes sharing a weight - are annotated as a single one. + Matches the custom annotation against the weights of the nodes that can be compressed. Keying by a weight + name annotates the nodes sharing a weight as a single one. :param custom_annotation: List of user-defined custom annotations. :param graph: NNCFGraph instance. - :param node_name_to_owner_name: A mapping from a node name to the name of the node under which its weight - is compressed. - :return: A mapping from a node name to the user-defined weight compression configuration. + :param node_name_to_weight_names_mapping: A mapping from a node name to the names of the weights of that node. + :return: A tuple of the mapping from a weight name to the annotated configuration, the matched node names + with no weight to compress and the weight names matched by several annotations. """ - node_name_to_config: dict[str, WeightCompressionConfig] = {} - overlapped_names = set() + weight_name_to_config_mapping: dict[str, WeightCompressionConfig] = {} + overlapped_annotation_weight_names = set() + node_names_without_weight = set() for annotation in custom_annotation: annotated_names = get_node_names_from_scope(annotation.scope, graph, strict=annotation.scope.validate) - owner_names = set(node_name_to_owner_name.get(name, name) for name in annotated_names) - overlapped_names.update(name for name in owner_names if name in node_name_to_config) - for node_name in owner_names: - node_name_to_config[node_name] = annotation.config + annotated_weight_names = set() + for node_name in annotated_names: + weight_names = node_name_to_weight_names_mapping.get(node_name) + if not weight_names: + node_names_without_weight.add(node_name) + continue + annotated_weight_names.update(weight_names) + # The weights annotated by the current annotation overlap with the ones annotated before it + overlapped_annotation_weight_names.update( + name for name in annotated_weight_names if name in weight_name_to_config_mapping + ) + for weight_name in annotated_weight_names: + weight_name_to_config_mapping[weight_name] = annotation.config + + return weight_name_to_config_mapping, node_names_without_weight, overlapped_annotation_weight_names - if overlapped_names: + def _get_custom_annotation_configs( + self, custom_annotation: list[CustomAnnotation], graph: NNCFGraph + ) -> dict[str, WeightCompressionConfig]: + """ + Maps the weights matched by the custom annotation to the user-defined weight compression configuration. + + :param custom_annotation: List of user-defined custom annotations. + :param graph: NNCFGraph instance. + :return: A mapping from a weight name to the user-defined weight compression configuration. + """ + node_name_to_weight_names_mapping = self._get_node_name_to_weight_names_mapping(graph) + + weight_name_to_config_mapping, node_names_without_weight, overlapped_annotation_weight_names = ( + self._get_weight_name_to_config_mapping(custom_annotation, graph, node_name_to_weight_names_mapping) + ) + + if node_names_without_weight: + nncf_logger.warning( + "The following nodes are matched by the custom annotation, but have no weight to compress. " + "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(node_names_without_weight)) + ) + if overlapped_annotation_weight_names: overlapped_configs = [] - for node_name in sorted(overlapped_names): - config = node_name_to_config[node_name] - overlapped_configs.append(f"{node_name} ({config.mode.value}, {config.group_size})") + for weight_name in sorted(overlapped_annotation_weight_names): + config = weight_name_to_config_mapping[weight_name] + node_names = sorted( + node_name + for node_name, weight_names in node_name_to_weight_names_mapping.items() + if weight_name in weight_names + ) + overlapped_configs.append(f"{', '.join(node_names)} ({config.mode.value}, {config.group_size})") nncf_logger.warning( "Several custom annotations match the same nodes. The configuration of the last matched custom " "annotation takes precedence and is used for the following nodes:\n\t" + "\n\t".join(overlapped_configs) ) - return node_name_to_config + return weight_name_to_config_mapping def _drop_annotation_configs_that_can_not_be_assigned( self, - node_name_to_config: dict[str, WeightCompressionConfig], - graph: NNCFGraph, + weight_name_to_config_mapping: dict[str, WeightCompressionConfig], all_weight_params: list[WeightCompressionParameters], skipped_weight_params: list[WeightCompressionParameters], ) -> dict[str, WeightCompressionConfig]: """ - Validate and drop the annotated configurations that can not be assigned to a weight, i.e. the ones - matching a node without a weight or a weight with a data type that the annotated mode can not compress. - """ - node_names_with_weight = set(node.node_name for node in self.get_nodes_to_compress(graph)) - node_names_without_weight = set(name for name in node_name_to_config if name not in node_names_with_weight) - if node_names_without_weight: - nncf_logger.warning( - "The following nodes are matched by the custom annotation, but have no weight to compress. " - "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(node_names_without_weight)) - ) + Validate and drop the annotated configurations that can not be assigned to a weight + :param weight_name_to_config_mapping: A mapping from a weight name to the annotated compression configuration. + :param all_weight_params: List of all weight compression parameters that can be compressed. + :param skipped_weight_params: List of weight compression parameters that are not compressed. + :return: The mapping with the configurations that can not be assigned dropped. + """ + unsupported_weight_names = set() unsupported_node_names = set() - weight_params = all_weight_params + skipped_weight_params - for w_params in weight_params: - node_name = w_params.node_with_weight.node_name - config = node_name_to_config.get(node_name) - if config is not None and not self.is_weight_compression_supported(w_params.weight_dtype, config.mode): - unsupported_node_names.add(node_name) + for w_params in all_weight_params + skipped_weight_params: + if w_params.weight_name not in weight_name_to_config_mapping: + continue + config = weight_name_to_config_mapping[w_params.weight_name] + if not self.is_weight_compression_supported(w_params.weight_dtype, config.mode): + unsupported_weight_names.add(w_params.weight_name) + unsupported_node_names.add(w_params.node_with_weight.node_name) if unsupported_node_names: nncf_logger.warning( "The compression mode defined by the custom annotation is not supported for the data type of the " @@ -1270,31 +1296,30 @@ def _drop_annotation_configs_that_can_not_be_assigned( + "\n\t".join(sorted(unsupported_node_names)) ) - dropped_node_names = set() - dropped_node_names.update(node_names_without_weight, unsupported_node_names) - return { - node_name: config - for node_name, config in node_name_to_config.items() - if node_name not in dropped_node_names + weight_name_to_config_mapping = { + weight_name: config + for weight_name, config in weight_name_to_config_mapping.items() + if weight_name not in unsupported_weight_names } + return weight_name_to_config_mapping + @staticmethod def _assign_annotation_configs( - self, - node_name_to_config: dict[str, WeightCompressionConfig], + weight_name_to_config_mapping: dict[str, WeightCompressionConfig], all_weight_params: list[WeightCompressionParameters], ratio_defining_params: list[WeightCompressionParameters], ) -> list[WeightCompressionParameters]: """ Assigns the annotated configurations to the matched weight parameters. - :param node_name_to_config: A mapping from a node name to the annotated compression configuration. + :param weight_name_to_config_mapping: A mapping from a weight name to the annotated compression configuration. :param all_weight_params: List of all weight compression parameters that can be compressed. :param ratio_defining_params: List of ratio-defining parameters. :return: The updated list of ratio-defining parameters. """ annotated_weight_names = set() for w_params in all_weight_params: - config = node_name_to_config.get(w_params.node_with_weight.node_name) + config = weight_name_to_config_mapping.get(w_params.weight_name) if config is not None: # A deep copy is required since some algorithms, e.g. codebook estimation, update the compression # config of a weight parameter in place. @@ -1306,21 +1331,27 @@ def _assign_annotation_configs( return [w_params for w_params in ratio_defining_params if w_params.weight_name not in annotated_weight_names] @staticmethod - def _move_annotated_ignored_scope_params_from_skipped_to_compressed( - node_name_to_config: dict[str, WeightCompressionConfig], + def _move_annotated_ignored_nodes_from_skipped_to_compressed( + weight_name_to_config_mapping: dict[str, WeightCompressionConfig], graph: NNCFGraph, all_weight_params: list[WeightCompressionParameters], skipped_weight_params: list[WeightCompressionParameters], ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters]]: """ Moves the annotated weight parameters that are not compressed, e.g. the ignored ones, from the skipped - parameters to the compressed ones, so that the annotation takes precedence over the ignored scope. + parameters to the compressed ones, so that custom annotation takes precedence over the ignored scope. + + :param weight_name_to_config_mapping: A mapping from a weight name to the annotated compression configuration. + :param graph: NNCFGraph instance. + :param all_weight_params: List of all weight compression parameters that can be compressed. + :param skipped_weight_params: List of weight compression parameters that are not compressed. + :return: The updated tuple of all and skipped weight compression parameters. """ restored_weight_names = set() restored_weight_params = [] for w_params in skipped_weight_params: # A weight is compressed under a single node, so a shared weight is restored only once - is_annotated = w_params.node_with_weight.node_name in node_name_to_config + is_annotated = w_params.weight_name in weight_name_to_config_mapping if is_annotated and w_params.weight_name not in restored_weight_names: restored_weight_names.add(w_params.weight_name) restored_weight_params.append(w_params) @@ -1337,9 +1368,9 @@ def _move_annotated_ignored_scope_params_from_skipped_to_compressed( w_params for w_params in skipped_weight_params if w_params.weight_name not in restored_weight_names ] # Keep the topological order of the weight parameters - node_name_to_position = {node.node_name: i for i, node in enumerate(graph.topological_sort())} + node_name_to_position_mapping = {node.node_name: i for i, node in enumerate(graph.topological_sort())} all_weight_params = all_weight_params + restored_weight_params - all_weight_params.sort(key=lambda wp: node_name_to_position[wp.node_with_weight.node_name]) + all_weight_params.sort(key=lambda wp: node_name_to_position_mapping[wp.node_with_weight.node_name]) return all_weight_params, skipped_weight_params def apply_custom_annotation( @@ -1356,10 +1387,9 @@ def apply_custom_annotation( """ Applies the user-defined weight compression configurations to the parameters matched by the annotation. The flow of this function is as follows: - 1. Convert the annotation to a mapping from a node name to the configuration, the last match wins. - The nodes sharing a weight are annotated as a single one, since the weight is compressed only once. + 1. Convert the annotation to a mapping from a weight name to the configuration. 2. Drop the annotated configurations that can not be assigned to a weight. - 3. Move the annotated weight parameters that are not compressed, e.g. the ignored ones, to the + 3. Move the annotated weight parameters that are not compressed initially, e.g. the ignored ones, to the compressed ones. 4. Assign the remaining configurations to the matched weight parameters. 5. Apply the group size fallback mode to the annotated group size values. @@ -1373,29 +1403,29 @@ def apply_custom_annotation( if not self._custom_annotation: return all_weight_params, ratio_defining_params, skipped_weight_params - node_name_to_owner_name = self._get_node_name_to_owner_name(graph, all_weight_params) - node_name_to_config = self._get_custom_annotation_configs( - self._custom_annotation, graph, node_name_to_owner_name - ) - node_name_to_config = self._drop_annotation_configs_that_can_not_be_assigned( - node_name_to_config, graph, all_weight_params, skipped_weight_params + weight_name_to_config_mapping = self._get_custom_annotation_configs(self._custom_annotation, graph) + weight_name_to_config_mapping = self._drop_annotation_configs_that_can_not_be_assigned( + weight_name_to_config_mapping, all_weight_params, skipped_weight_params ) - all_weight_params, skipped_weight_params = self._move_annotated_ignored_scope_params_from_skipped_to_compressed( - node_name_to_config, graph, all_weight_params, skipped_weight_params + # There might be some nodes which were ignored but a custom annotation was passed for them. + # Such nodes are stored in skipped_weight_params. They should be assigned a config from the custom + # annotation if available. + all_weight_params, skipped_weight_params = self._move_annotated_ignored_nodes_from_skipped_to_compressed( + weight_name_to_config_mapping, graph, all_weight_params, skipped_weight_params ) ratio_defining_params = self._assign_annotation_configs( - node_name_to_config, all_weight_params, ratio_defining_params + weight_name_to_config_mapping, all_weight_params, ratio_defining_params ) # The resulting group size values are validated together with the rest of the weights in the apply() method all_weight_params, skipped_weight_params = self._handle_group_size_fallback( - node_name_to_config, all_weight_params, skipped_weight_params + weight_name_to_config_mapping, all_weight_params, skipped_weight_params ) return all_weight_params, ratio_defining_params, skipped_weight_params def _handle_group_size_fallback( self, - node_name_to_config: dict[str, WeightCompressionConfig], + weight_name_to_config_mapping: dict[str, WeightCompressionConfig], all_weight_params: list[WeightCompressionParameters], skipped_weight_params: list[WeightCompressionParameters], ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters]]: @@ -1405,13 +1435,13 @@ def _handle_group_size_fallback( The group size of an annotated weight is defined by the user, so the fallback mode is applied to that value instead of the group size configured for the algorithm. - :param node_name_to_config: A mapping from a node name to the applied annotation configuration. + :param weight_name_to_config_mapping: A mapping from a weight name to the applied annotation configuration. :param all_weight_params: List of all weight compression parameters that can be compressed. :param skipped_weight_params: List of weight compression parameters that are not compressed. :return: The updated tuple of all and skipped weight compression parameters. """ annotated_weight_params = [ - w_params for w_params in all_weight_params if w_params.node_with_weight.node_name in node_name_to_config + w_params for w_params in all_weight_params if w_params.weight_name in weight_name_to_config_mapping ] group_sizes = self._get_requested_group_sizes(annotated_weight_params) if self._group_size_fallback_mode == GroupSizeFallbackMode.IGNORE: From 481cc681fb31bb6d0e577fe8068d8faa08b3c493 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 23 Sep 2026 21:11:26 +0400 Subject: [PATCH 27/43] fix refs --- .../template_test_weights_compression.py | 28 +++++++++++++++++++ ...ht_ignored_other_node_annotated_first.json | 7 +++++ ...ght_ignored_other_node_annotated_last.json | 7 +++++ .../group_size_fallback_adjust.json | 0 .../group_size_fallback_ignore.json | 0 .../ignored_scope.json | 0 .../mixed_precision.json | 0 .../overlapping_annotations.json | 0 .../primary_and_backup_precision.json | 0 .../shared_weight.json | 0 ...shared_weight_annotated_by_other_node.json | 0 ...ht_ignored_other_node_annotated_first.json | 17 +++++++++++ ...ght_ignored_other_node_annotated_last.json | 17 +++++++++++ ...red_weight_other_node_annotated_first.json | 0 ...ared_weight_other_node_annotated_last.json | 0 .../shared_weight_with_ignored_scope.json | 0 ...ht_ignored_other_node_annotated_first.json | 12 ++++++++ ...ght_ignored_other_node_annotated_last.json | 12 ++++++++ ...ht_ignored_other_node_annotated_first.json | 12 ++++++++ ...ght_ignored_other_node_annotated_last.json | 12 ++++++++ 20 files changed, 124 insertions(+) create mode 100644 tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json create mode 100644 tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/group_size_fallback_adjust.json (100%) rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/group_size_fallback_ignore.json (100%) rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/ignored_scope.json (100%) rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/mixed_precision.json (100%) rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/overlapping_annotations.json (100%) rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/primary_and_backup_precision.json (100%) rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/shared_weight.json (100%) rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/shared_weight_annotated_by_other_node.json (100%) create mode 100644 tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json create mode 100644 tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/shared_weight_other_node_annotated_first.json (100%) rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/shared_weight_other_node_annotated_last.json (100%) rename tests/openvino/native/data/{2026.4 => 2026.1}/references_custom_annotation/shared_weight_with_ignored_scope.json (100%) create mode 100644 tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json create mode 100644 tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json create mode 100644 tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json create mode 100644 tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index ef834f8e3b8..0ca32c449c1 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -1206,6 +1206,34 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa None, id="shared_weight_other_node_annotated_first", ), + # The nodes sharing a weight are annotated as a single one even if none of them is compressed, + # so that the last annotation takes precedence no matter which of the nodes it matches + pytest.param( + "shared_weight", + dict(ratio=1.0, all_layers=True), + [ + ( + nncf.CustomAnnotationScope(patterns=[".*"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ), + (None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2)), + ], + IgnoredScope(patterns=[".*"]), + id="shared_weight_ignored_other_node_annotated_last", + ), + pytest.param( + "shared_weight", + dict(ratio=1.0, all_layers=True), + [ + (None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2)), + ( + nncf.CustomAnnotationScope(patterns=[".*"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ), + ], + IgnoredScope(patterns=[".*"]), + id="shared_weight_ignored_other_node_annotated_first", + ), ], ) def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, request): diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json b/tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json new file mode 100644 index 00000000000..da4ffa19ee6 --- /dev/null +++ b/tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json @@ -0,0 +1,7 @@ +{ + "MatMul_0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json b/tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json new file mode 100644 index 00000000000..22e136ab8aa --- /dev/null +++ b/tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json @@ -0,0 +1,7 @@ +{ + "MatMul_0": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_adjust.json b/tests/openvino/native/data/2026.1/references_custom_annotation/group_size_fallback_adjust.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_adjust.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/group_size_fallback_adjust.json diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_ignore.json b/tests/openvino/native/data/2026.1/references_custom_annotation/group_size_fallback_ignore.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/group_size_fallback_ignore.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/group_size_fallback_ignore.json diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/ignored_scope.json b/tests/openvino/native/data/2026.1/references_custom_annotation/ignored_scope.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/ignored_scope.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/ignored_scope.json diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/mixed_precision.json b/tests/openvino/native/data/2026.1/references_custom_annotation/mixed_precision.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/mixed_precision.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/mixed_precision.json diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json b/tests/openvino/native/data/2026.1/references_custom_annotation/overlapping_annotations.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/overlapping_annotations.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/overlapping_annotations.json diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/primary_and_backup_precision.json b/tests/openvino/native/data/2026.1/references_custom_annotation/primary_and_backup_precision.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/primary_and_backup_precision.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/primary_and_backup_precision.json diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight.json b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight.json diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_annotated_by_other_node.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_annotated_by_other_node.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_annotated_by_other_node.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json new file mode 100644 index 00000000000..f2de085261e --- /dev/null +++ b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json @@ -0,0 +1,17 @@ +{ + "Gather_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "Gather_2": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json new file mode 100644 index 00000000000..1ffe64b9a93 --- /dev/null +++ b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json @@ -0,0 +1,17 @@ +{ + "Gather_1": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + }, + "Gather_2": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_first.json b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_other_node_annotated_first.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_first.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_other_node_annotated_first.json diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_last.json b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_other_node_annotated_last.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_other_node_annotated_last.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_other_node_annotated_last.json diff --git a/tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_with_ignored_scope.json b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_scope.json similarity index 100% rename from tests/openvino/native/data/2026.4/references_custom_annotation/shared_weight_with_ignored_scope.json rename to tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_scope.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json new file mode 100644 index 00000000000..4f4f385cf33 --- /dev/null +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json @@ -0,0 +1,12 @@ +{ + "wte/embedding/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "linear/linear/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json new file mode 100644 index 00000000000..6062e22f745 --- /dev/null +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json @@ -0,0 +1,12 @@ +{ + "wte/embedding/0": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + }, + "linear/linear/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json b/tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json new file mode 100644 index 00000000000..481a5fdb4b0 --- /dev/null +++ b/tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json @@ -0,0 +1,12 @@ +{ + "embedding": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "linear": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json b/tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json new file mode 100644 index 00000000000..641de66de68 --- /dev/null +++ b/tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json @@ -0,0 +1,12 @@ +{ + "embedding": { + "mode": "int4_asym", + "group_size": 2, + "codebook_values": null + }, + "linear": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file From 2bc0e201fe93bdc6e9f5b8766e10da46ec1f6b2d Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 23 Sep 2026 23:22:22 +0400 Subject: [PATCH 28/43] fix docstring --- .../post_training_compression/weights_compression/Usage.md | 2 +- src/nncf/quantization/advanced_parameters.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/usage/post_training_compression/weights_compression/Usage.md b/docs/usage/post_training_compression/weights_compression/Usage.md index 2178101a34a..1d85d99909e 100644 --- a/docs/usage/post_training_compression/weights_compression/Usage.md +++ b/docs/usage/post_training_compression/weights_compression/Usage.md @@ -141,7 +141,7 @@ compressed_model = compress_weights( layer. A layer that is matched by both the `ignored_scope` and a custom annotation is compressed with the user-defined configuration and a warning is logged. If several annotations match the same layer, the last one takes precedence. A weight shared by several layers is compressed once, so annotating any of them applies the - configuration to this weight. + configuration to this weight. The codebook compression modes are not supported by an annotation. #### Data-aware methods diff --git a/src/nncf/quantization/advanced_parameters.py b/src/nncf/quantization/advanced_parameters.py index f4c68d8bd5c..108df9b2885 100644 --- a/src/nncf/quantization/advanced_parameters.py +++ b/src/nncf/quantization/advanced_parameters.py @@ -457,10 +457,9 @@ class AdvancedCompressionParameters: @dataclass class CustomAnnotation: """ - Binds a user-defined weight compression configuration to a portion of a model. + Applies a user-defined weight compression configuration to a portion of a model. It overrides the configuration assigned by the weight compression algorithm, including the mixed precision algorithm and the `ignored_scope`, `all_layers` and `backup_mode` options. - If several annotations match the same node, the last one in the `custom_annotation` list wins. Example: From 0f35160a8f8d7e09f90676933579e1d73449e554 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 23 Sep 2026 23:36:15 +0400 Subject: [PATCH 29/43] remove support for codebook; support edge case with shared weight where 1 node is annotated by ignored scope and another by custom annotation --- .../weight_compression/algorithm.py | 23 +++++++---- .../template_test_weights_compression.py | 38 +++++++++++++++++-- ...d_weight_with_ignored_compressed_node.json | 7 ++++ .../quantization/test_weights_compression.py | 4 ++ ...d_weight_with_ignored_compressed_node.json | 17 +++++++++ .../quantization/test_weights_compression.py | 4 ++ ...d_weight_with_ignored_compressed_node.json | 12 ++++++ ...d_weight_with_ignored_compressed_node.json | 12 ++++++ .../quantization/test_weights_compression.py | 4 ++ tests/torch/fx/test_weights_compression.py | 4 ++ 10 files changed, 113 insertions(+), 12 deletions(-) create mode 100644 tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_compressed_node.json create mode 100644 tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_compressed_node.json create mode 100644 tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_compressed_node.json create mode 100644 tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_compressed_node.json diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 49d8860fb90..1f4b2ec525e 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -179,12 +179,19 @@ def validate_custom_annotation(custom_annotation: list[CustomAnnotation] | None) ) raise nncf.ValidationError(msg) - if annotation.config.is_codebook and annotation.config.codebook_values is None: + if annotation.config.is_codebook: msg = ( - f"Codebook values must be provided in the custom annotation config for the " - f"{annotation.config.mode.value} compression mode." + "Codebook compression modes are not supported by the custom annotation, but the " + f"{annotation.config.mode.value} mode is given." ) - raise nncf.ValidationError(msg) + raise nncf.ParameterNotSupportedError(msg) + + if annotation.config.codebook_values is not None: + msg = ( + "Codebook values are not supported by the custom annotation, but they are given for the " + f"{annotation.config.mode.value} mode." + ) + raise nncf.ParameterNotSupportedError(msg) if annotation.config.mode in INT8_MODES and annotation.config.group_size != -1: msg = ( @@ -1347,13 +1354,12 @@ def _move_annotated_ignored_nodes_from_skipped_to_compressed( :param skipped_weight_params: List of weight compression parameters that are not compressed. :return: The updated tuple of all and skipped weight compression parameters. """ - restored_weight_names = set() + compressed_weight_names = {w_params.weight_name for w_params in all_weight_params} restored_weight_params = [] for w_params in skipped_weight_params: - # A weight is compressed under a single node, so a shared weight is restored only once is_annotated = w_params.weight_name in weight_name_to_config_mapping - if is_annotated and w_params.weight_name not in restored_weight_names: - restored_weight_names.add(w_params.weight_name) + if is_annotated and w_params.weight_name not in compressed_weight_names: + compressed_weight_names.add(w_params.weight_name) restored_weight_params.append(w_params) if not restored_weight_params: @@ -1364,6 +1370,7 @@ def _move_annotated_ignored_nodes_from_skipped_to_compressed( "by the custom annotation. They will be compressed with the user-defined configuration:\n\t" + "\n\t".join(sorted(w_params.node_with_weight.node_name for w_params in restored_weight_params)) ) + restored_weight_names = {w_params.weight_name for w_params in restored_weight_params} skipped_weight_params = [ w_params for w_params in skipped_weight_params if w_params.weight_name not in restored_weight_names ] diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 0ca32c449c1..708ce2e7667 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -1027,6 +1027,13 @@ def get_shared_weight_node_name() -> str: node sharing it. """ + @staticmethod + @abstractmethod + def get_node_with_shared_weight_name() -> str: + """ + Returns the name of the node from `get_shared_weight_model` that the shared weight is compressed under, + """ + def _compress_and_get_configs(self, **kwargs) -> dict[str, dict[str, Any]]: """ Compresses a model and returns the compression config assigned to each compressed weight node. @@ -1234,6 +1241,19 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa IgnoredScope(patterns=[".*"]), id="shared_weight_ignored_other_node_annotated_first", ), + # The weight is compressed under the node that is not ignored, so the ignored one is not restored + pytest.param( + "shared_weight", + dict(ratio=1.0, all_layers=True), + [ + ( + nncf.CustomAnnotationScope(patterns=[".*"]), + WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + ) + ], + None, + id="shared_weight_with_ignored_compressed_node", + ), ], ) def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, request): @@ -1243,6 +1263,7 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, Set the NNCF_TEST_REGEN_DOT environment variable to regenerate the reference file. """ + test_id = request.node.callspec.id if model_name == "sequential_matmul": model = self.get_sequential_matmul_model(transpose_a=False) else: @@ -1253,6 +1274,8 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, nncf.CustomAnnotation(scope=shared_weight_scope if scope is None else scope, config=config) for scope, config in annotations ] + if test_id == "shared_weight_with_ignored_compressed_node": + ignored_scope = IgnoredScope(names=[self.get_node_with_shared_weight_name()]) configs = self._compress_and_get_configs( model=model, @@ -1263,7 +1286,7 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, custom_annotation=custom_annotation, ) - ref_path = self.get_custom_annotation_ref_path(request.node.callspec.id) + ref_path = self.get_custom_annotation_ref_path(test_id) if os.getenv("NNCF_TEST_REGEN_DOT") is not None: dump_to_json(ref_path, configs) @@ -1342,14 +1365,21 @@ def test_custom_annotation_warning(self, custom_annotation, ignored_scope, warni pytest.param(["anything"], nncf.ValidationError, id="not_an_annotation"), pytest.param([nncf.CustomAnnotation(scope=nncf.IgnoredScope())], nncf.ValidationError, id="wrong_scope"), pytest.param([nncf.CustomAnnotation(config="anything")], nncf.ValidationError, id="wrong_config"), + pytest.param( + [nncf.CustomAnnotation(config=WeightCompressionConfig(mode=CompressWeightsMode.CODEBOOK))], + nncf.ParameterNotSupportedError, + id="codebook_mode", + ), pytest.param( [ nncf.CustomAnnotation( - config=WeightCompressionConfig(mode=CompressWeightsMode.CODEBOOK, codebook_values=None) + config=WeightCompressionConfig( + mode=CompressWeightsMode.INT4_SYM, group_size=-1, codebook_values=Tensor(CB4_QUANTILES) + ) ) ], - nncf.ValidationError, - id="codebook_without_values", + nncf.ParameterNotSupportedError, + id="codebook_values", ), pytest.param( [ diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_compressed_node.json b/tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_compressed_node.json new file mode 100644 index 00000000000..b178e4b3f07 --- /dev/null +++ b/tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_compressed_node.json @@ -0,0 +1,7 @@ +{ + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/onnx/quantization/test_weights_compression.py b/tests/onnx/quantization/test_weights_compression.py index b0ba45b0278..de780dee663 100644 --- a/tests/onnx/quantization/test_weights_compression.py +++ b/tests/onnx/quantization/test_weights_compression.py @@ -753,6 +753,10 @@ def get_shared_weight_model() -> onnx.ModelProto: def get_shared_weight_node_name() -> str: return "MatMul_1" + @staticmethod + def get_node_with_shared_weight_name() -> str: + return "MatMul_0" + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_compressed_node.json b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_compressed_node.json new file mode 100644 index 00000000000..1f85fd8f6fb --- /dev/null +++ b/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_compressed_node.json @@ -0,0 +1,17 @@ +{ + "Gather_2": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "MatMul_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "MatMul_2": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index b76a24726c9..2f597f939f5 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -2555,6 +2555,10 @@ def get_shared_weight_model() -> ov.Model: def get_shared_weight_node_name() -> str: return "MatMul_2" + @staticmethod + def get_node_with_shared_weight_name() -> str: + return "Gather_1" + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return get_actual_reference_for_current_openvino(CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json") diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_compressed_node.json b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_compressed_node.json new file mode 100644 index 00000000000..c33c0f83cab --- /dev/null +++ b/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_compressed_node.json @@ -0,0 +1,12 @@ +{ + "linear/linear/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "lm_head/linear/0": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_compressed_node.json b/tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_compressed_node.json new file mode 100644 index 00000000000..25055fddf1d --- /dev/null +++ b/tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_compressed_node.json @@ -0,0 +1,12 @@ +{ + "linear": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + }, + "linear_1": { + "mode": "int8_sym", + "group_size": -1, + "codebook_values": null + } +} \ No newline at end of file diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index 31bab6c3e14..da881011789 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -736,6 +736,10 @@ def get_shared_weight_model() -> GraphModelWrapper: def get_shared_weight_node_name() -> str: return "lm_head/linear/0" + @staticmethod + def get_node_with_shared_weight_name() -> str: + return "wte/embedding/0" + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" diff --git a/tests/torch/fx/test_weights_compression.py b/tests/torch/fx/test_weights_compression.py index dc022d5e298..d01c6b1e9a4 100644 --- a/tests/torch/fx/test_weights_compression.py +++ b/tests/torch/fx/test_weights_compression.py @@ -523,6 +523,10 @@ def get_shared_weight_model() -> torch.fx.GraphModule: def get_shared_weight_node_name() -> str: return "linear_1" + @staticmethod + def get_node_with_shared_weight_name() -> str: + return "embedding" + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" From 101899965d36d81d9a3a43330628890df0f4f248 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Wed, 23 Sep 2026 23:38:10 +0400 Subject: [PATCH 30/43] update comments --- .../quantization/algorithms/weight_compression/algorithm.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 1f4b2ec525e..f4308840a70 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -1328,8 +1328,7 @@ def _assign_annotation_configs( for w_params in all_weight_params: config = weight_name_to_config_mapping.get(w_params.weight_name) if config is not None: - # A deep copy is required since some algorithms, e.g. codebook estimation, update the compression - # config of a weight parameter in place. + # A deep copy is required for some methods/modes like codebook w_params.compression_config = copy.deepcopy(config) annotated_weight_names.add(w_params.weight_name) From 78bc8b7d83a82531c0088a29668ff9ee2bdb957b Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 00:03:50 +0400 Subject: [PATCH 31/43] backend specific mode validity check --- .../weights_compression/Usage.md | 4 +- .../weight_compression/algorithm.py | 2 - src/nncf/quantization/quantize_model.py | 47 ++++++++++++------- .../template_test_weights_compression.py | 25 +++++++++- .../quantization/test_weights_compression.py | 4 ++ .../quantization/test_weights_compression.py | 4 ++ tests/torch/fx/test_weights_compression.py | 4 ++ 7 files changed, 70 insertions(+), 20 deletions(-) diff --git a/docs/usage/post_training_compression/weights_compression/Usage.md b/docs/usage/post_training_compression/weights_compression/Usage.md index 1d85d99909e..ffad3e9e0fb 100644 --- a/docs/usage/post_training_compression/weights_compression/Usage.md +++ b/docs/usage/post_training_compression/weights_compression/Usage.md @@ -141,7 +141,9 @@ compressed_model = compress_weights( layer. A layer that is matched by both the `ignored_scope` and a custom annotation is compressed with the user-defined configuration and a warning is logged. If several annotations match the same layer, the last one takes precedence. A weight shared by several layers is compressed once, so annotating any of them applies the - configuration to this weight. The codebook compression modes are not supported by an annotation. + configuration to this weight. The codebook compression modes are not supported by an annotation. The mode of an + annotation must be supported by the backend, the same as the `mode` option, e.g. NF4 can not be annotated for a + Torch, TorchFX or ONNX model. #### Data-aware methods diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index f4308840a70..d26b0fd96bb 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -218,12 +218,10 @@ def check_user_compression_configuration( backup_mode: BackupMode | None, compression_format: CompressionFormat | None, advanced_parameters: AdvancedCompressionParameters | None, - custom_annotation: list[CustomAnnotation] | None = None, ) -> None: """ Validates the user's weight compression configuration for correctness. """ - validate_custom_annotation(custom_annotation) if mode in INT8_MODES: if (ratio and ratio != 1) or (group_size and group_size != -1): msg = ( diff --git a/src/nncf/quantization/quantize_model.py b/src/nncf/quantization/quantize_model.py index af2fe483fa2..f9ae741dc14 100644 --- a/src/nncf/quantization/quantize_model.py +++ b/src/nncf/quantization/quantize_model.py @@ -37,6 +37,7 @@ from nncf.quantization.algorithms.post_training.pipeline import create_ptq_pipeline from nncf.quantization.algorithms.weight_compression.algorithm import check_user_compression_configuration from nncf.quantization.algorithms.weight_compression.algorithm import get_weight_compression_configuration +from nncf.quantization.algorithms.weight_compression.algorithm import validate_custom_annotation from nncf.quantization.telemetry_extractors import CompressionStartedWithCompressWeightsApi from nncf.quantization.telemetry_extractors import CompressionStartedWithQuantizeApi from nncf.quantization.telemetry_extractors import CompressionStartedWithQuantizeWithAccuracyControlApi @@ -394,6 +395,31 @@ def quantize_with_accuracy_control( raise nncf.UnsupportedBackendError(msg) +def _validate_compression_modes_supported_by_backend( + backend_name: str, + not_supported_modes: list[CompressWeightsMode], + mode: CompressWeightsMode, + custom_annotation: list[CustomAnnotation] | None, +) -> None: + """ + Checks that neither the compression mode nor the mode defined by the custom annotation + are unsupported by the backend. + + :param backend_name: Name of the backend to use in the error message. + :param not_supported_modes: List of the compression modes that are not supported by the backend. + :param mode: Compression mode of the algorithm. + :param custom_annotation: List of the user-defined weight compression configurations, if given. + :raises nncf.ParameterNotSupportedError: If any of the given modes is not supported by the backend. + """ + given_modes = [mode] + [annotation.config.mode for annotation in custom_annotation or []] + if any(given_mode in not_supported_modes for given_mode in given_modes): + msg = ( + f"{backend_name} backend does not support {[m.value for m in not_supported_modes]} modes " + "for weight compression." + ) + raise nncf.ParameterNotSupportedError(msg) + + @api(canonical_alias="nncf.compress_weights") @tracked_function( MODEL_BASED_CATEGORY, @@ -506,6 +532,8 @@ def compress_weights( :type custom_annotation: list[nncf.CustomAnnotation] :return: The non-trainable model with compressed weights. """ + validate_custom_annotation(custom_annotation) + backend = get_backend(model) compression_weights_impl: Callable[..., Any] | None = None @@ -526,11 +554,7 @@ def compress_weights( CompressWeightsMode.ADAPTIVE_CODEBOOK, CompressWeightsMode.CB4, ] - if mode in not_supported_modes: - msg = ( - f"Torch backend does not support {[m.value for m in not_supported_modes]} modes for weight compression." - ) - raise nncf.ParameterNotSupportedError(msg) + _validate_compression_modes_supported_by_backend("Torch", not_supported_modes, mode, custom_annotation) options = {"gptq": gptq, "lora_correction": lora_correction} unsupported_options = [name for name, value in options.items() if value is not None] @@ -576,11 +600,7 @@ def compress_weights( CompressWeightsMode.ADAPTIVE_CODEBOOK, CompressWeightsMode.CB4, ] - if mode in not_supported_modes: - msg = ( - f"Torch backend does not support {[m.value for m in not_supported_modes]} modes for weight compression." - ) - raise nncf.ParameterNotSupportedError(msg) + _validate_compression_modes_supported_by_backend("Torch", not_supported_modes, mode, custom_annotation) options = {"gptq": gptq, "lora_correction": lora_correction} unsupported_options = [name for name, value in options.items() if value is not None] @@ -647,11 +667,7 @@ def compress_weights( CompressWeightsMode.ADAPTIVE_CODEBOOK, CompressWeightsMode.CB4, ] - if mode in not_supported_modes: - msg = ( - f"ONNX backend does not support {[m.value for m in not_supported_modes]} modes for weight compression." - ) - raise nncf.ParameterNotSupportedError(msg) + _validate_compression_modes_supported_by_backend("ONNX", not_supported_modes, mode, custom_annotation) options = {"gptq": gptq, "lora_correction": lora_correction} unsupported_options = [name for name, value in options.items() if value is not None] @@ -683,7 +699,6 @@ def compress_weights( backup_mode, compression_format, advanced_parameters, - custom_annotation, ) weight_compression_configuration = get_weight_compression_configuration( mode, diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 708ce2e7667..becb18cd08e 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -1034,6 +1034,13 @@ def get_node_with_shared_weight_name() -> str: Returns the name of the node from `get_shared_weight_model` that the shared weight is compressed under, """ + @staticmethod + def get_mode_not_supported_by_backend() -> CompressWeightsMode | None: + """ + Returns a compression mode that is not supported by the backend, or None if all the modes are supported. + """ + return None + def _compress_and_get_configs(self, **kwargs) -> dict[str, dict[str, Any]]: """ Compresses a model and returns the compression config assigned to each compressed weight node. @@ -1410,12 +1417,28 @@ def test_custom_annotation_warning(self, custom_annotation, ignored_scope, warni InvalidGroupSizeError, id="invalid_group_size", ), + pytest.param( + None, + nncf.ParameterNotSupportedError, + id="mode_not_supported_by_backend", + ), ], ) - def test_invalid_custom_annotation(self, custom_annotation, error): + def test_invalid_custom_annotation(self, custom_annotation, error, request): """ Checks that an error is raised for an invalid custom annotation. """ + if request.node.callspec.id == "mode_not_supported_by_backend": + mode = self.get_mode_not_supported_by_backend() + if mode is None: + pytest.skip("The backend supports all the compression modes") + custom_annotation = [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*"]), + config=WeightCompressionConfig(mode=mode, group_size=-1), + ) + ] + with pytest.raises(error): compress_weights( model=self.get_sequential_matmul_model(transpose_a=False), diff --git a/tests/onnx/quantization/test_weights_compression.py b/tests/onnx/quantization/test_weights_compression.py index de780dee663..1204fe12301 100644 --- a/tests/onnx/quantization/test_weights_compression.py +++ b/tests/onnx/quantization/test_weights_compression.py @@ -757,6 +757,10 @@ def get_shared_weight_node_name() -> str: def get_node_with_shared_weight_name() -> str: return "MatMul_0" + @staticmethod + def get_mode_not_supported_by_backend() -> CompressWeightsMode | None: + return CompressWeightsMode.NF4 + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index da881011789..45f5af7b268 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -740,6 +740,10 @@ def get_shared_weight_node_name() -> str: def get_node_with_shared_weight_name() -> str: return "wte/embedding/0" + @staticmethod + def get_mode_not_supported_by_backend() -> CompressWeightsMode | None: + return CompressWeightsMode.NF4 + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" diff --git a/tests/torch/fx/test_weights_compression.py b/tests/torch/fx/test_weights_compression.py index d01c6b1e9a4..24ceb3da252 100644 --- a/tests/torch/fx/test_weights_compression.py +++ b/tests/torch/fx/test_weights_compression.py @@ -527,6 +527,10 @@ def get_shared_weight_node_name() -> str: def get_node_with_shared_weight_name() -> str: return "embedding" + @staticmethod + def get_mode_not_supported_by_backend() -> CompressWeightsMode | None: + return CompressWeightsMode.NF4 + @staticmethod def get_custom_annotation_ref_path(ref_name: str) -> Path: return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" From 0366367baebca058b1453e8184ece81cf8d48243 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 11:05:08 +0400 Subject: [PATCH 32/43] support backup mode none with data aware compression --- .../weight_compression/algorithm.py | 13 +++++++- .../template_test_weights_compression.py | 30 ++++++++++++++++--- .../quantization/test_weights_compression.py | 18 ++++++++++- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index d26b0fd96bb..44b290310e7 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -1019,7 +1019,18 @@ def collect_statistics_and_statistic_points( """ if not dataset or not (self._data_aware_mixed_precision or self._data_aware_compression): return None, statistic_points - weight_params = ratio_defining_params if self._backup_mode == BackupMode.NONE else all_weight_params + weight_params = all_weight_params + if self._backup_mode == BackupMode.NONE: + # Only the compressed weights need statistics. When backup mode is none, only ratio defining params are + # considered. In the case of custom annotation, some weights may be considered backup nodes, but they + # may also be annotated. This ensures that such annotated nodes are also considered for statistics + # collection. + ratio_defining_weight_names = set(wp.weight_name for wp in ratio_defining_params) + weight_params = ratio_defining_params + [ + wp + for wp in all_weight_params + if wp.weight_name not in ratio_defining_weight_names and wp.compression_config is not None + ] matmul_nodes_to_compress = [ wp.node_with_weight for wp in weight_params diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index becb18cd08e..e3be036c7f1 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -25,6 +25,7 @@ import nncf import nncf.tensor.functions as fns +from nncf import BackupMode from nncf import CompressWeightsMode from nncf import SensitivityMetric from nncf import nncf_logger @@ -346,9 +347,10 @@ def get_orig_weight(model: TModel) -> Tensor: def get_decompressed_weight(compressed_model: TModel, input: TTensor) -> Tensor: """Returns decompressed weight""" + @pytest.mark.parametrize("backup_mode", [BackupMode.INT8_ASYM, BackupMode.NONE], ids=["int8_backup", "no_backup"]) @pytest.mark.parametrize("annotated_mode", [None, CompressWeightsMode.INT4_SYM], ids=["default", "annotated"]) @pytest.mark.parametrize("transpose_a", [False, True], ids=["no_tr_a", "tr_a"]) - def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a, annotated_mode): + def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a, annotated_mode, backup_mode): """ Checks that act_ch_axis parameter and the config defined by the custom annotation are passed to calculate_quantization_params. @@ -377,6 +379,7 @@ def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a, annotated group_size=8, scale_estimation=True, all_layers=True, + backup_mode=backup_mode, dataset=dataset, custom_annotation=custom_annotation, ) @@ -668,17 +671,36 @@ def int4_mode(self, request): @pytest.mark.parametrize("is_3d_weights", [True, False]) @pytest.mark.parametrize("with_multiply", (True, False)) + @pytest.mark.parametrize("annotate_all", [False, True], ids=["default", "annotated"]) def test_call_max_var_criterion_with_dataset_by_default_awq_act_matmul( - self, int4_mode, with_multiply, is_3d_weights, mocker + self, int4_mode, with_multiply, is_3d_weights, annotate_all, mocker ): n_layers = 8 - n_awq_target = n_layers - 1 # first MatMul is always int8 + # The first MatMul is always int8, unless the custom annotation assigns 4 bits to it + n_awq_target = n_layers if annotate_all else n_layers - 1 model = self.get_awq_act_model(is_3d_weights, with_multiply, n_layers) dataset = Dataset([self.to_tensor(np.ones([2, 8, 8], dtype=np.float32))], self.get_transform_func()) + custom_annotation = None + if annotate_all: + custom_annotation = [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(patterns=[".*"]), + config=WeightCompressionConfig(mode=int4_mode, group_size=2), + ) + ] + with SpyWeightCompressionStatisticsContext(mocker): - model = compress_weights(model, mode=int4_mode, ratio=1.0, group_size=2, dataset=dataset, awq=True) + model = compress_weights( + model, + mode=int4_mode, + ratio=1.0, + group_size=2, + dataset=dataset, + awq=True, + custom_annotation=custom_annotation, + ) awq_num = self.get_num_multiply_from_awq(model) assert awq_num == n_awq_target diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 2f597f939f5..10fc20a6b93 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -1961,6 +1961,7 @@ def test_data_free_compression_with_backup_mode(backup_mode): assert act_num == num_compressed +@pytest.mark.parametrize("annotate", [False, True], ids=["default", "annotated"]) @pytest.mark.parametrize("backup_mode", [BackupMode.NONE, BackupMode.INT8_ASYM, BackupMode.INT8_SYM]) @pytest.mark.parametrize( ("params", "num_compressed"), @@ -1977,12 +1978,22 @@ def test_data_free_compression_with_backup_mode(backup_mode): ({"awq": True}, 6), ), ) -def test_data_based_compression_with_backup_mode(backup_mode, params, num_compressed): +def test_data_based_compression_with_backup_mode(backup_mode, params, num_compressed, annotate): model = AWQMatmulModel().ov_model sz = 8 n_samples = 10 dataset = Dataset([np.ones([1, i + 1, sz]) for i in range(n_samples)]) + custom_annotation = None + if annotate: + custom_annotation = [ + nncf.CustomAnnotation( + scope=nncf.CustomAnnotationScope(names=["MatMul_1"]), + config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_SYM, group_size=-1), + ) + ] + num_compressed = 6 + compressed_model = compress_weights( model, mode=CompressWeightsMode.INT4_ASYM, @@ -1990,9 +2001,11 @@ def test_data_based_compression_with_backup_mode(backup_mode, params, num_compre group_size=-1, dataset=dataset, backup_mode=backup_mode, + custom_annotation=custom_annotation, **params, ) act_num = 0 + annotated_num = 0 if backup_mode == BackupMode.INT8_ASYM: backup_ov_mode = ov.Type.u8 elif backup_mode == BackupMode.INT8_SYM: @@ -2003,6 +2016,8 @@ def test_data_based_compression_with_backup_mode(backup_mode, params, num_compre if op.get_type_name() == "Constant": if op.get_element_type() == ov.Type.u4: act_num += 1 + elif op.get_element_type() == ov.Type.i4: + annotated_num += 1 elif "/scale" in op.get_friendly_name(): assert op.get_element_type() == ov.Type.f16 elif "_lora_" in op.get_friendly_name(): @@ -2010,6 +2025,7 @@ def test_data_based_compression_with_backup_mode(backup_mode, params, num_compre else: assert op.get_element_type() == backup_ov_mode assert act_num == num_compressed + assert annotated_num == (1 if annotate else 0) @pytest.mark.parametrize( From ee2f43721fa537a1b5855ab32b840d1305e91dee Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 11:24:09 +0400 Subject: [PATCH 33/43] remove remaining codebook support --- src/nncf/openvino/rt_info.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/nncf/openvino/rt_info.py b/src/nncf/openvino/rt_info.py index cb1f81dbcdf..1cc755bf37e 100644 --- a/src/nncf/openvino/rt_info.py +++ b/src/nncf/openvino/rt_info.py @@ -38,15 +38,11 @@ def exclude_empty_fields(value: dict[str, Any]) -> dict[str, Any]: def compression_config_to_dict(config: WeightCompressionConfig) -> dict[str, Any]: """ Converts a weight compression config into a dictionary suitable for dumping into Model's meta section. - Codebook values are saved as their size since the values themselves are too large to be dumped. :param config: Weight compression config. :return: Dictionary with the compression config parameters. """ - value: dict[str, Any] = {"mode": config.mode.value, "group_size": config.group_size} - if config.codebook_values is not None: - value["codebook_size"] = config.codebook_values.size - return value + return {"mode": config.mode.value, "group_size": config.group_size} def dump_parameters( From a231b6a0981b12be643f0b66ee0195efd268b729 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 12:02:46 +0400 Subject: [PATCH 34/43] validate group sizes correctly for mx, nv datatypes --- .../weight_compression/algorithm.py | 49 +++++++++++++------ .../template_test_weights_compression.py | 2 +- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 44b290310e7..a436d73b032 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -63,6 +63,11 @@ INT8_MODES = [CompressWeightsMode.INT8_ASYM, CompressWeightsMode.INT8_SYM] NON_INT8_MODES = [mode for mode in CompressWeightsMode if mode not in INT8_MODES] +FIXED_GROUP_SIZE_MODES = { + CompressWeightsMode.MXFP4: 32, + CompressWeightsMode.MXFP8_E4M3: 32, + CompressWeightsMode.NVFP4: 16, +} SUPPORTED_DATA_TYPES = [ TensorDataType.float16, TensorDataType.bfloat16, @@ -141,6 +146,19 @@ def get_weight_compression_configuration( } +def validate_fixed_group_size(mode: CompressWeightsMode, group_size: int | None) -> None: + """ + Validates the group size requested for a mode which accepts only specific group sizes. + + :param mode: Compression mode to validate. + :param group_size: Group size requested for the given mode. None means that the default value is used. + """ + fixed_group_size = FIXED_GROUP_SIZE_MODES.get(mode) + if fixed_group_size is not None and group_size not in [None, fixed_group_size]: + msg = f"{mode.value} type only supports group size of {fixed_group_size}, group size of {group_size} is given" + raise nncf.ValidationError(msg) + + def validate_custom_annotation(custom_annotation: list[CustomAnnotation] | None) -> None: """ Validates the user-defined custom annotation. @@ -201,6 +219,8 @@ def validate_custom_annotation(custom_annotation: list[CustomAnnotation] | None) ) raise nncf.ParameterNotSupportedError(msg) + validate_fixed_group_size(annotation.config.mode, annotation.config.group_size) + def check_user_compression_configuration( mode: CompressWeightsMode, @@ -342,22 +362,19 @@ def check_user_compression_configuration( f"Supported modes are: {[e.value for e in GroupSizeFallbackMode]}." ) raise nncf.ValidationError(msg) - if mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3, CompressWeightsMode.NVFP4]: - if mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3] and group_size not in [None, 32]: - msg = f"MXFP4 and MXFP8_E4M3 types only support group size of 32, group size of {group_size} is given" - raise nncf.ValidationError(msg) - - if mode == CompressWeightsMode.NVFP4 and group_size not in [None, 16]: - msg = f"NVFP4 type only supports group size of 16, group size of {group_size} is given" - raise nncf.ValidationError(msg) - if advanced_parameters and advanced_parameters.group_size_fallback_mode is GroupSizeFallbackMode.ADJUST: - msg = ( - "MXFP4, MXFP8_E4M3 and NVFP4 types do not support the group size" - f" fallback mode {advanced_parameters.group_size_fallback_mode.value}." - " Please use other group size fallback mode." - ) - raise nncf.ValidationError(msg) + validate_fixed_group_size(mode, group_size) + if ( + mode in FIXED_GROUP_SIZE_MODES + and advanced_parameters + and advanced_parameters.group_size_fallback_mode is GroupSizeFallbackMode.ADJUST + ): + msg = ( + "MXFP4, MXFP8_E4M3 and NVFP4 types do not support the group size" + f" fallback mode {advanced_parameters.group_size_fallback_mode.value}." + " Please use other group size fallback mode." + ) + raise nncf.ValidationError(msg) class WeightCompression(Algorithm): @@ -745,7 +762,7 @@ def validate_group_size( for name, channel_size, group_size in failed_nodes ) msg = ( - f"Failed to apply group-wise quantization with group size value {self._group_size}.\n" + "Failed to apply group-wise quantization with the requested group size value.\n" "Ensure that the group size is divisible by the channel size, " "or consider setting `group_size_fallback_mode` to IGNORE or ADJUST. Failed nodes:\n\t" + names ) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index e3be036c7f1..38815921fe2 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -950,7 +950,7 @@ def test_error_message_for_invalid_group_size( with pytest.raises(InvalidGroupSizeError) as exc_info: compress_weights(**kwargs) - assert "Failed to apply group-wise quantization with group size value" in str(exc_info.value) + assert "Failed to apply group-wise quantization with the requested group size value" in str(exc_info.value) elif expected_outcome == "warn_ignored": with patch.object(nncf_logger, "warning") as mock_warning: compress_weights(**kwargs) From 73a39dcaaec9c0194f073520738ff41f449b3725 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 13:51:05 +0400 Subject: [PATCH 35/43] get the group size appropriately at WC COnfig intialization --- .../weight_compression/algorithm.py | 24 +++---------- .../algorithms/weight_compression/config.py | 35 +++++++++++++++++-- .../algorithms/weight_compression/gptq.py | 4 ++- .../weight_compression/weight_lowering.py | 2 +- .../quantization/test_weights_compression.py | 26 +++++++------- .../test_compression_functions.py | 10 +++--- 6 files changed, 59 insertions(+), 42 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index a436d73b032..c618cc2de6b 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -43,7 +43,9 @@ from nncf.quantization.advanced_parameters import convert_to_dict_recursively from nncf.quantization.algorithms.algorithm import Algorithm from nncf.quantization.algorithms.weight_compression.awq import AWQ +from nncf.quantization.algorithms.weight_compression.config import FIXED_GROUP_SIZE_MODES from nncf.quantization.algorithms.weight_compression.config import WeightCompressionParameters +from nncf.quantization.algorithms.weight_compression.config import get_default_group_size from nncf.quantization.algorithms.weight_compression.constants import CB4_QUANTILES from nncf.quantization.algorithms.weight_compression.gptq import GPTQ from nncf.quantization.algorithms.weight_compression.lora_correction import LoraCorrectionAlgorithm @@ -63,11 +65,6 @@ INT8_MODES = [CompressWeightsMode.INT8_ASYM, CompressWeightsMode.INT8_SYM] NON_INT8_MODES = [mode for mode in CompressWeightsMode if mode not in INT8_MODES] -FIXED_GROUP_SIZE_MODES = { - CompressWeightsMode.MXFP4: 32, - CompressWeightsMode.MXFP8_E4M3: 32, - CompressWeightsMode.NVFP4: 16, -} SUPPORTED_DATA_TYPES = [ TensorDataType.float16, TensorDataType.bfloat16, @@ -97,21 +94,8 @@ def get_weight_compression_configuration( """ Generates a configuration dictionary for weight compression based on the provided parameters. """ - if group_size is None and mode in INT8_MODES: - group_size = -1 - elif group_size is None and mode in NON_INT8_MODES: - if mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3]: - group_size = 32 - elif mode == CompressWeightsMode.NVFP4: - group_size = 16 - elif mode in [ - CompressWeightsMode.CODEBOOK, - CompressWeightsMode.CB4, - CompressWeightsMode.ADAPTIVE_CODEBOOK, - ]: - group_size = -1 - else: - group_size = 128 + if group_size is None: + group_size = get_default_group_size(mode) if backup_mode is None: if mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3]: diff --git a/src/nncf/quantization/algorithms/weight_compression/config.py b/src/nncf/quantization/algorithms/weight_compression/config.py index c987d0003c0..05d44e8f008 100644 --- a/src/nncf/quantization/algorithms/weight_compression/config.py +++ b/src/nncf/quantization/algorithms/weight_compression/config.py @@ -23,6 +23,34 @@ from nncf.tensor import Tensor from nncf.tensor.definitions import TensorDataType +FIXED_GROUP_SIZE_MODES = { + CompressWeightsMode.MXFP4: 32, + CompressWeightsMode.MXFP8_E4M3: 32, + CompressWeightsMode.NVFP4: 16, +} +PER_CHANNEL_BY_DEFAULT_MODES = [ + CompressWeightsMode.INT8_ASYM, + CompressWeightsMode.INT8_SYM, + CompressWeightsMode.CODEBOOK, + CompressWeightsMode.CB4, + CompressWeightsMode.ADAPTIVE_CODEBOOK, +] +DEFAULT_GROUP_SIZE = 128 + + +def get_default_group_size(mode: CompressWeightsMode) -> int: + """ + Returns the group size to use for the given mode when no group size is requested explicitly. + + :param mode: Compression mode to get the default group size for. + :return: Default group size for the given mode. + """ + if mode in FIXED_GROUP_SIZE_MODES: + return FIXED_GROUP_SIZE_MODES[mode] + if mode in PER_CHANNEL_BY_DEFAULT_MODES: + return -1 + return DEFAULT_GROUP_SIZE + @api(canonical_alias="nncf.WeightCompressionConfig") @dataclass @@ -32,17 +60,20 @@ class WeightCompressionConfig: :param mode: Defines a mode for weight compression. Defaults to INT8_ASYM mode. :param group_size: Number of weights (e.g. 128) in the channel dimension that share quantization parameters (scale). - The value -1 means no grouping. Defaults to -1. + The value -1 means no grouping. Defaults to None, which means that the default group size of the given + mode is used, e.g. -1 for the INT8 modes, 32 for MXFP4 and 128 for INT4_SYM. :param codebook_values: Optional codebook values for CODEBOOK compression mode. Must be nncf.tensor.Tensor which wraps numpy array or ov tensor. Storing ov tensor is useful for having destination data type information available. """ mode: CompressWeightsMode = CompressWeightsMode.INT8_ASYM - group_size: int = -1 + group_size: int | None = None codebook_values: Tensor | None = None def __post_init__(self) -> None: + if self.group_size is None: + self.group_size = get_default_group_size(self.mode) if self.group_size == 0 or self.group_size < -1: msg = f"Invalid group_size={self.group_size}. Group size must be a positive integer or -1." raise ValidationError(msg) diff --git a/src/nncf/quantization/algorithms/weight_compression/gptq.py b/src/nncf/quantization/algorithms/weight_compression/gptq.py index 668c4646236..bbf49e504cb 100644 --- a/src/nncf/quantization/algorithms/weight_compression/gptq.py +++ b/src/nncf/quantization/algorithms/weight_compression/gptq.py @@ -288,7 +288,9 @@ def _quantize_weights( ) reduction_axes = wc_params.reduction_axes block_compression_config = WeightCompressionConfig( - mode=wc_params.compression_config.mode, codebook_values=wc_params.compression_config.codebook_values + mode=wc_params.compression_config.mode, + group_size=-1, + codebook_values=wc_params.compression_config.codebook_values, ) damp = self._damp_percent * fns.mean(fns.diag(hessian)) diff --git a/src/nncf/quantization/algorithms/weight_compression/weight_lowering.py b/src/nncf/quantization/algorithms/weight_compression/weight_lowering.py index 548c5252fd3..f3639447807 100644 --- a/src/nncf/quantization/algorithms/weight_compression/weight_lowering.py +++ b/src/nncf/quantization/algorithms/weight_compression/weight_lowering.py @@ -180,7 +180,7 @@ def do_float_quantization( weight_config.mode = CompressWeightsMode.FP4 compressed_weight = _do_float_quantization_single_scale(weight, weight_config, reduction_axes) - scale_config = WeightCompressionConfig(mode=CompressWeightsMode.FP8_E4M3) + scale_config = WeightCompressionConfig(mode=CompressWeightsMode.FP8_E4M3, group_size=-1) # Explicit reduction_axes are required for the optimized OV path, # which does not support reduction_axes=None. scale_reduction_axes = tuple(range(compressed_weight.scale.ndim)) diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 10fc20a6b93..125663dfe2f 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -979,7 +979,7 @@ def test_nvfp4_optimized_scale_compression(): The optimized path is forced via patching to avoid needing a large weight tensor. """ weight = Tensor(np.random.randn(2, 16).astype(np.float32)) - config = WeightCompressionConfig(mode=CompressWeightsMode.NVFP4) + config = WeightCompressionConfig(mode=CompressWeightsMode.NVFP4, group_size=-1) reduction_axes = -1 # Force the optimized OV path for all tensors regardless of size @@ -1510,7 +1510,7 @@ def test_int_compressed_weighs_range(mode, data): data = np.array(data).astype(np.float32) w = Tensor(data) - config = WeightCompressionConfig(mode=mode) + config = WeightCompressionConfig(mode=mode, group_size=-1) compressed_weight = do_integer_quantization(w, config, -1) assert np.allclose(np.abs(compressed_weight.tensor.data), np.abs(w.data)) @@ -1614,7 +1614,7 @@ def test_float_compressed_weighs_range(mode, id_, data): data = np.array(data).astype(np.float32) w = Tensor(data) - config = WeightCompressionConfig(mode=mode) + config = WeightCompressionConfig(mode=mode, group_size=-1) cw = do_float_quantization(w, config, -1) decompressed_weight = do_float_dequantization(cw, -1).data @@ -1659,18 +1659,18 @@ def test_codebook_weights_range(data): (WeightCompressionConfig(CompressWeightsMode.INT8_ASYM), True, True, False), (WeightCompressionConfig(CompressWeightsMode.INT8_ASYM), True, False, True), (WeightCompressionConfig(CompressWeightsMode.INT8_ASYM), False, True, True), - (WeightCompressionConfig(CompressWeightsMode.INT4_ASYM), False, False, False), - (WeightCompressionConfig(CompressWeightsMode.INT4_ASYM), True, True, False), - (WeightCompressionConfig(CompressWeightsMode.INT4_ASYM), True, False, True), - (WeightCompressionConfig(CompressWeightsMode.INT4_ASYM), False, True, True), + (WeightCompressionConfig(CompressWeightsMode.INT4_ASYM, group_size=-1), False, False, False), + (WeightCompressionConfig(CompressWeightsMode.INT4_ASYM, group_size=-1), True, True, False), + (WeightCompressionConfig(CompressWeightsMode.INT4_ASYM, group_size=-1), True, False, True), + (WeightCompressionConfig(CompressWeightsMode.INT4_ASYM, group_size=-1), False, True, True), (WeightCompressionConfig(CompressWeightsMode.INT8_SYM), True, False, False), (WeightCompressionConfig(CompressWeightsMode.INT8_SYM), False, False, False), - (WeightCompressionConfig(CompressWeightsMode.INT4_SYM), True, False, False), - (WeightCompressionConfig(CompressWeightsMode.INT4_SYM), False, False, False), - (WeightCompressionConfig(CompressWeightsMode.INT3_SYM), True, False, False), - (WeightCompressionConfig(CompressWeightsMode.INT3_SYM), False, False, False), - (WeightCompressionConfig(CompressWeightsMode.INT2_SYM), True, False, False), - (WeightCompressionConfig(CompressWeightsMode.INT2_SYM), False, False, False), + (WeightCompressionConfig(CompressWeightsMode.INT4_SYM, group_size=-1), True, False, False), + (WeightCompressionConfig(CompressWeightsMode.INT4_SYM, group_size=-1), False, False, False), + (WeightCompressionConfig(CompressWeightsMode.INT3_SYM, group_size=-1), True, False, False), + (WeightCompressionConfig(CompressWeightsMode.INT3_SYM, group_size=-1), False, False, False), + (WeightCompressionConfig(CompressWeightsMode.INT2_SYM, group_size=-1), True, False, False), + (WeightCompressionConfig(CompressWeightsMode.INT2_SYM, group_size=-1), False, False, False), ], ) def test_int_quantization_with_precomputed_parameters(config, precompute_scale, precompute_zero_point, raises): diff --git a/tests/openvino/optimized_functions/test_compression_functions.py b/tests/openvino/optimized_functions/test_compression_functions.py index 74dc2582e65..8497d37e662 100644 --- a/tests/openvino/optimized_functions/test_compression_functions.py +++ b/tests/openvino/optimized_functions/test_compression_functions.py @@ -61,22 +61,22 @@ class QuantizationTask(Enum): ] INT4_COMPRESSION_CONFIGS = [ - WeightCompressionConfig(CompressWeightsMode.INT4_ASYM), - WeightCompressionConfig(CompressWeightsMode.INT4_SYM), + WeightCompressionConfig(CompressWeightsMode.INT4_ASYM, group_size=-1), + WeightCompressionConfig(CompressWeightsMode.INT4_SYM, group_size=-1), WeightCompressionConfig(CompressWeightsMode.INT4_ASYM, group_size=2), WeightCompressionConfig(CompressWeightsMode.INT4_SYM, group_size=2), ] FP4_COMPRESSION_CONFIGS = [ - WeightCompressionConfig(CompressWeightsMode.NF4), - WeightCompressionConfig(CompressWeightsMode.FP4), + WeightCompressionConfig(CompressWeightsMode.NF4, group_size=-1), + WeightCompressionConfig(CompressWeightsMode.FP4, group_size=-1), WeightCompressionConfig(CompressWeightsMode.NF4, group_size=2), WeightCompressionConfig(CompressWeightsMode.FP4, group_size=2), WeightCompressionConfig(CompressWeightsMode.MXFP4, group_size=32), ] FP8_COMPRESSION_CONFIGS = [ - WeightCompressionConfig(CompressWeightsMode.FP8_E4M3), + WeightCompressionConfig(CompressWeightsMode.FP8_E4M3, group_size=-1), WeightCompressionConfig(CompressWeightsMode.FP8_E4M3, group_size=2), WeightCompressionConfig(CompressWeightsMode.MXFP8_E4M3, group_size=32), ] From bc8c923c4cdd65a0b8a94cb9225c219791b22dd6 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 14:50:00 +0400 Subject: [PATCH 36/43] add error for group size fallback with MX, NV types for common function --- .../weight_compression/algorithm.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index c618cc2de6b..1c7fdc984a9 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -130,17 +130,31 @@ def get_weight_compression_configuration( } -def validate_fixed_group_size(mode: CompressWeightsMode, group_size: int | None) -> None: +def validate_fixed_group_size( + mode: CompressWeightsMode, group_size: int | None, advanced_parameters: AdvancedCompressionParameters | None = None +) -> None: """ Validates the group size requested for a mode which accepts only specific group sizes. :param mode: Compression mode to validate. :param group_size: Group size requested for the given mode. None means that the default value is used. + :param advanced_parameters: Advanced compression parameters that may affect the validation. """ fixed_group_size = FIXED_GROUP_SIZE_MODES.get(mode) if fixed_group_size is not None and group_size not in [None, fixed_group_size]: msg = f"{mode.value} type only supports group size of {fixed_group_size}, group size of {group_size} is given" raise nncf.ValidationError(msg) + if ( + mode in FIXED_GROUP_SIZE_MODES + and advanced_parameters + and advanced_parameters.group_size_fallback_mode is GroupSizeFallbackMode.ADJUST + ): + msg = ( + "MXFP4, MXFP8_E4M3 and NVFP4 types do not support the group size" + f" fallback mode {advanced_parameters.group_size_fallback_mode.value}." + " Please use other group size fallback mode." + ) + raise nncf.ValidationError(msg) def validate_custom_annotation(custom_annotation: list[CustomAnnotation] | None) -> None: @@ -347,18 +361,7 @@ def check_user_compression_configuration( ) raise nncf.ValidationError(msg) - validate_fixed_group_size(mode, group_size) - if ( - mode in FIXED_GROUP_SIZE_MODES - and advanced_parameters - and advanced_parameters.group_size_fallback_mode is GroupSizeFallbackMode.ADJUST - ): - msg = ( - "MXFP4, MXFP8_E4M3 and NVFP4 types do not support the group size" - f" fallback mode {advanced_parameters.group_size_fallback_mode.value}." - " Please use other group size fallback mode." - ) - raise nncf.ValidationError(msg) + validate_fixed_group_size(mode, group_size, advanced_parameters) class WeightCompression(Algorithm): From d77e6345aec2a3c6b4644482a103fc766a4c7e5b Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 15:30:39 +0400 Subject: [PATCH 37/43] add validation for the openvino backend case. --- src/nncf/quantization/quantize_model.py | 29 ++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/nncf/quantization/quantize_model.py b/src/nncf/quantization/quantize_model.py index f9ae741dc14..5eecd513ca4 100644 --- a/src/nncf/quantization/quantize_model.py +++ b/src/nncf/quantization/quantize_model.py @@ -400,6 +400,7 @@ def _validate_compression_modes_supported_by_backend( not_supported_modes: list[CompressWeightsMode], mode: CompressWeightsMode, custom_annotation: list[CustomAnnotation] | None, + msg: str | None = None, ) -> None: """ Checks that neither the compression mode nor the mode defined by the custom annotation @@ -409,14 +410,16 @@ def _validate_compression_modes_supported_by_backend( :param not_supported_modes: List of the compression modes that are not supported by the backend. :param mode: Compression mode of the algorithm. :param custom_annotation: List of the user-defined weight compression configurations, if given. + :param msg: Error message to use instead of the default one :raises nncf.ParameterNotSupportedError: If any of the given modes is not supported by the backend. """ given_modes = [mode] + [annotation.config.mode for annotation in custom_annotation or []] if any(given_mode in not_supported_modes for given_mode in given_modes): - msg = ( - f"{backend_name} backend does not support {[m.value for m in not_supported_modes]} modes " - "for weight compression." - ) + if msg is None: + msg = ( + f"{backend_name} backend does not support {[m.value for m in not_supported_modes]} modes " + "for weight compression." + ) raise nncf.ParameterNotSupportedError(msg) @@ -637,12 +640,18 @@ def compress_weights( CompressWeightsMode.FP4, CompressWeightsMode.NVFP4, ] - if mode in not_supported_modes: - msg = ( - "AWQ, Scale estimation, GPTQ or Lora Correction algorithm is defined," - f" but mode in {[m.value for m in not_supported_modes]}." - ) - raise nncf.ParameterNotSupportedError(msg) + msg = ( + "AWQ, Scale estimation, GPTQ or Lora Correction algorithm is defined," + f" but mode in {[m.value for m in not_supported_modes]}." + ) + + _validate_compression_modes_supported_by_backend( + "OpenVINO", + not_supported_modes, + mode, + custom_annotation, + msg=msg, + ) if gptq and lora_correction: msg = "Simultaneous use of Lora correction and GPTQ algorithms is not supported. Select one of them." From 83690daaf6a6f63e816495f1ee0f18788da31274 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 15:41:02 +0400 Subject: [PATCH 38/43] remove warning for nodes without weight. --- .../weight_compression/algorithm.py | 27 +++++++------------ .../template_test_weights_compression.py | 11 -------- 2 files changed, 9 insertions(+), 29 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 1c7fdc984a9..382171cc856 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -1217,29 +1217,25 @@ def _get_weight_name_to_config_mapping( custom_annotation: list[CustomAnnotation], graph: NNCFGraph, node_name_to_weight_names_mapping: dict[str, list[str]], - ) -> tuple[dict[str, WeightCompressionConfig], set[str], set[str]]: + ) -> tuple[dict[str, WeightCompressionConfig], set[str]]: """ - Matches the custom annotation against the weights of the nodes that can be compressed. Keying by a weight - name annotates the nodes sharing a weight as a single one. + Matches the custom annotation against the weights of the nodes that can be compressed. :param custom_annotation: List of user-defined custom annotations. :param graph: NNCFGraph instance. :param node_name_to_weight_names_mapping: A mapping from a node name to the names of the weights of that node. - :return: A tuple of the mapping from a weight name to the annotated configuration, the matched node names - with no weight to compress and the weight names matched by several annotations. + :return: A tuple of the mapping from a weight name to the annotated configuration and the weight names + matched by several annotations. """ weight_name_to_config_mapping: dict[str, WeightCompressionConfig] = {} overlapped_annotation_weight_names = set() - node_names_without_weight = set() for annotation in custom_annotation: annotated_names = get_node_names_from_scope(annotation.scope, graph, strict=annotation.scope.validate) annotated_weight_names = set() for node_name in annotated_names: weight_names = node_name_to_weight_names_mapping.get(node_name) - if not weight_names: - node_names_without_weight.add(node_name) - continue - annotated_weight_names.update(weight_names) + if weight_names is not None: + annotated_weight_names.update(weight_names) # The weights annotated by the current annotation overlap with the ones annotated before it overlapped_annotation_weight_names.update( name for name in annotated_weight_names if name in weight_name_to_config_mapping @@ -1247,7 +1243,7 @@ def _get_weight_name_to_config_mapping( for weight_name in annotated_weight_names: weight_name_to_config_mapping[weight_name] = annotation.config - return weight_name_to_config_mapping, node_names_without_weight, overlapped_annotation_weight_names + return weight_name_to_config_mapping, overlapped_annotation_weight_names def _get_custom_annotation_configs( self, custom_annotation: list[CustomAnnotation], graph: NNCFGraph @@ -1261,15 +1257,10 @@ def _get_custom_annotation_configs( """ node_name_to_weight_names_mapping = self._get_node_name_to_weight_names_mapping(graph) - weight_name_to_config_mapping, node_names_without_weight, overlapped_annotation_weight_names = ( - self._get_weight_name_to_config_mapping(custom_annotation, graph, node_name_to_weight_names_mapping) + weight_name_to_config_mapping, overlapped_annotation_weight_names = self._get_weight_name_to_config_mapping( + custom_annotation, graph, node_name_to_weight_names_mapping ) - if node_names_without_weight: - nncf_logger.warning( - "The following nodes are matched by the custom annotation, but have no weight to compress. " - "The annotation has no effect for them:\n\t" + "\n\t".join(sorted(node_names_without_weight)) - ) if overlapped_annotation_weight_names: overlapped_configs = [] for weight_name in sorted(overlapped_annotation_weight_names): diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index 38815921fe2..8b7f7f81c28 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -1343,17 +1343,6 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, "Several custom annotations match the same nodes", id="overlapping_annotations", ), - pytest.param( - [ - nncf.CustomAnnotation( - scope=nncf.CustomAnnotationScope(patterns=[".*"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), - ) - ], - None, - "have no weight to compress", - id="node_without_weight", - ), pytest.param( [ nncf.CustomAnnotation( From e93e5747c391b52ada26c07dc4e46065ed30951b Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 16:03:52 +0400 Subject: [PATCH 39/43] validate custom annotation with advanced parameters --- .../algorithms/weight_compression/algorithm.py | 12 ++++++++---- src/nncf/quantization/quantize_model.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index 382171cc856..d50d340d660 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -145,23 +145,27 @@ def validate_fixed_group_size( msg = f"{mode.value} type only supports group size of {fixed_group_size}, group size of {group_size} is given" raise nncf.ValidationError(msg) if ( - mode in FIXED_GROUP_SIZE_MODES + fixed_group_size is not None and advanced_parameters and advanced_parameters.group_size_fallback_mode is GroupSizeFallbackMode.ADJUST ): msg = ( - "MXFP4, MXFP8_E4M3 and NVFP4 types do not support the group size" + f"{mode.value} type only supports group size of {fixed_group_size}, so it does not support the group size" f" fallback mode {advanced_parameters.group_size_fallback_mode.value}." " Please use other group size fallback mode." ) raise nncf.ValidationError(msg) -def validate_custom_annotation(custom_annotation: list[CustomAnnotation] | None) -> None: +def validate_custom_annotation( + custom_annotation: list[CustomAnnotation] | None, + advanced_parameters: AdvancedCompressionParameters | None = None, +) -> None: """ Validates the user-defined custom annotation. :param custom_annotation: List of custom annotations to validate. + :param advanced_parameters: Advanced compression parameters that may affect the validation. """ if custom_annotation is None: return @@ -217,7 +221,7 @@ def validate_custom_annotation(custom_annotation: list[CustomAnnotation] | None) ) raise nncf.ParameterNotSupportedError(msg) - validate_fixed_group_size(annotation.config.mode, annotation.config.group_size) + validate_fixed_group_size(annotation.config.mode, annotation.config.group_size, advanced_parameters) def check_user_compression_configuration( diff --git a/src/nncf/quantization/quantize_model.py b/src/nncf/quantization/quantize_model.py index 5eecd513ca4..f7585fb14b3 100644 --- a/src/nncf/quantization/quantize_model.py +++ b/src/nncf/quantization/quantize_model.py @@ -535,7 +535,7 @@ def compress_weights( :type custom_annotation: list[nncf.CustomAnnotation] :return: The non-trainable model with compressed weights. """ - validate_custom_annotation(custom_annotation) + validate_custom_annotation(custom_annotation, advanced_parameters) backend = get_backend(model) compression_weights_impl: Callable[..., Any] | None = None From 48b3f6ca10705f3e828364a7e5cb2870d455522e Mon Sep 17 00:00:00 2001 From: anzr299 Date: Thu, 24 Sep 2026 17:22:46 +0400 Subject: [PATCH 40/43] revert path + [Key] --- src/nncf/openvino/rt_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nncf/openvino/rt_info.py b/src/nncf/openvino/rt_info.py index 1cc755bf37e..28cb6d9e537 100644 --- a/src/nncf/openvino/rt_info.py +++ b/src/nncf/openvino/rt_info.py @@ -63,7 +63,7 @@ def dump_parameters( if isinstance(value, BaseScope): value = exclude_empty_fields(asdict(value)) if bool(value): - dump_parameters(model, value, algo_name, path + [key]) + dump_parameters(model, value, algo_name, [key]) continue # The default value in case empty ignored_scope parameter passed value = [] From 461862213a52dac7d35737d538325461a0909b41 Mon Sep 17 00:00:00 2001 From: Aamir Nazir Date: Mon, 12 Oct 2026 12:52:48 +0400 Subject: [PATCH 41/43] [Torch] Add the GroupedMatMul operation for data-free weight compression (#4241) ### Changes Add the GroupedMatMul operation for data-free weight compression ### Reason for changes Support the GroupedMatMul operation in torch backend ### Related tickets CVS-191922 ### Tests test_get_reduction_axes [Test examples](https://github.com/openvinotoolkit/nncf/actions/runs/38219191486) - success [Weight compression](https://github.com/openvinotoolkit/nncf/actions/runs/38215562790) - success (cherry picked from commit 93180261e9383d1a945abfe276e6386c082a13e6) --- .../weight_compression/algorithm.py | 12 +++++++ .../weight_compression/torch_backend.py | 18 ++++++---- .../weight_compression/torch_fx_backend.py | 4 ++- src/nncf/torch/graph/operator_metatypes.py | 12 ++++++- src/nncf/torch/model_graph_manager.py | 6 ++-- src/nncf/torch/node_utils.py | 2 +- src/nncf/torch/quantization/layers.py | 12 ++++--- .../quantization/test_fq_lora.py | 24 +++++++++++++ .../quantization/test_weights_compression.py | 34 +++++++++++++++++++ tests/torch/test_model_graph_manager.py | 1 + tests/torch/test_models/synthetic.py | 23 +++++++++++++ 11 files changed, 132 insertions(+), 16 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index d50d340d660..90433003517 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -1148,6 +1148,18 @@ def get_weight_compression_parameters( ) model_backend = get_backend(model) + if model_backend == BackendType.TORCH and ( + self._data_aware_compression or self._data_aware_mixed_precision + ): + from nncf.torch.graph.operator_metatypes import PTGroupedMatMulMetatype + + if node.metatype == PTGroupedMatMulMetatype: + msg = ( + "Data-aware weight compression is not supported for GroupedMatMul in the Torch " + f"backend. Node with weight: {node.node_name}." + ) + raise nncf.ParameterNotSupportedError(msg) + ov_version = self._maybe_get_ov_version() if ( model_backend == BackendType.OPENVINO diff --git a/src/nncf/quantization/algorithms/weight_compression/torch_backend.py b/src/nncf/quantization/algorithms/weight_compression/torch_backend.py index 0a53e872709..84532985f26 100644 --- a/src/nncf/quantization/algorithms/weight_compression/torch_backend.py +++ b/src/nncf/quantization/algorithms/weight_compression/torch_backend.py @@ -51,8 +51,10 @@ from nncf.torch.graph.operator_metatypes import EMBEDDING_METATYPES from nncf.torch.graph.operator_metatypes import MATMUL_METATYPES from nncf.torch.graph.operator_metatypes import PTDropoutMetatype +from nncf.torch.graph.operator_metatypes import PTGroupedMatMulMetatype from nncf.torch.graph.operator_metatypes import PTMulMetatype from nncf.torch.graph.operator_metatypes import PTNoopMetatype +from nncf.torch.graph.operator_metatypes import PTTransposeMetatype from nncf.torch.graph.pattern_operations import ATOMIC_ACTIVATIONS_OPERATIONS from nncf.torch.graph.transformations.commands import PTSharedFnInsertionCommand from nncf.torch.graph.transformations.commands import PTTransformationCommand @@ -134,6 +136,10 @@ def get_reduction_axes(node_with_weight: NNCFNode, weight_port_id: int, graph: N ndims = len(weight_node.layer_attributes.shape) reduction_axes = get_weight_compression_reduction_axes(node_with_weight_metatype, weight_port_id, ndims) + weight_input_metatype = graph.get_input_edge_by_port_id(node_with_weight, weight_port_id).from_node.metatype + if node_with_weight_metatype == PTGroupedMatMulMetatype and weight_input_metatype != PTTransposeMetatype: + # A weight that is not transposed is consumed with the [num_groups, C_IN, C_OUT] layout. + reduction_axes = [ndims - 2] return tuple(reduction_axes) @staticmethod @@ -220,14 +226,14 @@ def init_lora_adapters(svd_residual: torch.Tensor, rank: int | None = None) -> t :param rank: The rank for the decomposition. If None, the full rank is used. :return: A tuple containing the U and V matrices from the SVD. """ - # O stands for output dimension, H - input dimension or hidden size, R - rank. + # O stands for output dimension, H - input dimension or hidden size, R - rank, B - batch dimension. U_full, S_full, V_full = torch.linalg.svd(svd_residual, full_matrices=False) - U = U_full[:, :rank] # [H, R] + U = U_full[..., :rank] # [H, R] or [B, H, R] S_sqrt = torch.sqrt(S_full) - S = torch.diag(S_sqrt[:rank]) # [R, R] - V = V_full[:rank, :] # [R, O] - V = S @ V # [R, O] - U = U @ S # [H, R] + S = torch.diag_embed(S_sqrt[..., :rank]) # [R, R] or [B, R, R] + V = V_full[..., :rank, :] # [R, O] or [B, R, O] + V = S @ V # [R, O] or [B, R, O] + U = U @ S # [H, R] or [B, H, R] return U, V @staticmethod diff --git a/src/nncf/quantization/algorithms/weight_compression/torch_fx_backend.py b/src/nncf/quantization/algorithms/weight_compression/torch_fx_backend.py index 01c3281531e..3fb80bca3bd 100644 --- a/src/nncf/quantization/algorithms/weight_compression/torch_fx_backend.py +++ b/src/nncf/quantization/algorithms/weight_compression/torch_fx_backend.py @@ -50,6 +50,7 @@ from nncf.torch.graph.operator_metatypes import CONVOLUTION_METATYPES from nncf.torch.graph.operator_metatypes import EMBEDDING_METATYPES from nncf.torch.graph.operator_metatypes import MATMUL_METATYPES +from nncf.torch.graph.operator_metatypes import PTGroupedMatMulMetatype from nncf.torch.graph.transformations.commands import PTTargetPoint from nncf.torch.model_graph_manager import get_const_node from nncf.torch.model_graph_manager import get_weight_compression_reduction_axes @@ -66,7 +67,8 @@ class FXWeightCompressionAlgoBackend(WeightCompressionAlgoBackend): @property def matmul_metatypes(self) -> list[OperatorMetatype]: - return MATMUL_METATYPES + # TODO(anzr299): add groupedmatmul support in torch fx + return [metatype for metatype in MATMUL_METATYPES if metatype != PTGroupedMatMulMetatype] @property def embedding_metatypes(self) -> list[OperatorMetatype]: diff --git a/src/nncf/torch/graph/operator_metatypes.py b/src/nncf/torch/graph/operator_metatypes.py index ce8807bb378..74c6998e7f6 100644 --- a/src/nncf/torch/graph/operator_metatypes.py +++ b/src/nncf/torch/graph/operator_metatypes.py @@ -522,6 +522,16 @@ class PTMatMulMetatype(PTOperatorMetatype): weight_port_ids = [0, 1] +@PT_OPERATOR_METATYPES.register() +class PTGroupedMatMulMetatype(PTOperatorMetatype): + name = "GroupedMatMulOp" + module_to_function_names = { + NamespaceTarget.TORCH: ["_grouped_mm"], + NamespaceTarget.TORCH_NN_FUNCTIONAL: ["grouped_mm"], + } + weight_port_ids = [1] + + @PT_OPERATOR_METATYPES.register() class PTAddmmMetatype(PTOperatorMetatype): name = "MatMulOp" @@ -1041,7 +1051,7 @@ def get_operator_metatypes() -> list[type[OperatorMetatype]]: PTAtenEmbeddingBagMetatype, ] -MATMUL_METATYPES = [PTLinearMetatype, PTMatMulMetatype, PTAddmmMetatype] +MATMUL_METATYPES = [PTLinearMetatype, PTMatMulMetatype, PTAddmmMetatype, PTGroupedMatMulMetatype] EMBEDDING_METATYPES = [PTEmbeddingMetatype, PTAtenEmbeddingMetatype] diff --git a/src/nncf/torch/model_graph_manager.py b/src/nncf/torch/model_graph_manager.py index b471b16c7b3..735587db49c 100644 --- a/src/nncf/torch/model_graph_manager.py +++ b/src/nncf/torch/model_graph_manager.py @@ -36,7 +36,9 @@ OPERATORS_WITH_BIAS_METATYPES = CONV_META_TYPES + [om.PTLinearMetatype] CONV_FUSED_META_TYPES = [om.PTBatchNormMetatype] -AVAILABLE_NODE_TYPES_FOR_WEIGHT_SUBGRAPH = om.QUANTIZE_NODE_TYPES + om.PRUNING_NODE_TYPES +AVAILABLE_NODE_TYPES_FOR_WEIGHT_SUBGRAPH = ( + om.QUANTIZE_NODE_TYPES + om.PRUNING_NODE_TYPES + ["transpose", "permute", "transpose_"] +) def find_const_node_in_constant_subgraph(node: NNCFNode, graph: NNCFGraph) -> NNCFNode | None: @@ -359,7 +361,7 @@ def get_weight_compression_reduction_axes(metatype: OperatorMetatype, weight_por """ if metatype in [om.PTAtenEmbeddingMetatype, om.PTEmbeddingMetatype]: return [1] - if metatype == om.PTLinearMetatype: + if metatype in [om.PTLinearMetatype, om.PTGroupedMatMulMetatype]: return [ndims - 1] if metatype == om.PTMatMulMetatype: if weight_port_id == 0: diff --git a/src/nncf/torch/node_utils.py b/src/nncf/torch/node_utils.py index bb46f5f6c82..c8c49b0c578 100644 --- a/src/nncf/torch/node_utils.py +++ b/src/nncf/torch/node_utils.py @@ -28,7 +28,7 @@ def get_activation_channel_axis(node: NNCFNode, port_id: int) -> int: msg = f"Activation channel axis retrieval from node with metatype {node.metatype} is not supported" raise nncf.InternalError(msg) - if node.metatype not in [PTMatMulMetatype, PTAddmmMetatype]: + if node.metatype not in [PTMatMulMetatype, PTAddmmMetatype, op.PTGroupedMatMulMetatype]: if not isinstance(node.metatype.output_channel_axis, int): msg = f"Node metatype {node.metatype} does not have defined output channel axis" raise nncf.InternalError(msg) diff --git a/src/nncf/torch/quantization/layers.py b/src/nncf/torch/quantization/layers.py index b21f79d4d62..a090a9db9ac 100644 --- a/src/nncf/torch/quantization/layers.py +++ b/src/nncf/torch/quantization/layers.py @@ -987,7 +987,9 @@ class LoraMixin: def init_lora(self, lspec: PTLoraSpec): self._lspec = lspec default_lora_dtype = torch.bfloat16 - out_features, in_features = lspec.orig_weight_shape + # 3D weights have leading batch dimension. A separate pair of adapters + # is created for each group. + *group_dims, out_features, in_features = lspec.orig_weight_shape rank = lspec.lora_rank if rank > out_features or rank > in_features: msg = ( @@ -995,8 +997,8 @@ def init_lora(self, lspec: PTLoraSpec): f"[{out_features}, {in_features}]" ) raise nncf.ValidationError(msg) - self.lora_A = torch.nn.Parameter(torch.ones((rank, in_features), dtype=default_lora_dtype)) - self.lora_B = torch.nn.Parameter(torch.zeros((out_features, rank), dtype=default_lora_dtype)) + self.lora_A = torch.nn.Parameter(torch.ones((*group_dims, rank, in_features), dtype=default_lora_dtype)) + self.lora_B = torch.nn.Parameter(torch.zeros((*group_dims, out_features, rank), dtype=default_lora_dtype)) def enable_gradients(self): self.lora_A.requires_grad = True @@ -1042,8 +1044,8 @@ def get_active_adapters(self) -> tuple[torch.Tensor, torch.Tensor]: :return: A dictionary containing the active LoRA adapters. """ - lora_A = self.lora_A[: self.active_lora_rank, :] - lora_B = self.lora_B[:, : self.active_lora_rank] + lora_A = self.lora_A[..., : self.active_lora_rank, :] + lora_B = self.lora_B[..., : self.active_lora_rank] return lora_A, lora_B diff --git a/tests/torch/function_hook/quantization/test_fq_lora.py b/tests/torch/function_hook/quantization/test_fq_lora.py index f717ad15149..874a313f4b7 100644 --- a/tests/torch/function_hook/quantization/test_fq_lora.py +++ b/tests/torch/function_hook/quantization/test_fq_lora.py @@ -33,6 +33,7 @@ from nncf.torch.quantization.layers import SymmetricQuantizer as SQ from tests.cross_fw.shared.paths import TEST_ROOT from tests.torch.function_hook.quantization.test_weights_compression import AWQLinearModel +from tests.torch.test_models.synthetic import GroupedMatMulModel from tests.torch.test_models.synthetic import LinearModel from tests.torch.test_models.synthetic import ShortTransformer from tests.torch.utils import compare_with_reference_file @@ -199,6 +200,29 @@ def test_invalid_lora_rank(): ) +@pytest.mark.parametrize("transposed_weight", (False, True), ids=("native_weight", "transposed_weight")) +def test_fq_lora_grouped_matmul(transposed_weight): + rank = 2 + model = GroupedMatMulModel(transposed_weight=transposed_weight) + example_input = torch.ones(model.INPUT_SHAPE, dtype=model.INPUT_DTYPE) + wrapped_model = wrap_model(model, example_input=example_input, trace_parameters=True) + + compressed_model = compress_weights( + wrapped_model, + mode=CompressWeightsMode.INT4_SYM, + group_size=4, + all_layers=True, + compression_format=CompressionFormat.FQ_LORA, + advanced_parameters=AdvancedCompressionParameters(lora_adapter_rank=rank), + ) + + num_groups, c_in, c_out = GroupedMatMulModel.NUM_GROUPS, GroupedMatMulModel.C_IN, GroupedMatMulModel.C_OUT + out_features, in_features = (c_out, c_in) if transposed_weight else (c_in, c_out) + quantizer = next(module for module in compressed_model.modules() if isinstance(module, LoraMixin)) + assert quantizer.lora_A.shape == (num_groups, rank, in_features) + assert quantizer.lora_B.shape == (num_groups, out_features, rank) + + @pytest.mark.parametrize("all_layers", [True, False]) def test_compress_shared_weights(all_layers, regen_ref_data): model = ShortTransformer(8, 16, share_weights=True) diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index 45f5af7b268..b8d5f3f681a 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -48,6 +48,7 @@ from tests.cross_fw.test_templates.helpers import RoPEModel from tests.cross_fw.test_templates.helpers import SAMPEModel from tests.cross_fw.test_templates.template_test_weights_compression import TemplateWeightCompression +from tests.torch.test_models.synthetic import GroupedMatMulModel from tests.torch.test_models.synthetic import ShortTransformer from tests.torch.test_tensor import cast_to @@ -415,6 +416,39 @@ def test_compress_weights_conv(): assert n_compressed_weights == n_target_modules +@pytest.mark.parametrize("transposed_weight", (False, True), ids=("native_weight", "transposed_weight")) +@pytest.mark.parametrize("mode", SUPPORTED_MODES) +def test_compress_weights_grouped_matmul(mode, transposed_weight): + model = GroupedMatMulModel(transposed_weight=transposed_weight) + example_input = torch.ones(model.INPUT_SHAPE, dtype=model.INPUT_DTYPE) + wrapped_model = GraphModelWrapper(wrap_model(model), example_input=example_input) + + kwargs = {"group_size": 4, "all_layers": True} if mode in INT4_MODES else {} + compressed_model = compress_weights(wrapped_model, mode=mode, **kwargs) + + n_compressed_weights = 0 + for module in compressed_model.modules(): + if isinstance(module, BaseWeightsDecompressor): + n_compressed_weights += 1 + assert n_compressed_weights == 1 + + +@pytest.mark.parametrize("algorithm", ("awq", "scale_estimation")) +def test_raise_error_for_data_aware_grouped_matmul(algorithm): + model = GroupedMatMulModel() + example_input = torch.ones(model.INPUT_SHAPE, dtype=model.INPUT_DTYPE) + wrapped_model = GraphModelWrapper(wrap_model(model), example_input=example_input) + with pytest.raises(nncf.ParameterNotSupportedError, match="GroupedMatMul"): + compress_weights( + wrapped_model, + mode=CompressWeightsMode.INT4_SYM, + group_size=4, + all_layers=True, + dataset=nncf.Dataset([example_input]), + **{algorithm: True}, + ) + + @pytest.mark.parametrize("mode", SUPPORTED_MODES) def test_compress_shared_weights(mocker, mode): model = wrap_model(ShortTransformer(8, 16, share_weights=True)) diff --git a/tests/torch/test_model_graph_manager.py b/tests/torch/test_model_graph_manager.py index d791439d405..d2203957faa 100644 --- a/tests/torch/test_model_graph_manager.py +++ b/tests/torch/test_model_graph_manager.py @@ -350,6 +350,7 @@ def test_update_fused_bias(model_cls): (om.PTAddmmMetatype, 1, 3, [2]), (om.PTAddmmMetatype, 2, 3, [1]), (om.PTAddmmMetatype, 2, 1, [0]), + (om.PTGroupedMatMulMetatype, 1, 3, [2]), (om.PTConv1dMetatype, 0, 3, [1, 2]), (om.PTConv2dMetatype, 0, 4, [1, 2, 3]), (om.PTConv3dMetatype, 0, 5, [1, 2, 3, 4]), diff --git a/tests/torch/test_models/synthetic.py b/tests/torch/test_models/synthetic.py index 2379efd9d29..6a8e4ba0317 100644 --- a/tests/torch/test_models/synthetic.py +++ b/tests/torch/test_models/synthetic.py @@ -591,6 +591,29 @@ def forward(self, input): return self.linear(input) +class GroupedMatMulModel(torch.nn.Module): + NUM_GROUPS = 2 + C_IN = 8 + C_OUT = 16 + NUM_TOKENS = 6 + INPUT_SHAPE = (NUM_TOKENS, C_IN) + INPUT_DTYPE = torch.bfloat16 + + def __init__(self, transposed_weight: bool = False): + super().__init__() + self.transposed_weight = transposed_weight + shape = ( + (self.NUM_GROUPS, self.C_OUT, self.C_IN) if transposed_weight else (self.NUM_GROUPS, self.C_IN, self.C_OUT) + ) + weight = torch.arange(1, self.NUM_GROUPS * self.C_IN * self.C_OUT + 1, dtype=self.INPUT_DTYPE) + self.weight = torch.nn.Parameter(weight.reshape(shape)) + self.register_buffer("offs", torch.tensor([self.NUM_TOKENS // 2, self.NUM_TOKENS], dtype=torch.int32)) + + def forward(self, x): + weight = self.weight.transpose(-2, -1) if self.transposed_weight else self.weight + return torch._grouped_mm(x, weight, offs=self.offs) + + class YOLO11N_SDPABlock(torch.nn.Module): INPUT_SIZE = (1, 2, 4) From 25d09cbd9aad125dd8c0b51d420aa197d7d62076 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Mon, 12 Oct 2026 18:49:36 +0400 Subject: [PATCH 42/43] custom_annotation -> scoped_configs --- .../weights_compression/Usage.md | 35 ++- src/nncf/__init__.py | 3 +- .../torch/fx/quantization/quantize_model.py | 6 +- src/nncf/onnx/quantization/quantize_model.py | 6 +- .../openvino/quantization/quantize_model.py | 6 +- src/nncf/openvino/rt_info.py | 31 +- src/nncf/quantization/advanced_parameters.py | 30 +- .../weight_compression/algorithm.py | 233 +++++++------- .../algorithms/weight_compression/config.py | 2 - src/nncf/quantization/quantize_model.py | 36 +-- .../quantization/quantize_model.py | 6 +- .../template_test_weights_compression.py | 284 ++++++++---------- .../group_size_fallback_adjust.json | 0 .../group_size_fallback_ignore.json | 0 .../ignored_scope.json | 0 .../mixed_precision.json | 0 .../overlapping_scoped_configs.json} | 0 .../primary_and_backup_precision.json | 0 .../shared_weight.json | 0 ...ght_ignored_other_node_matched_first.json} | 0 ...ight_ignored_other_node_matched_last.json} | 0 .../shared_weight_matched_by_other_node.json} | 0 ...ared_weight_other_node_matched_first.json} | 0 ...hared_weight_other_node_matched_last.json} | 0 ...d_weight_with_ignored_compressed_node.json | 0 .../shared_weight_with_ignored_scope.json | 0 .../quantization/test_weights_compression.py | 6 +- .../group_size_fallback_adjust.json | 0 .../group_size_fallback_ignore.json | 0 .../ignored_scope.json | 0 .../mixed_precision.json | 0 .../overlapping_scoped_configs.json} | 0 .../primary_and_backup_precision.json | 0 .../shared_weight.json | 0 ...ght_ignored_other_node_matched_first.json} | 0 ...ight_ignored_other_node_matched_last.json} | 0 .../shared_weight_matched_by_other_node.json} | 0 ...ared_weight_other_node_matched_first.json} | 0 ...hared_weight_other_node_matched_last.json} | 0 ...d_weight_with_ignored_compressed_node.json | 0 .../shared_weight_with_ignored_scope.json | 0 .../quantization/test_weights_compression.py | 29 +- .../group_size_fallback_adjust.json | 0 .../group_size_fallback_ignore.json | 0 .../ignored_scope.json | 0 .../mixed_precision.json | 0 .../overlapping_scoped_configs.json} | 0 .../primary_and_backup_precision.json | 0 .../shared_weight.json | 0 ...ght_ignored_other_node_matched_first.json} | 0 ...ight_ignored_other_node_matched_last.json} | 0 .../shared_weight_matched_by_other_node.json} | 0 ...ared_weight_other_node_matched_first.json} | 0 ...hared_weight_other_node_matched_last.json} | 0 ...d_weight_with_ignored_compressed_node.json | 0 .../shared_weight_with_ignored_scope.json | 0 .../group_size_fallback_adjust.json | 0 .../group_size_fallback_ignore.json | 0 .../ignored_scope.json | 0 .../mixed_precision.json | 0 .../overlapping_scoped_configs.json} | 0 .../primary_and_backup_precision.json | 0 .../shared_weight.json | 0 ...ght_ignored_other_node_matched_first.json} | 0 ...ight_ignored_other_node_matched_last.json} | 0 .../shared_weight_matched_by_other_node.json} | 0 ...ared_weight_other_node_matched_first.json} | 0 ...hared_weight_other_node_matched_last.json} | 0 ...d_weight_with_ignored_compressed_node.json | 0 .../shared_weight_with_ignored_scope.json | 0 .../quantization/test_weights_compression.py | 8 +- tests/torch/fx/test_weights_compression.py | 6 +- 72 files changed, 340 insertions(+), 387 deletions(-) rename tests/onnx/data/{references_custom_annotation => references_scoped_configs}/group_size_fallback_adjust.json (100%) rename tests/onnx/data/{references_custom_annotation => references_scoped_configs}/group_size_fallback_ignore.json (100%) rename tests/onnx/data/{references_custom_annotation => references_scoped_configs}/ignored_scope.json (100%) rename tests/onnx/data/{references_custom_annotation => references_scoped_configs}/mixed_precision.json (100%) rename tests/onnx/data/{references_custom_annotation/overlapping_annotations.json => references_scoped_configs/overlapping_scoped_configs.json} (100%) rename tests/onnx/data/{references_custom_annotation => references_scoped_configs}/primary_and_backup_precision.json (100%) rename tests/onnx/data/{references_custom_annotation => references_scoped_configs}/shared_weight.json (100%) rename tests/onnx/data/{references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json => references_scoped_configs/shared_weight_ignored_other_node_matched_first.json} (100%) rename tests/onnx/data/{references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json => references_scoped_configs/shared_weight_ignored_other_node_matched_last.json} (100%) rename tests/onnx/data/{references_custom_annotation/shared_weight_annotated_by_other_node.json => references_scoped_configs/shared_weight_matched_by_other_node.json} (100%) rename tests/onnx/data/{references_custom_annotation/shared_weight_other_node_annotated_first.json => references_scoped_configs/shared_weight_other_node_matched_first.json} (100%) rename tests/onnx/data/{references_custom_annotation/shared_weight_other_node_annotated_last.json => references_scoped_configs/shared_weight_other_node_matched_last.json} (100%) rename tests/onnx/data/{references_custom_annotation => references_scoped_configs}/shared_weight_with_ignored_compressed_node.json (100%) rename tests/onnx/data/{references_custom_annotation => references_scoped_configs}/shared_weight_with_ignored_scope.json (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation => references_scoped_configs}/group_size_fallback_adjust.json (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation => references_scoped_configs}/group_size_fallback_ignore.json (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation => references_scoped_configs}/ignored_scope.json (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation => references_scoped_configs}/mixed_precision.json (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation/overlapping_annotations.json => references_scoped_configs/overlapping_scoped_configs.json} (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation => references_scoped_configs}/primary_and_backup_precision.json (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation => references_scoped_configs}/shared_weight.json (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json => references_scoped_configs/shared_weight_ignored_other_node_matched_first.json} (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json => references_scoped_configs/shared_weight_ignored_other_node_matched_last.json} (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation/shared_weight_annotated_by_other_node.json => references_scoped_configs/shared_weight_matched_by_other_node.json} (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation/shared_weight_other_node_annotated_first.json => references_scoped_configs/shared_weight_other_node_matched_first.json} (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation/shared_weight_other_node_annotated_last.json => references_scoped_configs/shared_weight_other_node_matched_last.json} (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation => references_scoped_configs}/shared_weight_with_ignored_compressed_node.json (100%) rename tests/openvino/native/data/2026.1/{references_custom_annotation => references_scoped_configs}/shared_weight_with_ignored_scope.json (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation => references_scoped_configs}/group_size_fallback_adjust.json (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation => references_scoped_configs}/group_size_fallback_ignore.json (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation => references_scoped_configs}/ignored_scope.json (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation => references_scoped_configs}/mixed_precision.json (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation/overlapping_annotations.json => references_scoped_configs/overlapping_scoped_configs.json} (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation => references_scoped_configs}/primary_and_backup_precision.json (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation => references_scoped_configs}/shared_weight.json (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json => references_scoped_configs/shared_weight_ignored_other_node_matched_first.json} (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json => references_scoped_configs/shared_weight_ignored_other_node_matched_last.json} (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation/shared_weight_annotated_by_other_node.json => references_scoped_configs/shared_weight_matched_by_other_node.json} (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation/shared_weight_other_node_annotated_first.json => references_scoped_configs/shared_weight_other_node_matched_first.json} (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation/shared_weight_other_node_annotated_last.json => references_scoped_configs/shared_weight_other_node_matched_last.json} (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation => references_scoped_configs}/shared_weight_with_ignored_compressed_node.json (100%) rename tests/torch/data/function_hook/quantization/{references_custom_annotation => references_scoped_configs}/shared_weight_with_ignored_scope.json (100%) rename tests/torch/data/fx/{references_custom_annotation => references_scoped_configs}/group_size_fallback_adjust.json (100%) rename tests/torch/data/fx/{references_custom_annotation => references_scoped_configs}/group_size_fallback_ignore.json (100%) rename tests/torch/data/fx/{references_custom_annotation => references_scoped_configs}/ignored_scope.json (100%) rename tests/torch/data/fx/{references_custom_annotation => references_scoped_configs}/mixed_precision.json (100%) rename tests/torch/data/fx/{references_custom_annotation/overlapping_annotations.json => references_scoped_configs/overlapping_scoped_configs.json} (100%) rename tests/torch/data/fx/{references_custom_annotation => references_scoped_configs}/primary_and_backup_precision.json (100%) rename tests/torch/data/fx/{references_custom_annotation => references_scoped_configs}/shared_weight.json (100%) rename tests/torch/data/fx/{references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json => references_scoped_configs/shared_weight_ignored_other_node_matched_first.json} (100%) rename tests/torch/data/fx/{references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json => references_scoped_configs/shared_weight_ignored_other_node_matched_last.json} (100%) rename tests/torch/data/fx/{references_custom_annotation/shared_weight_annotated_by_other_node.json => references_scoped_configs/shared_weight_matched_by_other_node.json} (100%) rename tests/torch/data/fx/{references_custom_annotation/shared_weight_other_node_annotated_first.json => references_scoped_configs/shared_weight_other_node_matched_first.json} (100%) rename tests/torch/data/fx/{references_custom_annotation/shared_weight_other_node_annotated_last.json => references_scoped_configs/shared_weight_other_node_matched_last.json} (100%) rename tests/torch/data/fx/{references_custom_annotation => references_scoped_configs}/shared_weight_with_ignored_compressed_node.json (100%) rename tests/torch/data/fx/{references_custom_annotation => references_scoped_configs}/shared_weight_with_ignored_scope.json (100%) diff --git a/docs/usage/post_training_compression/weights_compression/Usage.md b/docs/usage/post_training_compression/weights_compression/Usage.md index 166158bc181..ab5f04cf0b0 100644 --- a/docs/usage/post_training_compression/weights_compression/Usage.md +++ b/docs/usage/post_training_compression/weights_compression/Usage.md @@ -121,38 +121,45 @@ compressed_model = compress_weights( - The `mode`, `ratio` and `backup_mode` parameters define the precision of a layer indirectly: the mixed-precision algorithm decides which layers are compressed to the primary precision, and the rest is compressed to the backup one. - The `custom_annotation` parameter makes it possible to assign a compression configuration to certain layers + The `scoped_configs` parameter makes it possible to assign a compression configuration to certain layers explicitly. A typical use case is a Mixture-of-Experts model, where the experts are compressed to 4 bits, while the attention and the router layers are kept in 8 bits to preserve accuracy. ```python import nncf -from nncf import compress_weights, CompressWeightsMode +from nncf import compress_weights, CompressWeightsMode, ScopedConfig compressed_model = compress_weights( model, # model is openvino.Model object mode=CompressWeightsMode.INT4_SYM, group_size=64, ratio=1.0, - custom_annotation=[ - nncf.CustomAnnotation( + scoped_configs=[ + ScopedConfig( scope=nncf.Scope(patterns=[".*self_attn.*", ".*router.*"]), - config=nncf.WeightCompressionConfig(mode=CompressWeightsMode.INT8_ASYM, group_size=-1), + mode=CompressWeightsMode.INT8_ASYM, + group_size=-1, + ), + ScopedConfig( + scope=nncf.Scope(patterns=[".*mul.*"]), + mode=CompressWeightsMode.INT4_ASYM, + group_size=128, ), ], ) ``` -- The scope of an annotation is defined by the same rules as [the ignored scope](/docs/usage/IgnoredScope.md): node names, - regular expressions, operation types and subgraphs. - The configuration given by an annotation takes precedence over the decision made by the algorithm, namely over the +- `nncf.ScopedConfig` binds a weight compression configuration (`mode`, `group_size`) to a `scope`. The scope is an + `nncf.Scope` defined by the same rules as [the ignored scope](/docs/usage/IgnoredScope.md): node names, regular + expressions, operation types and subgraphs. + The configuration given by a scoped config takes precedence over the decision made by the algorithm, namely over the mixed-precision assignment, the `all_layers` option and the backup precision of the embeddings and the last linear - layer. A layer that is matched by both the `ignored_scope` and a custom annotation is compressed with the - user-defined configuration and a warning is logged. If several annotations match the same layer, the last one takes - precedence. A weight shared by several layers is compressed once, so annotating any of them applies the - configuration to this weight. The codebook compression modes are not supported by an annotation. The mode of an - annotation must be supported by the backend, the same as the `mode` option, e.g. NF4 can not be annotated for a - Torch, TorchFX or ONNX model. + layer. A layer that is matched by both the `ignored_scope` and a scoped config is compressed with the + user-defined configuration and a warning is logged. If several scoped configs match the same layer, the last one + takes precedence. A weight shared by several layers is compressed once, so matching any of them applies the + configuration to this weight. The codebook compression modes are not supported by a scoped config. The mode of a + scoped config must be supported by the backend, the same as the `mode` option, e.g. NF4 can not be used in a scoped + config for a Torch, TorchFX or ONNX model. #### Data-aware methods diff --git a/src/nncf/__init__.py b/src/nncf/__init__.py index 7e1322a54a0..215ff5b44ac 100644 --- a/src/nncf/__init__.py +++ b/src/nncf/__init__.py @@ -65,10 +65,9 @@ from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters as AdvancedQuantizationParameters from nncf.quantization.advanced_parameters import AdvancedScaleEstimationParameters as AdvancedScaleEstimationParameters from nncf.quantization.advanced_parameters import AdvancedSmoothQuantParameters as AdvancedSmoothQuantParameters -from nncf.quantization.advanced_parameters import CustomAnnotation as CustomAnnotation from nncf.quantization.advanced_parameters import GroupSizeFallbackMode as GroupSizeFallbackMode from nncf.quantization.advanced_parameters import OverflowFix as OverflowFix -from nncf.quantization.algorithms.weight_compression.config import WeightCompressionConfig as WeightCompressionConfig +from nncf.quantization.advanced_parameters import ScopedConfig as ScopedConfig from nncf.scopes import IgnoredScope as IgnoredScope # Deprecated alias for nncf.Scope from nncf.scopes import Scope as Scope from nncf.scopes import Subgraph as Subgraph diff --git a/src/nncf/experimental/torch/fx/quantization/quantize_model.py b/src/nncf/experimental/torch/fx/quantization/quantize_model.py index fe859a61958..bffefe834c0 100644 --- a/src/nncf/experimental/torch/fx/quantization/quantize_model.py +++ b/src/nncf/experimental/torch/fx/quantization/quantize_model.py @@ -37,7 +37,7 @@ from nncf.parameters import TargetDevice from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters -from nncf.quantization.advanced_parameters import CustomAnnotation +from nncf.quantization.advanced_parameters import ScopedConfig from nncf.quantization.algorithms.post_training.algorithm import PostTrainingQuantization from nncf.quantization.algorithms.weight_compression.algorithm import WeightCompression from nncf.scopes import IgnoredScope @@ -133,7 +133,7 @@ def compress_weights_impl( backup_mode: BackupMode, compression_format: CompressionFormat, advanced_parameters: AdvancedCompressionParameters | None = None, - custom_annotation: list[CustomAnnotation] | None = None, + scoped_configs: list[ScopedConfig] | None = None, ) -> torch.fx.GraphModule: """ Implementation of the `compress_weights()` method for the Torch Fx backend. @@ -153,7 +153,7 @@ def compress_weights_impl( backup_mode, compression_format, advanced_parameters, - custom_annotation, + scoped_configs, ) graph = build_graph(model) compressed_model = compression_algorithm.apply(model, graph, dataset=dataset) diff --git a/src/nncf/onnx/quantization/quantize_model.py b/src/nncf/onnx/quantization/quantize_model.py index e8634f8ce64..36a2999cc71 100644 --- a/src/nncf/onnx/quantization/quantize_model.py +++ b/src/nncf/onnx/quantization/quantize_model.py @@ -46,8 +46,8 @@ from nncf.quantization.advanced_parameters import AdvancedAccuracyRestorerParameters from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters -from nncf.quantization.advanced_parameters import CustomAnnotation from nncf.quantization.advanced_parameters import QuantizationParameters +from nncf.quantization.advanced_parameters import ScopedConfig from nncf.quantization.algorithms.accuracy_control.algorithm import QuantizationAccuracyRestorer from nncf.quantization.algorithms.accuracy_control.algorithm import calculate_accuracy_drop from nncf.quantization.algorithms.accuracy_control.evaluator import Evaluator @@ -330,7 +330,7 @@ def compress_weights_impl( backup_mode: BackupMode, compression_format: CompressionFormat, advanced_parameters: AdvancedCompressionParameters | None = None, - custom_annotation: list[CustomAnnotation] | None = None, + scoped_configs: list[ScopedConfig] | None = None, ) -> onnx.ModelProto: if model.opset_import[0].version < 13: msg = "ONNX models with opset version < 13 do not support per-channel quantization." @@ -363,7 +363,7 @@ def compress_weights_impl( backup_mode, compression_format, advanced_parameters, - custom_annotation, + scoped_configs, ) graph = build_graph(model) diff --git a/src/nncf/openvino/quantization/quantize_model.py b/src/nncf/openvino/quantization/quantize_model.py index b68d48d2ca7..4dcb26e34ae 100644 --- a/src/nncf/openvino/quantization/quantize_model.py +++ b/src/nncf/openvino/quantization/quantize_model.py @@ -42,7 +42,7 @@ from nncf.quantization.advanced_parameters import AdvancedAccuracyRestorerParameters from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters -from nncf.quantization.advanced_parameters import CustomAnnotation +from nncf.quantization.advanced_parameters import ScopedConfig from nncf.quantization.advanced_parameters import convert_to_dict_recursively from nncf.quantization.algorithms.accuracy_control.algorithm import QuantizationAccuracyRestorer from nncf.quantization.algorithms.accuracy_control.algorithm import calculate_accuracy_drop @@ -381,7 +381,7 @@ def compress_weights_impl( backup_mode: BackupMode, compression_format: CompressionFormat, advanced_parameters: AdvancedCompressionParameters | None = None, - custom_annotation: list[CustomAnnotation] | None = None, + scoped_configs: list[ScopedConfig] | None = None, ) -> ov.Model: """ Implementation of the `compress_weights()` method for the OpenVINO backend. @@ -403,7 +403,7 @@ def compress_weights_impl( backup_mode, compression_format, advanced_parameters, - custom_annotation, + scoped_configs, ) statistics_points = None diff --git a/src/nncf/openvino/rt_info.py b/src/nncf/openvino/rt_info.py index f70674e1f0e..1bd8c3173ee 100644 --- a/src/nncf/openvino/rt_info.py +++ b/src/nncf/openvino/rt_info.py @@ -16,8 +16,7 @@ import nncf from nncf.common.logging import nncf_logger -from nncf.quantization.advanced_parameters import CustomAnnotation -from nncf.quantization.algorithms.weight_compression.config import WeightCompressionConfig +from nncf.quantization.advanced_parameters import ScopedConfig from nncf.scopes import Scope @@ -35,16 +34,6 @@ def exclude_empty_fields(value: dict[str, Any]) -> dict[str, Any]: return value -def compression_config_to_dict(config: WeightCompressionConfig) -> dict[str, Any]: - """ - Converts a weight compression config into a dictionary suitable for dumping into Model's meta section. - - :param config: Weight compression config. - :return: Dictionary with the compression config parameters. - """ - return {"mode": config.mode.value, "group_size": config.group_size} - - def dump_parameters( model: ov.Model, parameters: dict[str, Any], algo_name: str | None = "quantization", path: list[str] | None = None ) -> None: @@ -63,18 +52,22 @@ def dump_parameters( if isinstance(value, Scope): value = exclude_empty_fields(asdict(value)) if bool(value): - dump_parameters(model, value, algo_name, [key]) + dump_parameters(model, value, algo_name, path + [key]) continue # The default value in case empty ignored_scope parameter passed value = [] - # Special condition for the list of custom annotations - elif isinstance(value, (list, tuple)) and any(isinstance(item, CustomAnnotation) for item in value): - for i, annotation in enumerate(value): - # Index each annotation so its scope and config stay paired and annotations - # do not overwrite each other, e.g. custom_annotation/0/{scope/patterns,config}. + # Special condition for the list of scoped configs + elif isinstance(value, (list, tuple)) and any(isinstance(item, ScopedConfig) for item in value): + for i, scoped_config in enumerate(value): + # Index each scoped config so scoped configs do not overwrite each other, + # e.g. scoped_configs/0/{scope/patterns,mode,group_size}. dump_parameters( model, - {"scope": annotation.scope, "config": compression_config_to_dict(annotation.config)}, + { + "scope": scoped_config.scope, + "mode": scoped_config.mode, + "group_size": scoped_config.group_size, + }, algo_name, path + [key, str(i)], ) diff --git a/src/nncf/quantization/advanced_parameters.py b/src/nncf/quantization/advanced_parameters.py index 351a1db4324..d2ff85fde90 100644 --- a/src/nncf/quantization/advanced_parameters.py +++ b/src/nncf/quantization/advanced_parameters.py @@ -24,6 +24,7 @@ from nncf.common.quantization.quantizer_propagation.structs import QuantizerPropagationRule from nncf.common.quantization.structs import QuantizationScheme as QuantizationMode from nncf.common.utils.api_marker import api +from nncf.parameters import CompressWeightsMode from nncf.quantization.algorithms.weight_compression.config import WeightCompressionConfig from nncf.quantization.range_estimator import RangeEstimatorParameters from nncf.scopes import Scope @@ -453,9 +454,9 @@ class AdvancedCompressionParameters: ) -@api(canonical_alias="nncf.CustomAnnotation") +@api(canonical_alias="nncf.ScopedConfig") @dataclass -class CustomAnnotation: +class ScopedConfig: """ Applies a user-defined weight compression configuration to a portion of a model. It overrides the configuration assigned by the weight compression algorithm, including the mixed precision @@ -467,19 +468,32 @@ class CustomAnnotation: import nncf - annotation = nncf.CustomAnnotation( + scoped_config = nncf.ScopedConfig( scope=nncf.Scope(patterns=['.*self_attn.*', '.*router.*']), - config=nncf.WeightCompressionConfig(mode=nncf.CompressWeightsMode.INT8_ASYM, group_size=-1), + mode=nncf.CompressWeightsMode.INT8_ASYM, + group_size=-1, ) - :param scope: Defines the portion of a model to annotate. + :param scope: Defines the portion of a model to apply the configuration to. :type scope: nncf.Scope - :param config: Weight compression configuration to apply to the matched nodes. - :type config: nncf.WeightCompressionConfig + :param mode: Weight compression mode to apply to the matched nodes. Defaults to INT8_ASYM mode. + :type mode: nncf.CompressWeightsMode + :param group_size: Number of weights in the channel dimension that share quantization parameters (scale). + The value -1 means no grouping. Defaults to None, which means that the default group size of the given + mode is used. + :type group_size: int """ scope: Scope = field(default_factory=Scope) - config: WeightCompressionConfig = field(default_factory=WeightCompressionConfig) + mode: CompressWeightsMode = CompressWeightsMode.INT8_ASYM + group_size: int | None = None + + @property + def compression_config(self) -> WeightCompressionConfig: + """ + Weight compression configuration to apply to the nodes matched by the scope. + """ + return WeightCompressionConfig(mode=self.mode, group_size=self.group_size) @api() diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index a0f1d6d36ae..dc0fd942c9b 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -38,8 +38,8 @@ from nncf.parameters import CompressWeightsMode from nncf.parameters import SensitivityMetric from nncf.quantization.advanced_parameters import AdvancedCompressionParameters -from nncf.quantization.advanced_parameters import CustomAnnotation from nncf.quantization.advanced_parameters import GroupSizeFallbackMode +from nncf.quantization.advanced_parameters import ScopedConfig from nncf.quantization.advanced_parameters import convert_to_dict_recursively from nncf.quantization.algorithms.algorithm import Algorithm from nncf.quantization.algorithms.weight_compression.awq import AWQ @@ -89,7 +89,7 @@ def get_weight_compression_configuration( sensitivity_metric: SensitivityMetric | None = None, backup_mode: BackupMode | None = None, advanced_parameters: AdvancedCompressionParameters | None = None, - custom_annotation: list[CustomAnnotation] | None = None, + scoped_configs: list[ScopedConfig] | None = None, ) -> dict[str, Any]: """ Generates a configuration dictionary for weight compression based on the provided parameters. @@ -126,7 +126,7 @@ def get_weight_compression_configuration( ), "backup_mode": backup_mode, "advanced_parameters": advanced_parameters or AdvancedCompressionParameters(), - "custom_annotation": custom_annotation, + "scoped_configs": scoped_configs, } @@ -157,71 +157,59 @@ def validate_fixed_group_size( raise nncf.ValidationError(msg) -def validate_custom_annotation( - custom_annotation: list[CustomAnnotation] | None, +def validate_scoped_configs( + scoped_configs: list[ScopedConfig] | None, advanced_parameters: AdvancedCompressionParameters | None = None, ) -> None: """ - Validates the user-defined custom annotation. + Validates the user-defined scoped configs. - :param custom_annotation: List of custom annotations to validate. + :param scoped_configs: List of scoped configs to validate. :param advanced_parameters: Advanced compression parameters that may affect the validation. """ - if custom_annotation is None: + if scoped_configs is None: return - if not isinstance(custom_annotation, (list, tuple)): + if not isinstance(scoped_configs, (list, tuple)): msg = ( - "The custom_annotation option expects a list of nncf.CustomAnnotation instances, " - f"but {type(custom_annotation).__name__} is given." + "The scoped_configs option expects a list of nncf.ScopedConfig instances, " + f"but {type(scoped_configs).__name__} is given." ) raise nncf.ValidationError(msg) - for annotation in custom_annotation: - if not isinstance(annotation, CustomAnnotation): + for scoped_config in scoped_configs: + if not isinstance(scoped_config, ScopedConfig): msg = ( - "The custom_annotation argument expects a list of nncf.CustomAnnotation instances, " - f"but an element of type {type(annotation).__name__} is given." + "The scoped_configs option expects a list of nncf.ScopedConfig instances, " + f"but an element of type {type(scoped_config).__name__} is given." ) raise nncf.ValidationError(msg) - if not isinstance(annotation.config, WeightCompressionConfig): + if not isinstance(scoped_config.scope, Scope): msg = ( - "The config of a custom annotation must be an nncf.WeightCompressionConfig instance, " - f"but {type(annotation.config).__name__} is given." + "The scope of a scoped config must be an nncf.Scope instance, " + f"but {type(scoped_config.scope).__name__} is given." ) raise nncf.ValidationError(msg) - if not isinstance(annotation.scope, Scope): + # Resolves the default group size of the mode and validates the given one + config = scoped_config.compression_config + if config.is_codebook: msg = ( - "The scope of a custom annotation must be an nncf.Scope instance, " - f"but {type(annotation.scope).__name__} is given." - ) - raise nncf.ValidationError(msg) - - if annotation.config.is_codebook: - msg = ( - "Codebook compression modes are not supported by the custom annotation, but the " - f"{annotation.config.mode.value} mode is given." + "Codebook compression modes are not supported by the scoped configs, but the " + f"{config.mode.value} mode is given." ) raise nncf.ParameterNotSupportedError(msg) - if annotation.config.codebook_values is not None: + if config.mode in INT8_MODES and config.group_size != -1: msg = ( - "Codebook values are not supported by the custom annotation, but they are given for the " - f"{annotation.config.mode.value} mode." + "INT8 modes require per-channel quantization in 8 bit, so the group size of a scoped config " + f"with the {config.mode.value} mode must be -1, but " + f"group_size={config.group_size} is given." ) raise nncf.ParameterNotSupportedError(msg) - if annotation.config.mode in INT8_MODES and annotation.config.group_size != -1: - msg = ( - "INT8 modes require per-channel quantization in 8 bit, so the group size of a custom annotation " - f"with the {annotation.config.mode.value} mode must be -1, but " - f"group_size={annotation.config.group_size} is given." - ) - raise nncf.ParameterNotSupportedError(msg) - - validate_fixed_group_size(annotation.config.mode, annotation.config.group_size, advanced_parameters) + validate_fixed_group_size(config.mode, config.group_size, advanced_parameters) def check_user_compression_configuration( @@ -392,7 +380,7 @@ def __init__( backup_mode: BackupMode, compression_format: CompressionFormat = CompressionFormat.DQ, advanced_parameters: AdvancedCompressionParameters | None = None, - custom_annotation: list[CustomAnnotation] | None = None, + scoped_configs: list[ScopedConfig] | None = None, ): """ :param mode: Defines a mode for weight compression. @@ -439,9 +427,9 @@ def __init__( FP8_E4M3 stands for FP8 format with E4M3 values sharing group-level fp16 scale. :param compression_format: Describes the format in which the model is saved after weight compression. :param advanced_parameters: advanced parameters for algorithms in compression pipeline. - :param custom_annotation: A list of user-defined weight compression configurations bound to portions of - the model. The configuration of a custom annotation takes precedence over the one assigned by the - algorithm for all the matched nodes. If several annotations match the same node, the last one in the + :param scoped_configs: A list of user-defined weight compression configurations bound to portions of + the model. The configuration of a scoped config takes precedence over the one assigned by the + algorithm for all the matched nodes. If several scoped configs match the same node, the last one in the list takes precedence. """ super().__init__() @@ -465,7 +453,7 @@ def __init__( self._advanced_parameters = ( advanced_parameters if advanced_parameters is not None else AdvancedCompressionParameters() ) - self._custom_annotation = custom_annotation + self._scoped_configs = scoped_configs criterion_cls = MIXED_PRECISION_CRITERIA.get(self._sensitivity_metric) self._mixed_precision_algo = criterion_cls(self._ratio, self._subset_size) @@ -1030,8 +1018,8 @@ def collect_statistics_and_statistic_points( weight_params = all_weight_params if self._backup_mode == BackupMode.NONE: # Only the compressed weights need statistics. When backup mode is none, only ratio defining params are - # considered. In the case of custom annotation, some weights may be considered backup nodes, but they - # may also be annotated. This ensures that such annotated nodes are also considered for statistics + # considered. In the case of scoped configs, some weights may be considered backup nodes, but they + # may also be matched by a scoped config. This ensures that such nodes are also considered for statistics # collection. ratio_defining_weight_names = set(wp.weight_name for wp in ratio_defining_params) weight_params = ratio_defining_params + [ @@ -1230,56 +1218,59 @@ def _get_node_name_to_weight_names_mapping(self, graph: NNCFGraph) -> dict[str, @staticmethod def _get_weight_name_to_config_mapping( - custom_annotation: list[CustomAnnotation], + scoped_configs: list[ScopedConfig], graph: NNCFGraph, node_name_to_weight_names_mapping: dict[str, list[str]], ) -> tuple[dict[str, WeightCompressionConfig], set[str]]: """ - Matches the custom annotation against the weights of the nodes that can be compressed. + Matches the scoped configs against the weights of the nodes that can be compressed. - :param custom_annotation: List of user-defined custom annotations. + :param scoped_configs: List of user-defined scoped configs. :param graph: NNCFGraph instance. :param node_name_to_weight_names_mapping: A mapping from a node name to the names of the weights of that node. - :return: A tuple of the mapping from a weight name to the annotated configuration and the weight names - matched by several annotations. + :return: A tuple of the mapping from a weight name to the scoped configuration and the weight names + matched by several scoped configs. """ weight_name_to_config_mapping: dict[str, WeightCompressionConfig] = {} - overlapped_annotation_weight_names = set() - for annotation in custom_annotation: - annotated_names = get_node_names_from_scope(annotation.scope, graph, strict=annotation.scope.validate) - annotated_weight_names = set() - for node_name in annotated_names: + overlapped_weight_names = set() + for scoped_config in scoped_configs: + scoped_node_names = get_node_names_from_scope( + scoped_config.scope, graph, strict=scoped_config.scope.validate + ) + scoped_weight_names = set() + for node_name in scoped_node_names: weight_names = node_name_to_weight_names_mapping.get(node_name) if weight_names is not None: - annotated_weight_names.update(weight_names) - # The weights annotated by the current annotation overlap with the ones annotated before it - overlapped_annotation_weight_names.update( - name for name in annotated_weight_names if name in weight_name_to_config_mapping + scoped_weight_names.update(weight_names) + # The weights matched by the current scoped config overlap with the ones matched before it + overlapped_weight_names.update( + name for name in scoped_weight_names if name in weight_name_to_config_mapping ) - for weight_name in annotated_weight_names: - weight_name_to_config_mapping[weight_name] = annotation.config + config = scoped_config.compression_config + for weight_name in scoped_weight_names: + weight_name_to_config_mapping[weight_name] = config - return weight_name_to_config_mapping, overlapped_annotation_weight_names + return weight_name_to_config_mapping, overlapped_weight_names - def _get_custom_annotation_configs( - self, custom_annotation: list[CustomAnnotation], graph: NNCFGraph + def _get_scoped_configs_mapping( + self, scoped_configs: list[ScopedConfig], graph: NNCFGraph ) -> dict[str, WeightCompressionConfig]: """ - Maps the weights matched by the custom annotation to the user-defined weight compression configuration. + Maps the weights matched by the scoped configs to the user-defined weight compression configuration. - :param custom_annotation: List of user-defined custom annotations. + :param scoped_configs: List of user-defined scoped configs. :param graph: NNCFGraph instance. :return: A mapping from a weight name to the user-defined weight compression configuration. """ node_name_to_weight_names_mapping = self._get_node_name_to_weight_names_mapping(graph) - weight_name_to_config_mapping, overlapped_annotation_weight_names = self._get_weight_name_to_config_mapping( - custom_annotation, graph, node_name_to_weight_names_mapping + weight_name_to_config_mapping, overlapped_weight_names = self._get_weight_name_to_config_mapping( + scoped_configs, graph, node_name_to_weight_names_mapping ) - if overlapped_annotation_weight_names: + if overlapped_weight_names: overlapped_configs = [] - for weight_name in sorted(overlapped_annotation_weight_names): + for weight_name in sorted(overlapped_weight_names): config = weight_name_to_config_mapping[weight_name] node_names = sorted( node_name @@ -1288,21 +1279,21 @@ def _get_custom_annotation_configs( ) overlapped_configs.append(f"{', '.join(node_names)} ({config.mode.value}, {config.group_size})") nncf_logger.warning( - "Several custom annotations match the same nodes. The configuration of the last matched custom " - "annotation takes precedence and is used for the following nodes:\n\t" + "\n\t".join(overlapped_configs) + "Several scoped configs match the same nodes. The configuration of the last matched scoped " + "config takes precedence and is used for the following nodes:\n\t" + "\n\t".join(overlapped_configs) ) return weight_name_to_config_mapping - def _drop_annotation_configs_that_can_not_be_assigned( + def _drop_scoped_configs_that_can_not_be_assigned( self, weight_name_to_config_mapping: dict[str, WeightCompressionConfig], all_weight_params: list[WeightCompressionParameters], skipped_weight_params: list[WeightCompressionParameters], ) -> dict[str, WeightCompressionConfig]: """ - Validate and drop the annotated configurations that can not be assigned to a weight + Validate and drop the scoped configurations that can not be assigned to a weight - :param weight_name_to_config_mapping: A mapping from a weight name to the annotated compression configuration. + :param weight_name_to_config_mapping: A mapping from a weight name to the scoped compression configuration. :param all_weight_params: List of all weight compression parameters that can be compressed. :param skipped_weight_params: List of weight compression parameters that are not compressed. :return: The mapping with the configurations that can not be assigned dropped. @@ -1318,7 +1309,7 @@ def _drop_annotation_configs_that_can_not_be_assigned( unsupported_node_names.add(w_params.node_with_weight.node_name) if unsupported_node_names: nncf_logger.warning( - "The compression mode defined by the custom annotation is not supported for the data type of the " + "The compression mode defined by the scoped configs is not supported for the data type of the " "following nodes. They are kept with the configuration assigned by the algorithm:\n\t" + "\n\t".join(sorted(unsupported_node_names)) ) @@ -1331,43 +1322,43 @@ def _drop_annotation_configs_that_can_not_be_assigned( return weight_name_to_config_mapping @staticmethod - def _assign_annotation_configs( + def _assign_scoped_configs( weight_name_to_config_mapping: dict[str, WeightCompressionConfig], all_weight_params: list[WeightCompressionParameters], ratio_defining_params: list[WeightCompressionParameters], ) -> list[WeightCompressionParameters]: """ - Assigns the annotated configurations to the matched weight parameters. + Assigns the scoped configurations to the matched weight parameters. - :param weight_name_to_config_mapping: A mapping from a weight name to the annotated compression configuration. + :param weight_name_to_config_mapping: A mapping from a weight name to the scoped compression configuration. :param all_weight_params: List of all weight compression parameters that can be compressed. :param ratio_defining_params: List of ratio-defining parameters. :return: The updated list of ratio-defining parameters. """ - annotated_weight_names = set() + scoped_weight_names = set() for w_params in all_weight_params: config = weight_name_to_config_mapping.get(w_params.weight_name) if config is not None: # A deep copy is required for some methods/modes like codebook w_params.compression_config = copy.deepcopy(config) - annotated_weight_names.add(w_params.weight_name) + scoped_weight_names.add(w_params.weight_name) - # The annotated parameters are excluded from the ratio-defining ones, so that the mixed precision - # algorithm, the `all_layers` and the last layer rules can not override the annotated configuration - return [w_params for w_params in ratio_defining_params if w_params.weight_name not in annotated_weight_names] + # The scoped parameters are excluded from the ratio-defining ones, so that the mixed precision + # algorithm, the `all_layers` and the last layer rules can not override the scoped configuration + return [w_params for w_params in ratio_defining_params if w_params.weight_name not in scoped_weight_names] @staticmethod - def _move_annotated_ignored_nodes_from_skipped_to_compressed( + def _move_scoped_ignored_nodes_from_skipped_to_compressed( weight_name_to_config_mapping: dict[str, WeightCompressionConfig], graph: NNCFGraph, all_weight_params: list[WeightCompressionParameters], skipped_weight_params: list[WeightCompressionParameters], ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters]]: """ - Moves the annotated weight parameters that are not compressed, e.g. the ignored ones, from the skipped - parameters to the compressed ones, so that custom annotation takes precedence over the ignored scope. + Moves the scoped weight parameters that are not compressed, e.g. the ignored ones, from the skipped + parameters to the compressed ones, so that scoped config takes precedence over the ignored scope. - :param weight_name_to_config_mapping: A mapping from a weight name to the annotated compression configuration. + :param weight_name_to_config_mapping: A mapping from a weight name to the scoped compression configuration. :param graph: NNCFGraph instance. :param all_weight_params: List of all weight compression parameters that can be compressed. :param skipped_weight_params: List of weight compression parameters that are not compressed. @@ -1376,8 +1367,8 @@ def _move_annotated_ignored_nodes_from_skipped_to_compressed( compressed_weight_names = {w_params.weight_name for w_params in all_weight_params} restored_weight_params = [] for w_params in skipped_weight_params: - is_annotated = w_params.weight_name in weight_name_to_config_mapping - if is_annotated and w_params.weight_name not in compressed_weight_names: + is_scoped = w_params.weight_name in weight_name_to_config_mapping + if is_scoped and w_params.weight_name not in compressed_weight_names: compressed_weight_names.add(w_params.weight_name) restored_weight_params.append(w_params) @@ -1386,7 +1377,7 @@ def _move_annotated_ignored_nodes_from_skipped_to_compressed( nncf_logger.warning( "The following nodes are excluded from the compression, e.g. by the ignored scope, but are matched " - "by the custom annotation. They will be compressed with the user-defined configuration:\n\t" + "by the scoped configs. They will be compressed with the user-defined configuration:\n\t" + "\n\t".join(sorted(w_params.node_with_weight.node_name for w_params in restored_weight_params)) ) restored_weight_names = {w_params.weight_name for w_params in restored_weight_params} @@ -1399,7 +1390,7 @@ def _move_annotated_ignored_nodes_from_skipped_to_compressed( all_weight_params.sort(key=lambda wp: node_name_to_position_mapping[wp.node_with_weight.node_name]) return all_weight_params, skipped_weight_params - def apply_custom_annotation( + def apply_scoped_configs( self, graph: NNCFGraph, all_weight_params: list[WeightCompressionParameters], @@ -1411,14 +1402,14 @@ def apply_custom_annotation( list[WeightCompressionParameters], ]: """ - Applies the user-defined weight compression configurations to the parameters matched by the annotation. + Applies the user-defined weight compression configurations to the parameters matched by the scoped configs. The flow of this function is as follows: - 1. Convert the annotation to a mapping from a weight name to the configuration. - 2. Drop the annotated configurations that can not be assigned to a weight. - 3. Move the annotated weight parameters that are not compressed initially, e.g. the ignored ones, to the + 1. Convert the scoped configs to a mapping from a weight name to the configuration. + 2. Drop the scoped configurations that can not be assigned to a weight. + 3. Move the scoped weight parameters that are not compressed initially, e.g. the ignored ones, to the compressed ones. 4. Assign the remaining configurations to the matched weight parameters. - 5. Apply the group size fallback mode to the annotated group size values. + 5. Apply the group size fallback mode to the scoped group size values. :param graph: NNCFGraph instance. :param all_weight_params: List of all weight compression parameters that can be compressed. @@ -1426,20 +1417,20 @@ def apply_custom_annotation( :param skipped_weight_params: List of weight compression parameters that are not compressed. :return: The updated tuple of all, ratio-defining and skipped weight compression parameters. """ - if not self._custom_annotation: + if not self._scoped_configs: return all_weight_params, ratio_defining_params, skipped_weight_params - weight_name_to_config_mapping = self._get_custom_annotation_configs(self._custom_annotation, graph) - weight_name_to_config_mapping = self._drop_annotation_configs_that_can_not_be_assigned( + weight_name_to_config_mapping = self._get_scoped_configs_mapping(self._scoped_configs, graph) + weight_name_to_config_mapping = self._drop_scoped_configs_that_can_not_be_assigned( weight_name_to_config_mapping, all_weight_params, skipped_weight_params ) - # There might be some nodes which were ignored but a custom annotation was passed for them. - # Such nodes are stored in skipped_weight_params. They should be assigned a config from the custom - # annotation if available. - all_weight_params, skipped_weight_params = self._move_annotated_ignored_nodes_from_skipped_to_compressed( + # There might be some nodes which were ignored but a scoped config was passed for them. + # Such nodes are stored in skipped_weight_params. They should be assigned a config from the scoped + # configs if available. + all_weight_params, skipped_weight_params = self._move_scoped_ignored_nodes_from_skipped_to_compressed( weight_name_to_config_mapping, graph, all_weight_params, skipped_weight_params ) - ratio_defining_params = self._assign_annotation_configs( + ratio_defining_params = self._assign_scoped_configs( weight_name_to_config_mapping, all_weight_params, ratio_defining_params ) # The resulting group size values are validated together with the rest of the weights in the apply() method @@ -1456,38 +1447,38 @@ def _handle_group_size_fallback( skipped_weight_params: list[WeightCompressionParameters], ) -> tuple[list[WeightCompressionParameters], list[WeightCompressionParameters]]: """ - Applies the configured group size fallback mode to the weights annotated by the custom annotation. + Applies the configured group size fallback mode to the weights matched by the scoped configs. - The group size of an annotated weight is defined by the user, so the fallback mode is applied to that value + The group size of a scoped weight is defined by the user, so the fallback mode is applied to that value instead of the group size configured for the algorithm. - :param weight_name_to_config_mapping: A mapping from a weight name to the applied annotation configuration. + :param weight_name_to_config_mapping: A mapping from a weight name to the applied scoped configuration. :param all_weight_params: List of all weight compression parameters that can be compressed. :param skipped_weight_params: List of weight compression parameters that are not compressed. :return: The updated tuple of all and skipped weight compression parameters. """ - annotated_weight_params = [ + scoped_weight_params = [ w_params for w_params in all_weight_params if w_params.weight_name in weight_name_to_config_mapping ] - group_sizes = self._get_requested_group_sizes(annotated_weight_params) + group_sizes = self._get_requested_group_sizes(scoped_weight_params) if self._group_size_fallback_mode == GroupSizeFallbackMode.IGNORE: - all_weight_params, annotated_weight_params, skipped_weight_params = self._handle_ignore_group_size_fallback( - all_weight_params, annotated_weight_params, skipped_weight_params, group_sizes + all_weight_params, scoped_weight_params, skipped_weight_params = self._handle_ignore_group_size_fallback( + all_weight_params, scoped_weight_params, skipped_weight_params, group_sizes ) elif self._group_size_fallback_mode == GroupSizeFallbackMode.ADJUST: valid_weight_params, group_size_values = self._handle_adjust_group_size_fallback( - annotated_weight_params, group_sizes + scoped_weight_params, group_sizes ) valid_weight_names = set(w_params.weight_name for w_params in valid_weight_params) - for w_params in annotated_weight_params: + for w_params in scoped_weight_params: if w_params.weight_name in valid_weight_names: w_params.compression_config = dataclasses.replace( w_params.compression_config, group_size=group_size_values[w_params.weight_name] ) else: - # A valid adjusted group size wasn't found, the annotated config can not be applied + # A valid adjusted group size wasn't found, the scoped config can not be applied w_params.compression_config = self._get_backup_config(w_params.weight_dtype) - annotated_weight_params = valid_weight_params + scoped_weight_params = valid_weight_params return all_weight_params, skipped_weight_params @@ -1505,7 +1496,7 @@ def apply( model, graph ) # Assign the user-defined weight compression configurations - all_weight_params, ratio_defining_params, skipped_weight_params = self.apply_custom_annotation( + all_weight_params, ratio_defining_params, skipped_weight_params = self.apply_scoped_configs( graph, all_weight_params, ratio_defining_params, skipped_weight_params ) # Collect statistics for the weights compression @@ -1514,7 +1505,7 @@ def apply( ) # Apply Mixed precision algorithm to ratio defining parameters self.apply_mixed_precision(ratio_defining_params, model, graph, statistic_points) - # All the weights are validated, since the group size of a weight can also be defined by the custom annotation + # All the weights are validated, since the group size of a weight can also be defined by the scoped configs self.validate_group_size(all_weight_params) # Print statistics @@ -1635,7 +1626,7 @@ def apply_with_parameters( "backup_mode": self._backup_mode.value, "compression_format": self._compression_format.value, "advanced_parameters": convert_to_dict_recursively(self._advanced_parameters), - "custom_annotation": self._custom_annotation or [], + "scoped_configs": self._scoped_configs or [], }, algo_name="weight_compression", ) diff --git a/src/nncf/quantization/algorithms/weight_compression/config.py b/src/nncf/quantization/algorithms/weight_compression/config.py index 05d44e8f008..b238285e052 100644 --- a/src/nncf/quantization/algorithms/weight_compression/config.py +++ b/src/nncf/quantization/algorithms/weight_compression/config.py @@ -16,7 +16,6 @@ import numpy as np from nncf.common.graph.graph import NNCFNode -from nncf.common.utils.api_marker import api from nncf.errors import InternalError from nncf.errors import ValidationError from nncf.parameters import CompressWeightsMode @@ -52,7 +51,6 @@ def get_default_group_size(mode: CompressWeightsMode) -> int: return DEFAULT_GROUP_SIZE -@api(canonical_alias="nncf.WeightCompressionConfig") @dataclass class WeightCompressionConfig: """ diff --git a/src/nncf/quantization/quantize_model.py b/src/nncf/quantization/quantize_model.py index f7585fb14b3..9b8471682b3 100644 --- a/src/nncf/quantization/quantize_model.py +++ b/src/nncf/quantization/quantize_model.py @@ -30,14 +30,14 @@ from nncf.quantization.advanced_parameters import AdvancedAccuracyRestorerParameters from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters -from nncf.quantization.advanced_parameters import CustomAnnotation +from nncf.quantization.advanced_parameters import ScopedConfig from nncf.quantization.algorithms.accuracy_control.evaluator import MetricResults from nncf.quantization.algorithms.hyperparameter_tuner.algorithm import HyperparameterTuner from nncf.quantization.algorithms.hyperparameter_tuner.param_grid import get_quantization_param_grids from nncf.quantization.algorithms.post_training.pipeline import create_ptq_pipeline from nncf.quantization.algorithms.weight_compression.algorithm import check_user_compression_configuration from nncf.quantization.algorithms.weight_compression.algorithm import get_weight_compression_configuration -from nncf.quantization.algorithms.weight_compression.algorithm import validate_custom_annotation +from nncf.quantization.algorithms.weight_compression.algorithm import validate_scoped_configs from nncf.quantization.telemetry_extractors import CompressionStartedWithCompressWeightsApi from nncf.quantization.telemetry_extractors import CompressionStartedWithQuantizeApi from nncf.quantization.telemetry_extractors import CompressionStartedWithQuantizeWithAccuracyControlApi @@ -399,21 +399,21 @@ def _validate_compression_modes_supported_by_backend( backend_name: str, not_supported_modes: list[CompressWeightsMode], mode: CompressWeightsMode, - custom_annotation: list[CustomAnnotation] | None, + scoped_configs: list[ScopedConfig] | None, msg: str | None = None, ) -> None: """ - Checks that neither the compression mode nor the mode defined by the custom annotation + Checks that neither the compression mode nor the mode defined by the scoped configs are unsupported by the backend. :param backend_name: Name of the backend to use in the error message. :param not_supported_modes: List of the compression modes that are not supported by the backend. :param mode: Compression mode of the algorithm. - :param custom_annotation: List of the user-defined weight compression configurations, if given. + :param scoped_configs: List of the user-defined weight compression configurations, if given. :param msg: Error message to use instead of the default one :raises nncf.ParameterNotSupportedError: If any of the given modes is not supported by the backend. """ - given_modes = [mode] + [annotation.config.mode for annotation in custom_annotation or []] + given_modes = [mode] + [scoped_config.mode for scoped_config in scoped_configs or []] if any(given_mode in not_supported_modes for given_mode in given_modes): if msg is None: msg = ( @@ -454,7 +454,7 @@ def compress_weights( backup_mode: BackupMode | None = None, compression_format: CompressionFormat = CompressionFormat.DQ, advanced_parameters: AdvancedCompressionParameters | None = None, - custom_annotation: list[CustomAnnotation] | None = None, + scoped_configs: list[ScopedConfig] | None = None, ) -> TModel: """ Compress model weights. @@ -523,19 +523,19 @@ def compress_weights( :type compression_format: nncf.CompressionFormat :param advanced_parameters: Advanced parameters for compression algorithms. :type advanced_parameters: nncf.AdvancedCompressionParameters - :param custom_annotation: A list of user-defined weight compression configurations bound to portions of the + :param scoped_configs: A list of user-defined weight compression configurations bound to portions of the model. For all the matched nodes the given configuration is used instead of the one assigned by the algorithm, which makes it possible to compress certain layers, e.g. attention or MoE router layers, to a - precision different from the one defined by `mode`, `ratio` and `backup_mode`. The custom annotation + precision different from the one defined by `mode`, `ratio` and `backup_mode`. The scoped config takes precedence over the mixed precision assignment, the `ignored_scope` and the `all_layers` options. - If several annotations match the same node, the last one in the list takes precedence. For example, an - annotation of a whole block followed by an annotation of a single layer of this block compresses the block + If several scoped configs match the same node, the last one in the list takes precedence. For example, a + scoped config of a whole block followed by a scoped config of a single layer of this block compresses the block with the first configuration, except for this layer, while the opposite order compresses the whole block with the second configuration. - :type custom_annotation: list[nncf.CustomAnnotation] + :type scoped_configs: list[nncf.ScopedConfig] :return: The non-trainable model with compressed weights. """ - validate_custom_annotation(custom_annotation, advanced_parameters) + validate_scoped_configs(scoped_configs, advanced_parameters) backend = get_backend(model) compression_weights_impl: Callable[..., Any] | None = None @@ -557,7 +557,7 @@ def compress_weights( CompressWeightsMode.ADAPTIVE_CODEBOOK, CompressWeightsMode.CB4, ] - _validate_compression_modes_supported_by_backend("Torch", not_supported_modes, mode, custom_annotation) + _validate_compression_modes_supported_by_backend("Torch", not_supported_modes, mode, scoped_configs) options = {"gptq": gptq, "lora_correction": lora_correction} unsupported_options = [name for name, value in options.items() if value is not None] @@ -603,7 +603,7 @@ def compress_weights( CompressWeightsMode.ADAPTIVE_CODEBOOK, CompressWeightsMode.CB4, ] - _validate_compression_modes_supported_by_backend("Torch", not_supported_modes, mode, custom_annotation) + _validate_compression_modes_supported_by_backend("Torch", not_supported_modes, mode, scoped_configs) options = {"gptq": gptq, "lora_correction": lora_correction} unsupported_options = [name for name, value in options.items() if value is not None] @@ -649,7 +649,7 @@ def compress_weights( "OpenVINO", not_supported_modes, mode, - custom_annotation, + scoped_configs, msg=msg, ) @@ -676,7 +676,7 @@ def compress_weights( CompressWeightsMode.ADAPTIVE_CODEBOOK, CompressWeightsMode.CB4, ] - _validate_compression_modes_supported_by_backend("ONNX", not_supported_modes, mode, custom_annotation) + _validate_compression_modes_supported_by_backend("ONNX", not_supported_modes, mode, scoped_configs) options = {"gptq": gptq, "lora_correction": lora_correction} unsupported_options = [name for name, value in options.items() if value is not None] @@ -723,7 +723,7 @@ def compress_weights( sensitivity_metric, backup_mode, advanced_parameters, - custom_annotation, + scoped_configs, ) return compression_weights_impl( diff --git a/src/nncf/torch/function_hook/quantization/quantize_model.py b/src/nncf/torch/function_hook/quantization/quantize_model.py index 67170951452..506a024896b 100644 --- a/src/nncf/torch/function_hook/quantization/quantize_model.py +++ b/src/nncf/torch/function_hook/quantization/quantize_model.py @@ -26,7 +26,7 @@ from nncf.parameters import TargetDevice from nncf.quantization.advanced_parameters import AdvancedCompressionParameters from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters -from nncf.quantization.advanced_parameters import CustomAnnotation +from nncf.quantization.advanced_parameters import ScopedConfig from nncf.quantization.algorithms.post_training.algorithm import PostTrainingQuantization from nncf.quantization.algorithms.weight_compression.algorithm import WeightCompression from nncf.quantization.quantize_model import warning_model_no_batchwise_support @@ -101,7 +101,7 @@ def compress_weights_impl( backup_mode: BackupMode, compression_format: CompressionFormat, advanced_parameters: AdvancedCompressionParameters | None = None, - custom_annotation: list[CustomAnnotation] | None = None, + scoped_configs: list[ScopedConfig] | None = None, ) -> torch.nn.Module: """ Implementation of the `compress_weights()` method for the PyTorch backend. @@ -121,7 +121,7 @@ def compress_weights_impl( backup_mode, compression_format, advanced_parameters, - custom_annotation, + scoped_configs, ) graph = build_graph(model) diff --git a/tests/cross_fw/test_templates/template_test_weights_compression.py b/tests/cross_fw/test_templates/template_test_weights_compression.py index e4a43d4260c..7c1aff8dc96 100644 --- a/tests/cross_fw/test_templates/template_test_weights_compression.py +++ b/tests/cross_fw/test_templates/template_test_weights_compression.py @@ -15,6 +15,7 @@ from abc import abstractmethod from dataclasses import asdict from dataclasses import dataclass +from dataclasses import replace from functools import reduce from operator import mul from pathlib import Path @@ -349,11 +350,11 @@ def get_decompressed_weight(compressed_model: TModel, input: TTensor) -> Tensor: """Returns decompressed weight""" @pytest.mark.parametrize("backup_mode", [BackupMode.INT8_ASYM, BackupMode.NONE], ids=["int8_backup", "no_backup"]) - @pytest.mark.parametrize("annotated_mode", [None, CompressWeightsMode.INT4_SYM], ids=["default", "annotated"]) + @pytest.mark.parametrize("scoped_mode", [None, CompressWeightsMode.INT4_SYM], ids=["default", "scoped_config"]) @pytest.mark.parametrize("transpose_a", [False, True], ids=["no_tr_a", "tr_a"]) - def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a, annotated_mode, backup_mode): + def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a, scoped_mode, backup_mode): """ - Checks that act_ch_axis parameter and the config defined by the custom annotation are passed to + Checks that act_ch_axis parameter and the config defined by the scoped config are passed to calculate_quantization_params. """ calc_q_params_spy = mocker.spy(ScaleEstimation, "calculate_quantization_params") @@ -363,14 +364,9 @@ def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a, annotated input = self.to_tensor(input) dataset = Dataset([input], self.get_transform_func()) - custom_annotation = None - if annotated_mode is not None: - custom_annotation = [ - nncf.CustomAnnotation( - scope=nncf.Scope(patterns=[".*"]), - config=WeightCompressionConfig(mode=annotated_mode, group_size=8), - ) - ] + scoped_configs = None + if scoped_mode is not None: + scoped_configs = [nncf.ScopedConfig(scope=nncf.Scope(patterns=[".*"]), mode=scoped_mode, group_size=8)] with SpyWeightCompressionStatisticsContext(mocker): _ = compress_weights( @@ -382,7 +378,7 @@ def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a, annotated all_layers=True, backup_mode=backup_mode, dataset=dataset, - custom_annotation=custom_annotation, + scoped_configs=scoped_configs, ) # Verify calculate_quantization_params was called @@ -393,9 +389,9 @@ def test_scale_estimation_act_ch_axis_param(self, mocker, transpose_a, annotated call_args = calc_q_params_spy.call_args[0] assert len(call_args) >= 5, "calculate_quantization_params should have at least 5 positional arguments" - # Verify the config is the annotated one, if the custom annotation is given + # Verify the config is the scoped one, if the scoped config is given config = call_args[3] - assert config.mode == (annotated_mode or CompressWeightsMode.INT4_ASYM) + assert config.mode == (scoped_mode or CompressWeightsMode.INT4_ASYM) # Verify act_ch_axis has a valid value (should be an integer) act_ch_axis = call_args[4] @@ -672,25 +668,20 @@ def int4_mode(self, request): @pytest.mark.parametrize("is_3d_weights", [True, False]) @pytest.mark.parametrize("with_multiply", (True, False)) - @pytest.mark.parametrize("annotate_all", [False, True], ids=["default", "annotated"]) + @pytest.mark.parametrize("scope_all", [False, True], ids=["default", "scoped_config"]) def test_call_max_var_criterion_with_dataset_by_default_awq_act_matmul( - self, int4_mode, with_multiply, is_3d_weights, annotate_all, mocker + self, int4_mode, with_multiply, is_3d_weights, scope_all, mocker ): n_layers = 8 - # The first MatMul is always int8, unless the custom annotation assigns 4 bits to it - n_awq_target = n_layers if annotate_all else n_layers - 1 + # The first MatMul is always int8, unless the scoped config assigns 4 bits to it + n_awq_target = n_layers if scope_all else n_layers - 1 model = self.get_awq_act_model(is_3d_weights, with_multiply, n_layers) dataset = Dataset([self.to_tensor(np.ones([2, 8, 8], dtype=np.float32))], self.get_transform_func()) - custom_annotation = None - if annotate_all: - custom_annotation = [ - nncf.CustomAnnotation( - scope=nncf.Scope(patterns=[".*"]), - config=WeightCompressionConfig(mode=int4_mode, group_size=2), - ) - ] + scoped_configs = None + if scope_all: + scoped_configs = [nncf.ScopedConfig(scope=nncf.Scope(patterns=[".*"]), mode=int4_mode, group_size=2)] with SpyWeightCompressionStatisticsContext(mocker): model = compress_weights( @@ -700,7 +691,7 @@ def test_call_max_var_criterion_with_dataset_by_default_awq_act_matmul( group_size=2, dataset=dataset, awq=True, - custom_annotation=custom_annotation, + scoped_configs=scoped_configs, ) awq_num = self.get_num_multiply_from_awq(model) @@ -742,11 +733,11 @@ def get_ignored_scope_name(is_3d_weights) -> str: "Returns the name of the node that is excluded from the 4 bit compression in test_awq_with_excluded_node." @pytest.mark.parametrize("is_3d_weights", [True, False]) - @pytest.mark.parametrize("exclude_by_annotation", [False, True], ids=["ignored_scope", "custom_annotation"]) - def test_awq_with_excluded_node(self, mocker, is_3d_weights, exclude_by_annotation): + @pytest.mark.parametrize("exclude_by_scoped_config", [False, True], ids=["ignored_scope", "scoped_configs"]) + def test_awq_with_excluded_node(self, mocker, is_3d_weights, exclude_by_scoped_config): """ Checks that a node is kept out of the 4 bit compression when AWQ is enabled, either by the ignored scope - or by a custom annotation with an 8-bit configuration. + or by a scoped config with an 8-bit configuration. """ model = self.get_awq_model(non_mergable_pattern=False, is_3d_weights=is_3d_weights) sz = 8 @@ -761,12 +752,11 @@ def test_awq_with_excluded_node(self, mocker, is_3d_weights, exclude_by_annotati excluded_node_name = self.get_ignored_scope_name(is_3d_weights) ignored_scope = None - custom_annotation = None - if exclude_by_annotation: - custom_annotation = [ - nncf.CustomAnnotation( - scope=nncf.Scope(names=[excluded_node_name]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + scoped_configs = None + if exclude_by_scoped_config: + scoped_configs = [ + nncf.ScopedConfig( + scope=nncf.Scope(names=[excluded_node_name]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ) ] else: @@ -781,7 +771,7 @@ def test_awq_with_excluded_node(self, mocker, is_3d_weights, exclude_by_annotati dataset=dataset, awq=True, ignored_scope=ignored_scope, - custom_annotation=custom_annotation, + scoped_configs=scoped_configs, ) int4_ref_num_compressed = 4 # last MatMul is always int8; one - is excluded; total 6 matmuls @@ -1027,13 +1017,13 @@ def test_group_size_fallback_modes( f"Expected {ref_num_group_sizes} group size values, but got {num_group_sizes}." ) - # Test Custom Annotation + # Test Scoped Configs @staticmethod @abstractmethod - def get_custom_annotation_ref_path(ref_name: str) -> Path: + def get_scoped_configs_ref_path(ref_name: str) -> Path: """ - Returns the path to the reference compression configs of the given custom annotation test case. + Returns the path to the reference compression configs of the given scoped config test case. """ @abstractmethod @@ -1087,19 +1077,17 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa # The weight nodes of the sequential MatMul model are named "MatMul_" / "linear_" / "/linear/" # depending on the backend, so they are referred to by a pattern that matches the node index. @pytest.mark.parametrize( - ("model_name", "kwargs", "annotations", "ignored_scope"), + ("model_name", "kwargs", "scoped_configs", "ignored_scope"), [ pytest.param( "sequential_matmul", dict(ratio=1.0, all_layers=False), [ - ( - nncf.Scope(patterns=[".*1$"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ), - ( - nncf.Scope(patterns=[".*4$"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*4$"]), mode=CompressWeightsMode.INT4_ASYM, group_size=4 ), ], None, @@ -1109,9 +1097,8 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa "sequential_matmul", dict(ratio=0.5, all_layers=True), [ - ( - nncf.Scope(patterns=[".*1$"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$"]), mode=CompressWeightsMode.INT4_ASYM, group_size=-1 ) ], None, @@ -1121,9 +1108,8 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa "sequential_matmul", dict(ratio=1.0, all_layers=True), [ - ( - nncf.Scope(patterns=[".*1$"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ) ], IgnoredScope(patterns=[".*1$"]), @@ -1133,21 +1119,18 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa "sequential_matmul", dict(ratio=1.0, all_layers=True), [ - ( - nncf.Scope(patterns=[".*1$", ".*2$"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$", ".*2$"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ), - ( - nncf.Scope(patterns=[".*2$"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*2$"]), mode=CompressWeightsMode.INT4_ASYM, group_size=4 ), - ( - nncf.Scope(patterns=[".*2$"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT4_SYM, group_size=2), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*2$"]), mode=CompressWeightsMode.INT4_SYM, group_size=2 ), ], None, - id="overlapping_annotations", + id="overlapping_scoped_configs", ), pytest.param( "sequential_matmul", @@ -1159,9 +1142,8 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa ), ), [ - ( - nncf.Scope(patterns=[".*1$"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$"]), mode=CompressWeightsMode.INT4_ASYM, group_size=3 ) ], None, @@ -1177,9 +1159,8 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa ), ), [ - ( - nncf.Scope(patterns=[".*1$"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$"]), mode=CompressWeightsMode.INT4_ASYM, group_size=3 ) ], None, @@ -1189,9 +1170,8 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa "shared_weight", dict(ratio=1.0, all_layers=True), [ - ( - nncf.Scope(patterns=[".*"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ) ], None, @@ -1201,9 +1181,8 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa "shared_weight", dict(ratio=1.0, all_layers=True), [ - ( - nncf.Scope(patterns=[".*"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ) ], IgnoredScope(patterns=[".*"]), @@ -1212,73 +1191,68 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa pytest.param( "shared_weight", dict(ratio=1.0, all_layers=True), - # The scope is None, since the annotated node name is backend-specific - [(None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2))], + # No scope rules are given, since the matched node name is backend-specific + [nncf.ScopedConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2)], None, - id="shared_weight_annotated_by_other_node", + id="shared_weight_matched_by_other_node", ), pytest.param( "shared_weight", dict(ratio=1.0, all_layers=True), [ - ( - nncf.Scope(patterns=[".*"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ), - (None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2)), + nncf.ScopedConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2), ], None, - id="shared_weight_other_node_annotated_last", + id="shared_weight_other_node_matched_last", ), pytest.param( "shared_weight", dict(ratio=1.0, all_layers=True), [ - (None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2)), - ( - nncf.Scope(patterns=[".*"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ), ], None, - id="shared_weight_other_node_annotated_first", + id="shared_weight_other_node_matched_first", ), - # The nodes sharing a weight are annotated as a single one even if none of them is compressed, - # so that the last annotation takes precedence no matter which of the nodes it matches + # The nodes sharing a weight are matched as a single one even if none of them is compressed, + # so that the last scoped config takes precedence no matter which of the nodes it matches pytest.param( "shared_weight", dict(ratio=1.0, all_layers=True), [ - ( - nncf.Scope(patterns=[".*"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ), - (None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2)), + nncf.ScopedConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2), ], IgnoredScope(patterns=[".*"]), - id="shared_weight_ignored_other_node_annotated_last", + id="shared_weight_ignored_other_node_matched_last", ), pytest.param( "shared_weight", dict(ratio=1.0, all_layers=True), [ - (None, WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2)), - ( - nncf.Scope(patterns=[".*"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=2), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ), ], IgnoredScope(patterns=[".*"]), - id="shared_weight_ignored_other_node_annotated_first", + id="shared_weight_ignored_other_node_matched_first", ), # The weight is compressed under the node that is not ignored, so the ignored one is not restored pytest.param( "shared_weight", dict(ratio=1.0, all_layers=True), [ - ( - nncf.Scope(patterns=[".*"]), - WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ) ], None, @@ -1286,10 +1260,10 @@ def apply_with_parameters(self, model, graph, dataset, statistics, all_weight_pa ), ], ) - def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, request): + def test_scoped_configs(self, model_name, kwargs, scoped_configs, ignored_scope, request): """ - Compares the compression config assigned to each weight node with the reference one when the given custom - annotation is applied. A node that is not compressed has no config in the reference file. + Compares the compression config assigned to each weight node with the reference one when the given + scoped configs are applied. A node that is not compressed has no config in the reference file. Set the NNCF_TEST_REGEN_DOT environment variable to regenerate the reference file. """ @@ -1299,10 +1273,11 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, else: model = self.get_shared_weight_model() + # The scoped configs without the scope rules are applied to the node with the shared weight shared_weight_scope = nncf.Scope(names=[self.get_shared_weight_node_name()]) - custom_annotation = [ - nncf.CustomAnnotation(scope=shared_weight_scope if scope is None else scope, config=config) - for scope, config in annotations + scoped_configs = [ + scoped_config if scoped_config.scope.patterns else replace(scoped_config, scope=shared_weight_scope) + for scoped_config in scoped_configs ] if test_id == "shared_weight_with_ignored_compressed_node": ignored_scope = IgnoredScope(names=[self.get_node_with_shared_weight_name()]) @@ -1313,42 +1288,38 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, group_size=-1, ignored_scope=ignored_scope, **kwargs, - custom_annotation=custom_annotation, + scoped_configs=scoped_configs, ) - ref_path = self.get_custom_annotation_ref_path(test_id) + ref_path = self.get_scoped_configs_ref_path(test_id) if os.getenv("NNCF_TEST_REGEN_DOT") is not None: dump_to_json(ref_path, configs) assert configs == load_json(ref_path) @pytest.mark.parametrize( - ("custom_annotation", "ignored_scope", "warning"), + ("scoped_configs", "ignored_scope", "warning"), [ pytest.param( [ - nncf.CustomAnnotation( - scope=nncf.Scope(patterns=[".*1$"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ), - nncf.CustomAnnotation( - scope=nncf.Scope(patterns=[".*1$"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=4), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$"]), mode=CompressWeightsMode.INT4_ASYM, group_size=4 ), - nncf.CustomAnnotation( - scope=nncf.Scope(patterns=[".*1$"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_SYM, group_size=2), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$"]), mode=CompressWeightsMode.INT4_SYM, group_size=2 ), ], None, - "Several custom annotations match the same nodes", - id="overlapping_annotations", + "Several scoped configs match the same nodes", + id="overlapping_scoped_configs", ), pytest.param( [ - nncf.CustomAnnotation( - scope=nncf.Scope(patterns=[".*1$"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*1$"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ) ], IgnoredScope(patterns=[".*1$"]), @@ -1357,9 +1328,9 @@ def test_custom_annotation(self, model_name, kwargs, annotations, ignored_scope, ), ], ) - def test_custom_annotation_warning(self, custom_annotation, ignored_scope, warning): + def test_scoped_configs_warning(self, scoped_configs, ignored_scope, warning): """ - Checks that appropriate warnings are logged when the custom annotation is overridden or has no effect. + Checks that appropriate warnings are logged when the scoped config is overridden or has no effect. """ with patch.object(nncf_logger, "warning") as mock_warning: compress_weights( @@ -1369,7 +1340,7 @@ def test_custom_annotation_warning(self, custom_annotation, ignored_scope, warni group_size=-1, all_layers=True, ignored_scope=ignored_scope, - custom_annotation=custom_annotation, + scoped_configs=scoped_configs, ) messages = [args[0] for args, _ in mock_warning.call_args_list] @@ -1378,42 +1349,31 @@ def test_custom_annotation_warning(self, custom_annotation, ignored_scope, warni assert len(matched) == 1, f"Expected a single '{warning}' warning, but got: {messages}" @pytest.mark.parametrize( - ("custom_annotation", "error"), + ("scoped_configs", "error"), [ - pytest.param(nncf.CustomAnnotation(), nncf.ValidationError, id="not_a_list"), - pytest.param(["anything"], nncf.ValidationError, id="not_an_annotation"), - pytest.param([nncf.CustomAnnotation(scope=nncf.IgnoredScope())], nncf.ValidationError, id="wrong_scope"), - pytest.param([nncf.CustomAnnotation(config="anything")], nncf.ValidationError, id="wrong_config"), + pytest.param(nncf.ScopedConfig(), nncf.ValidationError, id="not_a_list"), + pytest.param(["anything"], nncf.ValidationError, id="not_a_scoped_config"), + pytest.param([nncf.Scope(patterns=[".*"])], nncf.ValidationError, id="scope"), pytest.param( - [nncf.CustomAnnotation(config=WeightCompressionConfig(mode=CompressWeightsMode.CODEBOOK))], - nncf.ParameterNotSupportedError, - id="codebook_mode", + [WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1)], + nncf.ValidationError, + id="weight_compression_config", ), pytest.param( - [ - nncf.CustomAnnotation( - config=WeightCompressionConfig( - mode=CompressWeightsMode.INT4_SYM, group_size=-1, codebook_values=Tensor(CB4_QUANTILES) - ) - ) - ], + [nncf.ScopedConfig(mode=CompressWeightsMode.CODEBOOK)], nncf.ParameterNotSupportedError, - id="codebook_values", + id="codebook_mode", ), + pytest.param([nncf.ScopedConfig(scope="anything")], nncf.ValidationError, id="wrong_scope"), pytest.param( - [ - nncf.CustomAnnotation( - config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=4) - ) - ], + [nncf.ScopedConfig(mode=CompressWeightsMode.INT8_SYM, group_size=4)], nncf.ParameterNotSupportedError, id="int8_with_group_size", ), pytest.param( [ - nncf.CustomAnnotation( - scope=nncf.Scope(names=["unknown_node"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT8_SYM, group_size=-1), + nncf.ScopedConfig( + scope=nncf.Scope(names=["unknown_node"]), mode=CompressWeightsMode.INT8_SYM, group_size=-1 ) ], nncf.ValidationError, @@ -1421,9 +1381,8 @@ def test_custom_annotation_warning(self, custom_annotation, ignored_scope, warni ), pytest.param( [ - nncf.CustomAnnotation( - scope=nncf.Scope(patterns=[".*"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_ASYM, group_size=3), + nncf.ScopedConfig( + scope=nncf.Scope(patterns=[".*"]), mode=CompressWeightsMode.INT4_ASYM, group_size=3 ) ], InvalidGroupSizeError, @@ -1436,20 +1395,15 @@ def test_custom_annotation_warning(self, custom_annotation, ignored_scope, warni ), ], ) - def test_invalid_custom_annotation(self, custom_annotation, error, request): + def test_invalid_scoped_configs(self, scoped_configs, error, request): """ - Checks that an error is raised for an invalid custom annotation. + Checks that an error is raised for an invalid scoped config. """ if request.node.callspec.id == "mode_not_supported_by_backend": mode = self.get_mode_not_supported_by_backend() if mode is None: pytest.skip("The backend supports all the compression modes") - custom_annotation = [ - nncf.CustomAnnotation( - scope=nncf.Scope(patterns=[".*"]), - config=WeightCompressionConfig(mode=mode, group_size=-1), - ) - ] + scoped_configs = [nncf.ScopedConfig(scope=nncf.Scope(patterns=[".*"]), mode=mode, group_size=-1)] with pytest.raises(error): compress_weights( @@ -1458,7 +1412,7 @@ def test_invalid_custom_annotation(self, custom_annotation, error, request): ratio=1.0, group_size=-1, all_layers=True, - custom_annotation=custom_annotation, + scoped_configs=scoped_configs, ) @pytest.mark.parametrize("is_3d_weights", [True, False]) diff --git a/tests/onnx/data/references_custom_annotation/group_size_fallback_adjust.json b/tests/onnx/data/references_scoped_configs/group_size_fallback_adjust.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/group_size_fallback_adjust.json rename to tests/onnx/data/references_scoped_configs/group_size_fallback_adjust.json diff --git a/tests/onnx/data/references_custom_annotation/group_size_fallback_ignore.json b/tests/onnx/data/references_scoped_configs/group_size_fallback_ignore.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/group_size_fallback_ignore.json rename to tests/onnx/data/references_scoped_configs/group_size_fallback_ignore.json diff --git a/tests/onnx/data/references_custom_annotation/ignored_scope.json b/tests/onnx/data/references_scoped_configs/ignored_scope.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/ignored_scope.json rename to tests/onnx/data/references_scoped_configs/ignored_scope.json diff --git a/tests/onnx/data/references_custom_annotation/mixed_precision.json b/tests/onnx/data/references_scoped_configs/mixed_precision.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/mixed_precision.json rename to tests/onnx/data/references_scoped_configs/mixed_precision.json diff --git a/tests/onnx/data/references_custom_annotation/overlapping_annotations.json b/tests/onnx/data/references_scoped_configs/overlapping_scoped_configs.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/overlapping_annotations.json rename to tests/onnx/data/references_scoped_configs/overlapping_scoped_configs.json diff --git a/tests/onnx/data/references_custom_annotation/primary_and_backup_precision.json b/tests/onnx/data/references_scoped_configs/primary_and_backup_precision.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/primary_and_backup_precision.json rename to tests/onnx/data/references_scoped_configs/primary_and_backup_precision.json diff --git a/tests/onnx/data/references_custom_annotation/shared_weight.json b/tests/onnx/data/references_scoped_configs/shared_weight.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/shared_weight.json rename to tests/onnx/data/references_scoped_configs/shared_weight.json diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json b/tests/onnx/data/references_scoped_configs/shared_weight_ignored_other_node_matched_first.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json rename to tests/onnx/data/references_scoped_configs/shared_weight_ignored_other_node_matched_first.json diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json b/tests/onnx/data/references_scoped_configs/shared_weight_ignored_other_node_matched_last.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json rename to tests/onnx/data/references_scoped_configs/shared_weight_ignored_other_node_matched_last.json diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/onnx/data/references_scoped_configs/shared_weight_matched_by_other_node.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/shared_weight_annotated_by_other_node.json rename to tests/onnx/data/references_scoped_configs/shared_weight_matched_by_other_node.json diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_first.json b/tests/onnx/data/references_scoped_configs/shared_weight_other_node_matched_first.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_first.json rename to tests/onnx/data/references_scoped_configs/shared_weight_other_node_matched_first.json diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_last.json b/tests/onnx/data/references_scoped_configs/shared_weight_other_node_matched_last.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/shared_weight_other_node_annotated_last.json rename to tests/onnx/data/references_scoped_configs/shared_weight_other_node_matched_last.json diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_compressed_node.json b/tests/onnx/data/references_scoped_configs/shared_weight_with_ignored_compressed_node.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_compressed_node.json rename to tests/onnx/data/references_scoped_configs/shared_weight_with_ignored_compressed_node.json diff --git a/tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_scope.json b/tests/onnx/data/references_scoped_configs/shared_weight_with_ignored_scope.json similarity index 100% rename from tests/onnx/data/references_custom_annotation/shared_weight_with_ignored_scope.json rename to tests/onnx/data/references_scoped_configs/shared_weight_with_ignored_scope.json diff --git a/tests/onnx/quantization/test_weights_compression.py b/tests/onnx/quantization/test_weights_compression.py index 1204fe12301..07437d77042 100644 --- a/tests/onnx/quantization/test_weights_compression.py +++ b/tests/onnx/quantization/test_weights_compression.py @@ -50,7 +50,7 @@ from tests.onnx.conftest import ONNX_TEST_ROOT REFERENCE_SCALES_DIR = ONNX_TEST_ROOT / "data" / "reference_scales" -CUSTOM_ANNOTATION_REFERENCES_DIR = ONNX_TEST_ROOT / "data" / "references_custom_annotation" +SCOPED_CONFIGS_REFERENCES_DIR = ONNX_TEST_ROOT / "data" / "references_scoped_configs" UNSUPPORTED_MODES = ( CompressWeightsMode.NF4, @@ -762,8 +762,8 @@ def get_mode_not_supported_by_backend() -> CompressWeightsMode | None: return CompressWeightsMode.NF4 @staticmethod - def get_custom_annotation_ref_path(ref_name: str) -> Path: - return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" + def get_scoped_configs_ref_path(ref_name: str) -> Path: + return SCOPED_CONFIGS_REFERENCES_DIR / f"{ref_name}.json" @staticmethod def get_transform_func() -> Callable[..., Any] | None: diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/group_size_fallback_adjust.json b/tests/openvino/native/data/2026.1/references_scoped_configs/group_size_fallback_adjust.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/group_size_fallback_adjust.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/group_size_fallback_adjust.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/group_size_fallback_ignore.json b/tests/openvino/native/data/2026.1/references_scoped_configs/group_size_fallback_ignore.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/group_size_fallback_ignore.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/group_size_fallback_ignore.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/ignored_scope.json b/tests/openvino/native/data/2026.1/references_scoped_configs/ignored_scope.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/ignored_scope.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/ignored_scope.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/mixed_precision.json b/tests/openvino/native/data/2026.1/references_scoped_configs/mixed_precision.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/mixed_precision.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/mixed_precision.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/overlapping_annotations.json b/tests/openvino/native/data/2026.1/references_scoped_configs/overlapping_scoped_configs.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/overlapping_annotations.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/overlapping_scoped_configs.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/primary_and_backup_precision.json b/tests/openvino/native/data/2026.1/references_scoped_configs/primary_and_backup_precision.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/primary_and_backup_precision.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/primary_and_backup_precision.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight.json b/tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json b/tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_ignored_other_node_matched_first.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_ignored_other_node_matched_first.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json b/tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_ignored_other_node_matched_last.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_ignored_other_node_matched_last.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_matched_by_other_node.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_annotated_by_other_node.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_matched_by_other_node.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_other_node_annotated_first.json b/tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_other_node_matched_first.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_other_node_annotated_first.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_other_node_matched_first.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_other_node_annotated_last.json b/tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_other_node_matched_last.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_other_node_annotated_last.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_other_node_matched_last.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_compressed_node.json b/tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_with_ignored_compressed_node.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_compressed_node.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_with_ignored_compressed_node.json diff --git a/tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_scope.json b/tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_with_ignored_scope.json similarity index 100% rename from tests/openvino/native/data/2026.1/references_custom_annotation/shared_weight_with_ignored_scope.json rename to tests/openvino/native/data/2026.1/references_scoped_configs/shared_weight_with_ignored_scope.json diff --git a/tests/openvino/native/quantization/test_weights_compression.py b/tests/openvino/native/quantization/test_weights_compression.py index 4adbc0b227b..d5dfb591f0a 100644 --- a/tests/openvino/native/quantization/test_weights_compression.py +++ b/tests/openvino/native/quantization/test_weights_compression.py @@ -88,7 +88,7 @@ from tests.openvino.native.models import WeightsModel from tests.openvino.native.quantization.test_fq_params_calculation import REFERENCE_SCALES_DIR -CUSTOM_ANNOTATION_REFERENCES_DIR = Path("references_custom_annotation") +SCOPED_CONFIGS_REFERENCES_DIR = Path("references_scoped_configs") TEST_MODELS = { IntegerModel: ["matmul_2_data", "gather_2_data", "matmul_1_data"], @@ -1961,7 +1961,7 @@ def test_data_free_compression_with_backup_mode(backup_mode): assert act_num == num_compressed -@pytest.mark.parametrize("annotate", [False, True], ids=["default", "annotated"]) +@pytest.mark.parametrize("use_scoped_config", [False, True], ids=["default", "scoped_config"]) @pytest.mark.parametrize("backup_mode", [BackupMode.NONE, BackupMode.INT8_ASYM, BackupMode.INT8_SYM]) @pytest.mark.parametrize( ("params", "num_compressed"), @@ -1978,19 +1978,16 @@ def test_data_free_compression_with_backup_mode(backup_mode): ({"awq": True}, 6), ), ) -def test_data_based_compression_with_backup_mode(backup_mode, params, num_compressed, annotate): +def test_data_based_compression_with_backup_mode(backup_mode, params, num_compressed, use_scoped_config): model = AWQMatmulModel().ov_model sz = 8 n_samples = 10 dataset = Dataset([np.ones([1, i + 1, sz]) for i in range(n_samples)]) - custom_annotation = None - if annotate: - custom_annotation = [ - nncf.CustomAnnotation( - scope=nncf.Scope(names=["MatMul_1"]), - config=WeightCompressionConfig(mode=CompressWeightsMode.INT4_SYM, group_size=-1), - ) + scoped_configs = None + if use_scoped_config: + scoped_configs = [ + nncf.ScopedConfig(scope=nncf.Scope(names=["MatMul_1"]), mode=CompressWeightsMode.INT4_SYM, group_size=-1) ] num_compressed = 6 @@ -2001,11 +1998,11 @@ def test_data_based_compression_with_backup_mode(backup_mode, params, num_compre group_size=-1, dataset=dataset, backup_mode=backup_mode, - custom_annotation=custom_annotation, + scoped_configs=scoped_configs, **params, ) act_num = 0 - annotated_num = 0 + scoped_num = 0 if backup_mode == BackupMode.INT8_ASYM: backup_ov_mode = ov.Type.u8 elif backup_mode == BackupMode.INT8_SYM: @@ -2017,7 +2014,7 @@ def test_data_based_compression_with_backup_mode(backup_mode, params, num_compre if op.get_element_type() == ov.Type.u4: act_num += 1 elif op.get_element_type() == ov.Type.i4: - annotated_num += 1 + scoped_num += 1 elif "/scale" in op.get_friendly_name(): assert op.get_element_type() == ov.Type.f16 elif "_lora_" in op.get_friendly_name(): @@ -2025,7 +2022,7 @@ def test_data_based_compression_with_backup_mode(backup_mode, params, num_compre else: assert op.get_element_type() == backup_ov_mode assert act_num == num_compressed - assert annotated_num == (1 if annotate else 0) + assert scoped_num == (1 if use_scoped_config else 0) @pytest.mark.parametrize( @@ -2576,8 +2573,8 @@ def get_node_with_shared_weight_name() -> str: return "Gather_1" @staticmethod - def get_custom_annotation_ref_path(ref_name: str) -> Path: - return get_actual_reference_for_current_openvino(CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json") + def get_scoped_configs_ref_path(ref_name: str) -> Path: + return get_actual_reference_for_current_openvino(SCOPED_CONFIGS_REFERENCES_DIR / f"{ref_name}.json") @pytest.fixture def transpose_a_supported(self) -> bool: diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_adjust.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/group_size_fallback_adjust.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_adjust.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/group_size_fallback_adjust.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_ignore.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/group_size_fallback_ignore.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/group_size_fallback_ignore.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/group_size_fallback_ignore.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/ignored_scope.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/ignored_scope.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/ignored_scope.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/ignored_scope.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/mixed_precision.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/mixed_precision.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/mixed_precision.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/mixed_precision.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/overlapping_scoped_configs.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/overlapping_annotations.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/overlapping_scoped_configs.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/primary_and_backup_precision.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/primary_and_backup_precision.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/primary_and_backup_precision.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/primary_and_backup_precision.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_ignored_other_node_matched_first.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_ignored_other_node_matched_first.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_ignored_other_node_matched_last.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_ignored_other_node_matched_last.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_matched_by_other_node.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_annotated_by_other_node.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_matched_by_other_node.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_first.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_other_node_matched_first.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_first.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_other_node_matched_first.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_last.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_other_node_matched_last.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_other_node_annotated_last.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_other_node_matched_last.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_compressed_node.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_with_ignored_compressed_node.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_compressed_node.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_with_ignored_compressed_node.json diff --git a/tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_scope.json b/tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_with_ignored_scope.json similarity index 100% rename from tests/torch/data/function_hook/quantization/references_custom_annotation/shared_weight_with_ignored_scope.json rename to tests/torch/data/function_hook/quantization/references_scoped_configs/shared_weight_with_ignored_scope.json diff --git a/tests/torch/data/fx/references_custom_annotation/group_size_fallback_adjust.json b/tests/torch/data/fx/references_scoped_configs/group_size_fallback_adjust.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/group_size_fallback_adjust.json rename to tests/torch/data/fx/references_scoped_configs/group_size_fallback_adjust.json diff --git a/tests/torch/data/fx/references_custom_annotation/group_size_fallback_ignore.json b/tests/torch/data/fx/references_scoped_configs/group_size_fallback_ignore.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/group_size_fallback_ignore.json rename to tests/torch/data/fx/references_scoped_configs/group_size_fallback_ignore.json diff --git a/tests/torch/data/fx/references_custom_annotation/ignored_scope.json b/tests/torch/data/fx/references_scoped_configs/ignored_scope.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/ignored_scope.json rename to tests/torch/data/fx/references_scoped_configs/ignored_scope.json diff --git a/tests/torch/data/fx/references_custom_annotation/mixed_precision.json b/tests/torch/data/fx/references_scoped_configs/mixed_precision.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/mixed_precision.json rename to tests/torch/data/fx/references_scoped_configs/mixed_precision.json diff --git a/tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json b/tests/torch/data/fx/references_scoped_configs/overlapping_scoped_configs.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/overlapping_annotations.json rename to tests/torch/data/fx/references_scoped_configs/overlapping_scoped_configs.json diff --git a/tests/torch/data/fx/references_custom_annotation/primary_and_backup_precision.json b/tests/torch/data/fx/references_scoped_configs/primary_and_backup_precision.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/primary_and_backup_precision.json rename to tests/torch/data/fx/references_scoped_configs/primary_and_backup_precision.json diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight.json b/tests/torch/data/fx/references_scoped_configs/shared_weight.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/shared_weight.json rename to tests/torch/data/fx/references_scoped_configs/shared_weight.json diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json b/tests/torch/data/fx/references_scoped_configs/shared_weight_ignored_other_node_matched_first.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_first.json rename to tests/torch/data/fx/references_scoped_configs/shared_weight_ignored_other_node_matched_first.json diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json b/tests/torch/data/fx/references_scoped_configs/shared_weight_ignored_other_node_matched_last.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/shared_weight_ignored_other_node_annotated_last.json rename to tests/torch/data/fx/references_scoped_configs/shared_weight_ignored_other_node_matched_last.json diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json b/tests/torch/data/fx/references_scoped_configs/shared_weight_matched_by_other_node.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/shared_weight_annotated_by_other_node.json rename to tests/torch/data/fx/references_scoped_configs/shared_weight_matched_by_other_node.json diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_first.json b/tests/torch/data/fx/references_scoped_configs/shared_weight_other_node_matched_first.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_first.json rename to tests/torch/data/fx/references_scoped_configs/shared_weight_other_node_matched_first.json diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_last.json b/tests/torch/data/fx/references_scoped_configs/shared_weight_other_node_matched_last.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/shared_weight_other_node_annotated_last.json rename to tests/torch/data/fx/references_scoped_configs/shared_weight_other_node_matched_last.json diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_compressed_node.json b/tests/torch/data/fx/references_scoped_configs/shared_weight_with_ignored_compressed_node.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_compressed_node.json rename to tests/torch/data/fx/references_scoped_configs/shared_weight_with_ignored_compressed_node.json diff --git a/tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_scope.json b/tests/torch/data/fx/references_scoped_configs/shared_weight_with_ignored_scope.json similarity index 100% rename from tests/torch/data/fx/references_custom_annotation/shared_weight_with_ignored_scope.json rename to tests/torch/data/fx/references_scoped_configs/shared_weight_with_ignored_scope.json diff --git a/tests/torch/function_hook/quantization/test_weights_compression.py b/tests/torch/function_hook/quantization/test_weights_compression.py index b8d5f3f681a..b13a283772a 100644 --- a/tests/torch/function_hook/quantization/test_weights_compression.py +++ b/tests/torch/function_hook/quantization/test_weights_compression.py @@ -53,8 +53,8 @@ from tests.torch.test_tensor import cast_to REFERENCE_SCALES_DIR = TEST_ROOT / "torch" / "data" / "function_hook" / "quantization" / "reference_scales" -CUSTOM_ANNOTATION_REFERENCES_DIR = ( - TEST_ROOT / "torch" / "data" / "function_hook" / "quantization" / "references_custom_annotation" +SCOPED_CONFIGS_REFERENCES_DIR = ( + TEST_ROOT / "torch" / "data" / "function_hook" / "quantization" / "references_scoped_configs" ) ALL_SENSITIVITY_METRICS = list(SensitivityMetric) @@ -779,8 +779,8 @@ def get_mode_not_supported_by_backend() -> CompressWeightsMode | None: return CompressWeightsMode.NF4 @staticmethod - def get_custom_annotation_ref_path(ref_name: str) -> Path: - return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" + def get_scoped_configs_ref_path(ref_name: str) -> Path: + return SCOPED_CONFIGS_REFERENCES_DIR / f"{ref_name}.json" @staticmethod def get_transposable_awq_model(transpose_a: bool, transpose_b: bool, input_shape=None, is_3d_weights: bool = False): diff --git a/tests/torch/fx/test_weights_compression.py b/tests/torch/fx/test_weights_compression.py index 24ceb3da252..03fe7869c76 100644 --- a/tests/torch/fx/test_weights_compression.py +++ b/tests/torch/fx/test_weights_compression.py @@ -60,7 +60,7 @@ from tests.torch.test_tensor import cast_to REFERENCE_SCALES_DIR = TEST_ROOT / "torch" / "data" / "fx" / "reference_scales" -CUSTOM_ANNOTATION_REFERENCES_DIR = TEST_ROOT / "torch" / "data" / "fx" / "references_custom_annotation" +SCOPED_CONFIGS_REFERENCES_DIR = TEST_ROOT / "torch" / "data" / "fx" / "references_scoped_configs" DATA_BASED_SENSITIVITY_METRICS = ( SensitivityMetric.HESSIAN_INPUT_ACTIVATION, @@ -532,8 +532,8 @@ def get_mode_not_supported_by_backend() -> CompressWeightsMode | None: return CompressWeightsMode.NF4 @staticmethod - def get_custom_annotation_ref_path(ref_name: str) -> Path: - return CUSTOM_ANNOTATION_REFERENCES_DIR / f"{ref_name}.json" + def get_scoped_configs_ref_path(ref_name: str) -> Path: + return SCOPED_CONFIGS_REFERENCES_DIR / f"{ref_name}.json" @staticmethod def get_transposable_awq_model(transpose_a: bool, transpose_b: bool, input_shape=None, is_3d_weights: bool = False): From 087285c810890570e47448f874e0cc05bbde94f1 Mon Sep 17 00:00:00 2001 From: anzr299 Date: Mon, 12 Oct 2026 20:37:34 +0400 Subject: [PATCH 43/43] make group size default to -1 in wc config init --- .../weight_compression/algorithm.py | 6 +---- .../algorithms/weight_compression/config.py | 26 +------------------ 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/src/nncf/quantization/algorithms/weight_compression/algorithm.py b/src/nncf/quantization/algorithms/weight_compression/algorithm.py index dc0fd942c9b..c869c32b2de 100644 --- a/src/nncf/quantization/algorithms/weight_compression/algorithm.py +++ b/src/nncf/quantization/algorithms/weight_compression/algorithm.py @@ -45,7 +45,6 @@ from nncf.quantization.algorithms.weight_compression.awq import AWQ from nncf.quantization.algorithms.weight_compression.config import FIXED_GROUP_SIZE_MODES from nncf.quantization.algorithms.weight_compression.config import WeightCompressionParameters -from nncf.quantization.algorithms.weight_compression.config import get_default_group_size from nncf.quantization.algorithms.weight_compression.constants import CB4_QUANTILES from nncf.quantization.algorithms.weight_compression.gptq import GPTQ from nncf.quantization.algorithms.weight_compression.lora_correction import LoraCorrectionAlgorithm @@ -94,9 +93,6 @@ def get_weight_compression_configuration( """ Generates a configuration dictionary for weight compression based on the provided parameters. """ - if group_size is None: - group_size = get_default_group_size(mode) - if backup_mode is None: if mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3]: backup_mode = BackupMode.MXFP8_E4M3 @@ -192,7 +188,7 @@ def validate_scoped_configs( ) raise nncf.ValidationError(msg) - # Resolves the default group size of the mode and validates the given one + # Validates the given group size config = scoped_config.compression_config if config.is_codebook: msg = ( diff --git a/src/nncf/quantization/algorithms/weight_compression/config.py b/src/nncf/quantization/algorithms/weight_compression/config.py index b238285e052..32e5d01e2a5 100644 --- a/src/nncf/quantization/algorithms/weight_compression/config.py +++ b/src/nncf/quantization/algorithms/weight_compression/config.py @@ -27,28 +27,6 @@ CompressWeightsMode.MXFP8_E4M3: 32, CompressWeightsMode.NVFP4: 16, } -PER_CHANNEL_BY_DEFAULT_MODES = [ - CompressWeightsMode.INT8_ASYM, - CompressWeightsMode.INT8_SYM, - CompressWeightsMode.CODEBOOK, - CompressWeightsMode.CB4, - CompressWeightsMode.ADAPTIVE_CODEBOOK, -] -DEFAULT_GROUP_SIZE = 128 - - -def get_default_group_size(mode: CompressWeightsMode) -> int: - """ - Returns the group size to use for the given mode when no group size is requested explicitly. - - :param mode: Compression mode to get the default group size for. - :return: Default group size for the given mode. - """ - if mode in FIXED_GROUP_SIZE_MODES: - return FIXED_GROUP_SIZE_MODES[mode] - if mode in PER_CHANNEL_BY_DEFAULT_MODES: - return -1 - return DEFAULT_GROUP_SIZE @dataclass @@ -66,12 +44,10 @@ class WeightCompressionConfig: """ mode: CompressWeightsMode = CompressWeightsMode.INT8_ASYM - group_size: int | None = None + group_size: int = -1 codebook_values: Tensor | None = None def __post_init__(self) -> None: - if self.group_size is None: - self.group_size = get_default_group_size(self.mode) if self.group_size == 0 or self.group_size < -1: msg = f"Invalid group_size={self.group_size}. Group size must be a positive integer or -1." raise ValidationError(msg)