From da7cfea88fb47569caa2bbabbabf3bbba3b8d4ad Mon Sep 17 00:00:00 2001 From: youn17 Date: Fri, 24 Oct 2025 21:29:05 +0900 Subject: [PATCH 1/7] introduce new int8 quantization API --- docs/source/quantization_overview.rst | 4 +- .../workflows/int8/test_int8_tensor.py | 266 ++++++++++++++++ torchao/quantization/__init__.py | 2 + torchao/quantization/quant_api.py | 98 ++++-- .../common/quantize_tensor_kwargs.py | 8 + .../quantize_/workflows/__init__.py | 6 + .../quantize_/workflows/int8/int8_tensor.py | 300 ++++++++++++++++++ 7 files changed, 651 insertions(+), 33 deletions(-) create mode 100644 test/quantization/quantize_/workflows/int8/test_int8_tensor.py create mode 100644 torchao/quantization/quantize_/workflows/int8/int8_tensor.py diff --git a/docs/source/quantization_overview.rst b/docs/source/quantization_overview.rst index f5c82bfe5f..df0a924b11 100644 --- a/docs/source/quantization_overview.rst +++ b/docs/source/quantization_overview.rst @@ -5,7 +5,7 @@ First we want to lay out the torchao stack:: Quantization Algorithms/Flows: weight only/dynamic/static quantization, hqq, awq, gptq etc. --------------------------------------------------------------------------------------------- - Quantized Tensors (derived dtypes): Int4Tensor, Int4PreshuffledTensor, Float8Tensor + Quantized Tensors (derived dtypes): Int4Tensor, Int4PreshuffledTensor, Int8Tensor, Float8Tensor --------------------------------------------------------------------------------------------- Quantization Primitive Ops/Efficient Kernels: matmul, quantize, dequantize --------------------------------------------------------------------------------------------- @@ -88,6 +88,8 @@ So in general we structure Tensor subclasses by dervied dtpype and packing forma - scaled int4 - preshuffled (special format to optimize for loading) - float8 act + int4 weight dynamic quantization and int4 weight only quantization + * - Int8Tensor + - plain .. note:: We don't have granularity specific tensor subclasses, i.e. no Float8RowwiseTensor or Float8BlockwiseTensor, all granularities are implemented in the same Tensor, we typically use a general `block_size` attribute to distinguish between different granularities, and each Tensor is allowed to support only a subset of all possible granularity options. diff --git a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py new file mode 100644 index 0000000000..0fc959beb7 --- /dev/null +++ b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py @@ -0,0 +1,266 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD 3-Clause license found in the +# LICENSE file in the root directory of this source tree. + +import copy +import unittest + +import torch +from torch.testing._internal import common_utils + +from torchao.quantization import ( + Int8DynamicActivationInt8WeightConfig, + Int8WeightOnlyConfig, + PerRow, + PerTensor, + quantize_, +) +from torchao.quantization.quant_primitives import MappingType, choose_qparams_affine +from torchao.quantization.quantize_.workflows.int8.int8_tensor import ( + Int8Tensor, + QuantizeTensorToInt8Kwargs, +) +from torchao.quantization.utils import compute_error +from torchao.testing.utils import TorchAOIntegrationTestCase + + +# TODO: Refactor after https://github.com/pytorch/ao/pull/2729 is merged +class ToyTwoLinearModel(torch.nn.Module): + def __init__( + self, + input_dim, + hidden_dim, + output_dim, + has_bias=False, + dtype=None, + device=None, + ): + super().__init__() + self.dtype = dtype + self.device = device + self.linear1 = torch.nn.Linear( + input_dim, hidden_dim, bias=has_bias, dtype=dtype, device=device + ) + self.linear2 = torch.nn.Linear( + hidden_dim, output_dim, bias=has_bias, dtype=dtype, device=device + ) + + def forward(self, x): + x = self.linear1(x) + x = self.linear2(x) + return x + + +@unittest.skipIf(not torch.cuda.is_available(), "Need CUDA available") +@common_utils.instantiate_parametrized_tests +class TestInt8Tensor(TorchAOIntegrationTestCase): + def setUp(self): + super().setUp() + + self.test_shape = (4, 3) + self.dtype = torch.bfloat16 + self.batch_size = 32 + self.int8_min = -128 + self.int8_max = 127 + + torch.manual_seed(42) + self.weight_fp = torch.randn(*self.test_shape, dtype=self.dtype) + self.input_fp = torch.randn(*self.test_shape, dtype=self.dtype) + self.bias = torch.randn(self.test_shape[0], dtype=self.dtype) + self.block_size = list(self.test_shape) + + def test_creation_and_attributes(self): + """Test tensor creation, dtypes, and ranges""" + tensor = Int8Tensor.from_hp(self.weight_fp, self.block_size) + + self.assertEqual(tensor.shape, self.test_shape) + self.assertEqual(tensor.qdata.dtype, torch.int8) + self.assertTrue( + torch.all(tensor.qdata >= self.int8_min) + and torch.all(tensor.qdata <= self.int8_max) + ) + + @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) + @common_utils.parametrize( + "sizes", + [ + ((128,), 256, 128), + ], + ) + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + def test_int8_linear_quantization_accuracy( + self, + dtype: torch.dtype, + sizes: tuple, + config, + ): + """Test quantization preserves reasonable accuracy""" + M, N, K = sizes + input_tensor = torch.randn(*M, K, dtype=dtype, device="cuda") + + # Create a linear layer + m = ToyTwoLinearModel(K, N, K).eval().to(dtype).to("cuda") + m_q = copy.deepcopy(m) + + # Quantize + quantize_(m_q, config) + + output_original = m(input_tensor) + output_quantized = m_q(input_tensor) + + error = compute_error(output_original, output_quantized) + assert error > 20, ( + f"Quantization quality is too low, SQNR: {error}dB (expected > {20}dB)" + ) + + @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) + def test_quantization_shapes(self, dtype): + """Test static and dynamic quantization output shapes""" + K, N = 128, 64 + weight = torch.randn(N, K, dtype=dtype, device="cuda") + input_tensor = torch.randn(self.batch_size, K, dtype=dtype, device="cuda") + + # Dynamic quantization (runtime scale computation) + dynamic_tensor = Int8Tensor.from_hp(weight, block_size=[N, K]) + + # Static quantization (pre-computed scale) + act_scale, _ = choose_qparams_affine( + input=input_tensor, + mapping_type=MappingType.SYMMETRIC, + block_size=(input_tensor.shape[0], K), + target_dtype=torch.int8, + quant_min=self.int8_min, + quant_max=self.int8_max, + scale_dtype=dtype, + zero_point_dtype=torch.int8, + ) + + # Static quantization (with pre-computed scale) + static_tensor = Int8Tensor.from_hp( + weight, + block_size=[N, K], + act_quant_kwargs=QuantizeTensorToInt8Kwargs( + block_size=[input_tensor.shape[0], K], + static_scale=act_scale, + ), + ) + + dynamic_output = torch.nn.functional.linear(input_tensor, dynamic_tensor) + static_output = torch.nn.functional.linear(input_tensor, static_tensor) + + expected_shape = (self.batch_size, N) + self.assertEqual(dynamic_output.shape, expected_shape) + self.assertEqual(static_output.shape, expected_shape) + self.assertEqual(dynamic_output.dtype, dtype) + self.assertEqual(static_output.dtype, dtype) + + @unittest.skip("granularity parameter not supported in current API") + @common_utils.parametrize("granularity", [PerTensor(), PerRow()]) + def test_slice_preserves_aliasing(self, granularity): + slice_size = 512 + tensor_size = 1024 + + config = Int8DynamicActivationInt8WeightConfig( + granularity=granularity, version=2 + ) + l = torch.nn.Linear(tensor_size, tensor_size).to("cuda").to(torch.bfloat16) + l.weight = torch.nn.Parameter( + torch.zeros(tensor_size, tensor_size, dtype=torch.bfloat16, device="cuda") + ) + quantize_(l, config) + param = l.weight + param_data = param.data + param_data = param_data.narrow(0, 0, slice_size) + # Making sure the aliasing is preserved in sliced quantized Tensor + assert param.data.qdata.data_ptr() == param_data.qdata.data_ptr() + assert param.data.scale.data_ptr() == param_data.scale.data_ptr() + + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + @common_utils.parametrize("device", ["cpu", "cuda"]) + @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) + def test_slice(self, config, device, dtype): + """Test tensor slicing""" + tensor_size = 256 + slice_sizes = (64, 128) + + dummy = torch.nn.Linear( + tensor_size, tensor_size, bias=False, dtype=dtype, device=device + ) + quantize_(dummy, config) + + weight1 = dummy.weight.clone().narrow(0, 0, slice_sizes[0]) + weight2 = dummy.weight.clone().narrow(1, 0, slice_sizes[1]) + + self.assertEqual(weight1.qdata, dummy.weight.qdata.narrow(0, 0, slice_sizes[0])) + self.assertEqual(weight2.qdata, dummy.weight.qdata.narrow(1, 0, slice_sizes[1])) + + # Int8DynamicActivationInt8WeightConfig uses per-row (PerRow) + # Int8WeightOnlyConfig uses per-tensor (PerTensor) + if isinstance(config, Int8DynamicActivationInt8WeightConfig): + # PerRow: dim 0 slicing affects scale, dim 1 doesn't + self.assertEqual( + weight1.scale, dummy.weight.scale.narrow(0, 0, slice_sizes[0]) + ) + self.assertEqual(weight2.scale, dummy.weight.scale) + else: + # PerTensor: scale unchanged by slicing + self.assertEqual(weight1.scale, dummy.weight.scale) + self.assertEqual(weight2.scale, dummy.weight.scale) + with self.assertRaises(NotImplementedError): + _ = dummy.weight[::2] + + def test_index_select(self): + """test that `x_0 = x[0]` works when `x` is a 2D `Int8Tensor`.""" + N, K = 256, 512 + x = torch.randn(N, K, device="cuda", dtype=torch.bfloat16) + x_int8 = Int8Tensor.from_hp(x, block_size=[N, K]) + x_int8_0 = x_int8[0] + torch.testing.assert_close( + x_int8.dequantize()[0], x_int8_0.dequantize(), atol=0, rtol=0 + ) + + def test_invalid_input_handling(self): + """Test input validation with specific error types""" + invalid_tensor = torch.randn(5) + incompatible_block_size = [1] + + with self.assertRaises( + ValueError, msg="Should reject incompatible tensor dimensions" + ): + Int8Tensor.from_hp(invalid_tensor, incompatible_block_size) + + with self.assertRaises( + ValueError, msg="Should reject mismatched block size dimensions" + ): + Int8Tensor.from_hp(self.weight_fp, [1]) + + def test_dequantization_accuracy(self): + """Test dequantization accuracy separately""" + test_data = torch.tensor([[1.0, -1.0]], dtype=torch.bfloat16) + tensor = Int8Tensor.from_hp(test_data, [1, 2]) + + dequantized = tensor.dequantize() + self.assertEqual(dequantized.shape, test_data.shape) + self.assertLess( + torch.abs(dequantized - test_data).max().item(), + 0.1, + msg=f"Dequantization error exceeds tolerance of {0.1}", + ) + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/torchao/quantization/__init__.py b/torchao/quantization/__init__.py index 87e011c57b..05172063ba 100644 --- a/torchao/quantization/__init__.py +++ b/torchao/quantization/__init__.py @@ -97,6 +97,7 @@ Int4PreshuffledTensor, Int4Tensor, Int4TilePackedTo4dTensor, + Int8Tensor, IntxOpaqueTensor, IntxUnpackedToInt8Tensor, ) @@ -168,6 +169,7 @@ "IntxOpaqueTensor", "IntxUnpackedToInt8Tensor", "Int4TilePackedTo4dTensor", + "Int8Tensor", "Float8Tensor", "Int4OpaqueTensor", # smooth quant - subject to change diff --git a/torchao/quantization/quant_api.py b/torchao/quantization/quant_api.py index ae8210a41a..b042d723c1 100644 --- a/torchao/quantization/quant_api.py +++ b/torchao/quantization/quant_api.py @@ -81,6 +81,7 @@ Int4PreshuffledTensor, Int4Tensor, Int4TilePackedTo4dTensor, + Int8Tensor, IntxChooseQParamsAlgorithm, IntxOpaqueTensor, IntxPackingFormat, @@ -1353,6 +1354,7 @@ class Int8WeightOnlyConfig(AOBaseConfig): group_size: Optional[int] = None set_inductor_config: bool = True + version: int = 2 def __post_init__(self): torch._C._log_api_usage_once("torchao.quantization.Int8WeightOnlyConfig") @@ -1363,22 +1365,30 @@ def __post_init__(self): def _int8_weight_only_quantize_tensor(weight, config): - mapping_type = MappingType.SYMMETRIC - target_dtype = torch.int8 - eps = torch.finfo(torch.float32).eps - zero_point_dtype = torch.int64 - group_size = config.group_size - if group_size is None: - group_size = weight.shape[-1] - block_size = tuple([1 for x in range(weight.dim() - 1)] + [group_size]) - new_weight = to_affine_quantized_intx( - weight, - mapping_type, - block_size, - target_dtype, - eps=eps, - zero_point_dtype=zero_point_dtype, - ) + if config.version == 1: + warnings.warn( + "Config Deprecation: version 1 of Int8WeightOnlyConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2752 for more details" + ) + mapping_type = MappingType.SYMMETRIC + target_dtype = torch.int8 + eps = torch.finfo(torch.float32).eps + zero_point_dtype = torch.int64 + group_size = config.group_size + if group_size is None: + group_size = weight.shape[-1] + block_size = tuple([1 for x in range(weight.dim() - 1)] + [group_size]) + new_weight = to_affine_quantized_intx( + weight, + mapping_type, + block_size, + target_dtype, + eps=eps, + zero_point_dtype=zero_point_dtype, + ) + else: + assert config.version == 2, f"Unexpected version: {config.version}" + block_size = [weight.shape[0], weight.shape[1]] + new_weight = Int8Tensor.from_hp(weight, block_size=block_size) return new_weight @@ -1506,12 +1516,15 @@ class Int8DynamicActivationInt8WeightConfig(AOBaseConfig): in original precision during decode operations. set_inductor_config: bool = True - If True, adjusts `torchinductor` settings to recommended values for better performance with this quantization scheme. + version (int): the version of the config, version 1 is using AffineQuantizedTensor that we plan to deprecate/split, version 2 is using Int8Tensor """ layout: Optional[Layout] = PlainLayout() act_mapping_type: Optional[MappingType] = MappingType.SYMMETRIC weight_only_decode: bool = False + granularity: Optional[Union[PerRow, PerTensor]] = PerRow() set_inductor_config: bool = True + version: int = 2 def __post_init__(self): torch._C._log_api_usage_once( @@ -1543,9 +1556,6 @@ def _int8_dynamic_activation_int8_weight_quantize_tensor(weight, config): mapping_type = MappingType.SYMMETRIC weight_zero_point_domain = ZeroPointDomain.NONE - def get_weight_block_size(x): - return tuple([1 for _ in range(x.dim() - 1)] + [x.shape[-1]]) - target_dtype = torch.int8 eps = torch.finfo(torch.float32).eps zero_point_dtype = torch.int64 @@ -1559,19 +1569,43 @@ def get_weight_block_size(x): else: input_quant_func = _int8_asymm_per_token_quant - block_size = get_weight_block_size(weight) - new_weight = to_affine_quantized_intx( - weight, - mapping_type, - block_size, - target_dtype, - eps=eps, - zero_point_dtype=zero_point_dtype, - _layout=layout, - zero_point_domain=weight_zero_point_domain, - ) - new_weight = to_linear_activation_quantized(new_weight, input_quant_func) - return new_weight + if isinstance(config.granularity, PerTensor): + # Tensor granularity + block_size = weight.shape + else: + # Per row granularity + block_size = tuple([1 for _ in range(weight.dim() - 1)] + [weight.shape[-1]]) + + if config.version == 1: + warnings.warn( + "Config Deprecation: version 1 of Int8DynamicActivationInt8WeightConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2752 for more details" + ) + quantized_weight = to_affine_quantized_intx( + weight, + mapping_type, + block_size, + target_dtype, + eps=eps, + zero_point_dtype=zero_point_dtype, + _layout=layout, + zero_point_domain=weight_zero_point_domain, + ) + quantized_weight = to_linear_activation_quantized( + quantized_weight, input_quant_func + ) + else: + from torchao.quantization.quantize_.workflows.int8.int8_tensor import ( + QuantizeTensorToInt8Kwargs, + ) + + assert config.version == 2, f"Unexpected version: {config.version}" + quantized_weight = Int8Tensor.from_hp( + weight, + block_size, + act_quant_kwargs=QuantizeTensorToInt8Kwargs(block_size=block_size), + ) + + return quantized_weight @register_quantize_module_handler(Int8DynamicActivationInt8WeightConfig) diff --git a/torchao/quantization/quantize_/common/quantize_tensor_kwargs.py b/torchao/quantization/quantize_/common/quantize_tensor_kwargs.py index 0adc8c786d..44dd09ff62 100644 --- a/torchao/quantization/quantize_/common/quantize_tensor_kwargs.py +++ b/torchao/quantization/quantize_/common/quantize_tensor_kwargs.py @@ -39,7 +39,9 @@ def _choose_quant_func_and_quantize_tensor( """ from torchao.quantization.quantize_.workflows import ( Float8Tensor, + Int8Tensor, QuantizeTensorToFloat8Kwargs, + QuantizeTensorToInt8Kwargs, ) if isinstance(quant_kwargs, QuantizeTensorToFloat8Kwargs): @@ -52,5 +54,11 @@ def _choose_quant_func_and_quantize_tensor( quant_kwargs.hp_value_ub, quant_kwargs.kernel_preference, ) + elif isinstance(quant_kwargs, QuantizeTensorToInt8Kwargs): + return Int8Tensor.from_hp( + tensor, + quant_kwargs.block_size, + act_quant_kwargs=quant_kwargs, + ) raise NotImplementedError(f"Quant kwargs not supported: {quant_kwargs}") diff --git a/torchao/quantization/quantize_/workflows/__init__.py b/torchao/quantization/quantize_/workflows/__init__.py index 4307637f8e..3891f28dbe 100644 --- a/torchao/quantization/quantize_/workflows/__init__.py +++ b/torchao/quantization/quantize_/workflows/__init__.py @@ -20,6 +20,10 @@ Int4Tensor, ) from .int4.int4_tile_packed_to_4d_tensor import Int4TilePackedTo4dTensor +from .int8.int8_tensor import ( + Int8Tensor, + QuantizeTensorToInt8Kwargs, +) from .intx.intx_choose_qparams_algorithm import IntxChooseQParamsAlgorithm from .intx.intx_opaque_tensor import ( IntxOpaqueTensor, @@ -37,6 +41,8 @@ "Int4MarlinSparseTensor", "Int4PlainInt32Tensor", "Int4TilePackedTo4dTensor", + "Int8Tensor", + "QuantizeTensorToInt8Kwargs", "Float8Tensor", "QuantizeTensorToFloat8Kwargs", "Int4OpaqueTensor", diff --git a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py new file mode 100644 index 0000000000..eee493b3e3 --- /dev/null +++ b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py @@ -0,0 +1,300 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD 3-Clause license found in the +# LICENSE file in the root directory of this source tree. + +from dataclasses import dataclass +from typing import Optional + +import torch +from torch.utils._python_dispatch import return_and_correct_aliasing + +from torchao.float8.inference import _slice_scale_for_dimension +from torchao.quantization.quant_primitives import ( + MappingType, + _maybe_expand_scale_to_tensor_shape, + choose_qparams_affine, + quantize_affine, +) +from torchao.quantization.quantize_.common import ( + QuantizeTensorKwargs, + _choose_quant_func_and_quantize_tensor, +) +from torchao.utils import TorchAOBaseTensor, fill_defaults + +__all__ = ["Int8Tensor", "QuantizeTensorToInt8Kwargs"] + +aten = torch.ops.aten + + +@dataclass +class QuantizeTensorToInt8Kwargs(QuantizeTensorKwargs): + """Tensor kwargs for creating int8 tensor (either activation or weight) + + Args: + block_size (list[int]): block size for quantization granularity + static_scale (Optional[torch.Tensor]): pre-computed scale for static quantization + """ + + block_size: list[int] + static_scale: Optional[torch.Tensor] = None + + +class Int8Tensor(TorchAOBaseTensor): + """ + int8 quantized tensor with plain layout + + Tensor Attributes: + qdata: (N, K) int8 quantized weight data + scale: scale factors for dequantization + + Non-Tensor Attributes: + block_size: block size for quantization granularity + act_quant_kwargs: flags for static/dynamic activation quantization + """ + + tensor_data_names = ["qdata", "scale"] + tensor_attribute_names = ["block_size"] + optional_tensor_attribute_names = [ + "act_quant_kwargs", + "dtype", + ] + + def __new__( + cls: type, + qdata: torch.Tensor, + scale: torch.Tensor, + block_size: list[int], + act_quant_kwargs=None, + dtype=None, + ): + kwargs = { + "device": qdata.device, + "dtype": dtype or scale.dtype, + "requires_grad": False, + } + return torch.Tensor._make_wrapper_subclass(cls, qdata.shape, **kwargs) + + def __init__( + self, + qdata: torch.Tensor, + scale: torch.Tensor, + block_size: list[int], + act_quant_kwargs=None, + dtype=None, + ): + super().__init__() + self.qdata = qdata + self.scale = scale + self.block_size = block_size + self.act_quant_kwargs = act_quant_kwargs + + def __repr__(self): + return ( + f"{self.__class__.__name__}({self.act_quant_kwargs=}, {self.qdata=}, {self.scale=}, " + f"{self.block_size=}, {self.shape=}, {self.device=}, {self.dtype=})" + ) + + @classmethod + def from_hp( + cls, + w: torch.Tensor, + block_size: list[int], + act_quant_kwargs: Optional[QuantizeTensorToInt8Kwargs] = None, + ): + if w.dim() != 2 or len(block_size) != 2: + raise ValueError("Expected 2D tensor and block_size length 2") + + if act_quant_kwargs is not None and act_quant_kwargs.static_scale is not None: + # INT8 × INT8 (static) + scale = act_quant_kwargs.static_scale + zero_point = torch.zeros_like(scale, dtype=torch.int8) + else: + # INT8 × INT8 (dynamic): compute scale at runtime + scale, zero_point = choose_qparams_affine( + input=w, + mapping_type=MappingType.SYMMETRIC, + block_size=block_size, + target_dtype=torch.int8, + quant_min=-128, + quant_max=127, + scale_dtype=w.dtype, + zero_point_dtype=torch.int8, + ) + + int_data = quantize_affine( + w, + block_size=block_size, + scale=scale, + zero_point=zero_point, + output_dtype=torch.int8, + ) + + if tuple(block_size) == w.shape: + # per-tensor + pass + elif len(scale.shape) == 1: + # per-row, 1D -> 2D + scale = scale.unsqueeze(-1) + + return cls( + int_data, + scale, + block_size, + act_quant_kwargs=act_quant_kwargs, + dtype=w.dtype, + ) + + def dequantize(self, output_dtype: Optional[torch.dtype] = None) -> torch.Tensor: + """Dequantize int8 tensor to floating point""" + + if output_dtype is None: + output_dtype = self.dtype + + qdata_fp = self.qdata.to(output_dtype) + # Reshape scale to broadcast if granularity is block-wise + scale_expanded = _maybe_expand_scale_to_tensor_shape( + self.scale, self.qdata.shape + ) + return qdata_fp * scale_expanded.to(output_dtype) + + +implements = Int8Tensor.implements +implements_torch_function = Int8Tensor.implements_torch_function + + +@implements(aten.linear.default) +@implements_torch_function(torch.nn.functional.linear) +def _(func, types, args, kwargs): + """quantization: dynamic, static, weight-only int8 quantization""" + activation_tensor, weight_tensor, bias = ( + args[0], + args[1], + args[2] if len(args) > 2 else None, + ) + + assert isinstance(weight_tensor, Int8Tensor), ( + f"Expected weight to be Int8Tensor, got {type(weight_tensor)}" + ) + + if weight_tensor.act_quant_kwargs is not None: + if not isinstance(activation_tensor, Int8Tensor): + # Activation quantization + activation_tensor = _choose_quant_func_and_quantize_tensor( + activation_tensor, weight_tensor.act_quant_kwargs + ) + + x_vals = activation_tensor.qdata + x_scales = activation_tensor.scale + w_vals_t = weight_tensor.qdata.contiguous().t() + w_scales = weight_tensor.scale + + tmp_shape = (-1, x_vals.shape[-1]) + tmp = x_vals.view(tmp_shape) + + # Cast fp16 scale to float + intermediate_dtype = ( + torch.float if x_scales.dtype == torch.half else x_scales.dtype + ) + # Note: CUDA doesn't support int32/int64 matmul, so we convert to float + # Error message is NotImplementedError: "addmm_cuda" not implemented for 'Int' + # This may introduce minor numerical differences compared to int arithmetic + y_dot = torch.mm(tmp.to(intermediate_dtype), w_vals_t.to(intermediate_dtype)) + + # Apply activation scale + is_per_tensor_act = x_scales.numel() == 1 + if is_per_tensor_act: + y_dot.mul_(x_scales.to(intermediate_dtype)) + else: + # For block-wise activation scale, reshape to match y_dot + x_scales_reshaped = x_scales.view(y_dot.shape[0], -1) + y_dot.mul_(x_scales_reshaped.to(intermediate_dtype)) + + # Apply weight scale + is_per_tensor_weight = w_scales.numel() == 1 + if is_per_tensor_weight: + result = y_dot.mul_(w_scales.to(intermediate_dtype)) + else: + # Per-row weight scale - transpose and broadcast + w_scales_broadcast = w_scales.t().expand_as(y_dot) + result = y_dot.mul_(w_scales_broadcast.to(intermediate_dtype)) + + # Reshape back to original shape + result = result.view(*x_vals.shape[:-1], result.shape[-1]) + result = result.to(activation_tensor.dtype) + else: + # FP × INT8 (weight-only) + result = func( + activation_tensor, weight_tensor.dequantize(activation_tensor.dtype), None + ) + + return result + bias if bias is not None else result + + +@implements(aten.slice.Tensor) +def _(func, types, args, kwargs): + """Slice operation for Int8Tensor""" + self, dim, start, end, step = fill_defaults(args, 5, [0, None, None, 1]) + + if step != 1: + raise NotImplementedError("Slicing with step > 1 is not supported") + + if end >= self.shape[dim]: + end = self.shape[dim] + + sliced_qdata = aten.slice.Tensor(self.qdata, dim, start, end, step) + + if self.scale.numel() == 1: + # Per-tensor quantization - scale doesn't change + sliced_scale = self.scale + elif dim < self.scale.ndim and self.scale.shape[dim] > 1: + # Block-wise quantization - need to slice the scale appropriately + sliced_scale = aten.slice.Tensor(self.scale, dim, start, end, step) + else: + # Block-wise quantization - need to slice the scale appropriately + sliced_scale = _slice_scale_for_dimension( + self.scale, self.qdata.shape, dim, start, end, step + ) + + block_size = list(self.block_size) + for i in range(len(block_size)): + block_size[i] = min(block_size[i], sliced_qdata.shape[i]) + + return return_and_correct_aliasing( + func, + args, + kwargs, + Int8Tensor( + sliced_qdata, + sliced_scale, + block_size, + self.act_quant_kwargs, + dtype=self.dtype, + ), + ) + + +@implements(aten.select.int) +def _(func, types, args, kwargs): + self, dim, index = args + assert dim == 0, f"Only dim=0 supported, got {dim}" + + selected_scale = self.scale if self.scale.ndim == 0 else self.scale[index] + + return return_and_correct_aliasing( + func, + args, + kwargs, + Int8Tensor( + self.qdata[index], + selected_scale, + self.block_size, + self.act_quant_kwargs, + self.dtype, + ), + ) + + +Int8Tensor.__module__ = "torchao.quantization" +torch.serialization.add_safe_globals([Int8Tensor, QuantizeTensorToInt8Kwargs]) From 27076d35a22fb1c42151588471842d17d5b8c112 Mon Sep 17 00:00:00 2001 From: younn17 Date: Sun, 26 Oct 2025 14:59:38 +0900 Subject: [PATCH 2/7] refactor ops and update test cases --- .../workflows/int8/test_int8_tensor.py | 95 +++++++------------ .../quantize_/workflows/int8/int8_tensor.py | 94 ++++++++---------- 2 files changed, 70 insertions(+), 119 deletions(-) diff --git a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py index 0fc959beb7..6e225ee108 100644 --- a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py +++ b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py @@ -13,14 +13,10 @@ from torchao.quantization import ( Int8DynamicActivationInt8WeightConfig, Int8WeightOnlyConfig, - PerRow, - PerTensor, quantize_, ) -from torchao.quantization.quant_primitives import MappingType, choose_qparams_affine from torchao.quantization.quantize_.workflows.int8.int8_tensor import ( Int8Tensor, - QuantizeTensorToInt8Kwargs, ) from torchao.quantization.utils import compute_error from torchao.testing.utils import TorchAOIntegrationTestCase @@ -62,8 +58,6 @@ def setUp(self): self.test_shape = (4, 3) self.dtype = torch.bfloat16 self.batch_size = 32 - self.int8_min = -128 - self.int8_max = 127 torch.manual_seed(42) self.weight_fp = torch.randn(*self.test_shape, dtype=self.dtype) @@ -78,8 +72,7 @@ def test_creation_and_attributes(self): self.assertEqual(tensor.shape, self.test_shape) self.assertEqual(tensor.qdata.dtype, torch.int8) self.assertTrue( - torch.all(tensor.qdata >= self.int8_min) - and torch.all(tensor.qdata <= self.int8_max) + torch.all(tensor.qdata >= -128) and torch.all(tensor.qdata <= 127) ) @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) @@ -122,66 +115,42 @@ def test_int8_linear_quantization_accuracy( ) @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) - def test_quantization_shapes(self, dtype): - """Test static and dynamic quantization output shapes""" - K, N = 128, 64 - weight = torch.randn(N, K, dtype=dtype, device="cuda") - input_tensor = torch.randn(self.batch_size, K, dtype=dtype, device="cuda") - - # Dynamic quantization (runtime scale computation) - dynamic_tensor = Int8Tensor.from_hp(weight, block_size=[N, K]) - - # Static quantization (pre-computed scale) - act_scale, _ = choose_qparams_affine( - input=input_tensor, - mapping_type=MappingType.SYMMETRIC, - block_size=(input_tensor.shape[0], K), - target_dtype=torch.int8, - quant_min=self.int8_min, - quant_max=self.int8_max, - scale_dtype=dtype, - zero_point_dtype=torch.int8, - ) + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + def test_per_row_scale_shape(self, dtype, config): + """Test per-row quantization maintains 1D scale""" + N, K = 64, 128 + linear = torch.nn.Linear(K, N, bias=False, dtype=dtype, device="cuda") + quantize_(linear, config) - # Static quantization (with pre-computed scale) - static_tensor = Int8Tensor.from_hp( - weight, - block_size=[N, K], - act_quant_kwargs=QuantizeTensorToInt8Kwargs( - block_size=[input_tensor.shape[0], K], - static_scale=act_scale, - ), - ) + # Dynamic: per-row (1D scale [N]), Weight-only: per-tensor (scalar) + if isinstance(config, Int8DynamicActivationInt8WeightConfig): + self.assertEqual(linear.weight.scale.shape, (N,)) + self.assertEqual(linear.weight.scale.ndim, 1) + else: + self.assertEqual(linear.weight.scale.numel(), 1) - dynamic_output = torch.nn.functional.linear(input_tensor, dynamic_tensor) - static_output = torch.nn.functional.linear(input_tensor, static_tensor) + @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) + @common_utils.parametrize("has_bias", [True, False]) + def test_weight_only_linear_with_bias(self, dtype, has_bias): + """Test weight-only quantization with and without bias""" + K, N = 128, 64 + linear = torch.nn.Linear(K, N, bias=has_bias, dtype=dtype, device="cuda") + input_tensor = torch.randn(self.batch_size, K, dtype=dtype, device="cuda") - expected_shape = (self.batch_size, N) - self.assertEqual(dynamic_output.shape, expected_shape) - self.assertEqual(static_output.shape, expected_shape) - self.assertEqual(dynamic_output.dtype, dtype) - self.assertEqual(static_output.dtype, dtype) + output_fp = linear(input_tensor) - @unittest.skip("granularity parameter not supported in current API") - @common_utils.parametrize("granularity", [PerTensor(), PerRow()]) - def test_slice_preserves_aliasing(self, granularity): - slice_size = 512 - tensor_size = 1024 + quantize_(linear, Int8WeightOnlyConfig(version=2)) + output_q = linear(input_tensor) - config = Int8DynamicActivationInt8WeightConfig( - granularity=granularity, version=2 - ) - l = torch.nn.Linear(tensor_size, tensor_size).to("cuda").to(torch.bfloat16) - l.weight = torch.nn.Parameter( - torch.zeros(tensor_size, tensor_size, dtype=torch.bfloat16, device="cuda") - ) - quantize_(l, config) - param = l.weight - param_data = param.data - param_data = param_data.narrow(0, 0, slice_size) - # Making sure the aliasing is preserved in sliced quantized Tensor - assert param.data.qdata.data_ptr() == param_data.qdata.data_ptr() - assert param.data.scale.data_ptr() == param_data.scale.data_ptr() + self.assertEqual(output_q.shape, output_fp.shape) + error = compute_error(output_fp, output_q) + self.assertGreater(error, 20) @common_utils.parametrize( "config", diff --git a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py index eee493b3e3..ded7020263 100644 --- a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py +++ b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py @@ -11,6 +11,7 @@ from torch.utils._python_dispatch import return_and_correct_aliasing from torchao.float8.inference import _slice_scale_for_dimension +from torchao.kernel import int_scaled_matmul from torchao.quantization.quant_primitives import ( MappingType, _maybe_expand_scale_to_tensor_shape, @@ -34,11 +35,10 @@ class QuantizeTensorToInt8Kwargs(QuantizeTensorKwargs): Args: block_size (list[int]): block size for quantization granularity - static_scale (Optional[torch.Tensor]): pre-computed scale for static quantization + # TODO: Static quantization support using `static_scale`, `static_zero_point` """ block_size: list[int] - static_scale: Optional[torch.Tensor] = None class Int8Tensor(TorchAOBaseTensor): @@ -51,7 +51,7 @@ class Int8Tensor(TorchAOBaseTensor): Non-Tensor Attributes: block_size: block size for quantization granularity - act_quant_kwargs: flags for static/dynamic activation quantization + act_quant_kwargs: flags for dynamic activation quantization """ tensor_data_names = ["qdata", "scale"] @@ -106,22 +106,16 @@ def from_hp( if w.dim() != 2 or len(block_size) != 2: raise ValueError("Expected 2D tensor and block_size length 2") - if act_quant_kwargs is not None and act_quant_kwargs.static_scale is not None: - # INT8 × INT8 (static) - scale = act_quant_kwargs.static_scale - zero_point = torch.zeros_like(scale, dtype=torch.int8) - else: - # INT8 × INT8 (dynamic): compute scale at runtime - scale, zero_point = choose_qparams_affine( - input=w, - mapping_type=MappingType.SYMMETRIC, - block_size=block_size, - target_dtype=torch.int8, - quant_min=-128, - quant_max=127, - scale_dtype=w.dtype, - zero_point_dtype=torch.int8, - ) + scale, zero_point = choose_qparams_affine( + input=w, + mapping_type=MappingType.SYMMETRIC, + block_size=block_size, + target_dtype=torch.int8, + quant_min=-128, + quant_max=127, + scale_dtype=w.dtype, + zero_point_dtype=torch.int8, + ) int_data = quantize_affine( w, @@ -131,12 +125,9 @@ def from_hp( output_dtype=torch.int8, ) - if tuple(block_size) == w.shape: - # per-tensor + if tuple(block_size) != w.shape and len(scale.shape) == 1: + # per-row pass - elif len(scale.shape) == 1: - # per-row, 1D -> 2D - scale = scale.unsqueeze(-1) return cls( int_data, @@ -167,7 +158,7 @@ def dequantize(self, output_dtype: Optional[torch.dtype] = None) -> torch.Tensor @implements(aten.linear.default) @implements_torch_function(torch.nn.functional.linear) def _(func, types, args, kwargs): - """quantization: dynamic, static, weight-only int8 quantization""" + """quantization: dynamic, weight-only int8 quantization""" activation_tensor, weight_tensor, bias = ( args[0], args[1], @@ -190,44 +181,29 @@ def _(func, types, args, kwargs): w_vals_t = weight_tensor.qdata.contiguous().t() w_scales = weight_tensor.scale - tmp_shape = (-1, x_vals.shape[-1]) - tmp = x_vals.view(tmp_shape) - - # Cast fp16 scale to float + tmp = x_vals.reshape(-1, x_vals.shape[-1]) intermediate_dtype = ( torch.float if x_scales.dtype == torch.half else x_scales.dtype ) - # Note: CUDA doesn't support int32/int64 matmul, so we convert to float - # Error message is NotImplementedError: "addmm_cuda" not implemented for 'Int' - # This may introduce minor numerical differences compared to int arithmetic - y_dot = torch.mm(tmp.to(intermediate_dtype), w_vals_t.to(intermediate_dtype)) - - # Apply activation scale - is_per_tensor_act = x_scales.numel() == 1 - if is_per_tensor_act: - y_dot.mul_(x_scales.to(intermediate_dtype)) - else: - # For block-wise activation scale, reshape to match y_dot - x_scales_reshaped = x_scales.view(y_dot.shape[0], -1) - y_dot.mul_(x_scales_reshaped.to(intermediate_dtype)) - - # Apply weight scale - is_per_tensor_weight = w_scales.numel() == 1 - if is_per_tensor_weight: - result = y_dot.mul_(w_scales.to(intermediate_dtype)) - else: - # Per-row weight scale - transpose and broadcast - w_scales_broadcast = w_scales.t().expand_as(y_dot) - result = y_dot.mul_(w_scales_broadcast.to(intermediate_dtype)) - # Reshape back to original shape - result = result.view(*x_vals.shape[:-1], result.shape[-1]) + y_dot_scaled = int_scaled_matmul( + tmp, w_vals_t, x_scales.reshape(-1, 1).to(intermediate_dtype) + ) + y_dot_scaled = y_dot_scaled.to(x_scales.dtype) + + result = (y_dot_scaled * w_scales).reshape( + *x_vals.shape[:-1], y_dot_scaled.shape[-1] + ) result = result.to(activation_tensor.dtype) else: # FP × INT8 (weight-only) - result = func( - activation_tensor, weight_tensor.dequantize(activation_tensor.dtype), None + w_vals_int8_t = weight_tensor.qdata.t() + m = torch.mm( + activation_tensor.reshape(-1, activation_tensor.shape[-1]), + w_vals_int8_t.to(activation_tensor.dtype), ) + result = m * weight_tensor.scale.to(m.dtype) + result = result.reshape(*activation_tensor.shape[:-1], result.shape[-1]) return result + bias if bias is not None else result @@ -248,11 +224,17 @@ def _(func, types, args, kwargs): if self.scale.numel() == 1: # Per-tensor quantization - scale doesn't change sliced_scale = self.scale + elif self.scale.ndim == 1: + # Per-row: 1D scale - only slice if dim=0 + if dim == 0: + sliced_scale = aten.slice.Tensor(self.scale, 0, start, end, step) + else: + sliced_scale = self.scale elif dim < self.scale.ndim and self.scale.shape[dim] > 1: # Block-wise quantization - need to slice the scale appropriately sliced_scale = aten.slice.Tensor(self.scale, dim, start, end, step) else: - # Block-wise quantization - need to slice the scale appropriately + # Block-wise quantization with different dimensions sliced_scale = _slice_scale_for_dimension( self.scale, self.qdata.shape, dim, start, end, step ) From cdb1d9f75a713a322c2b9acd7664844cd10ab439 Mon Sep 17 00:00:00 2001 From: younn17 Date: Sun, 26 Oct 2025 16:22:20 +0900 Subject: [PATCH 3/7] update granularity slicing support --- torchao/float8/inference.py | 25 ++++++++++++----- .../quantize_/workflows/int8/int8_tensor.py | 27 +++---------------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/torchao/float8/inference.py b/torchao/float8/inference.py index f15d38576c..da58b54269 100644 --- a/torchao/float8/inference.py +++ b/torchao/float8/inference.py @@ -140,7 +140,18 @@ def _slice_scale_for_dimension( """ aten = torch.ops.aten - # Unsupported case for now, this would be 1 scale per data element + # Per-tensor quantization (scalar scale) + if scale.numel() == 1: + return scale + + # Per-row quantization (1D scale) + if scale.ndim == 1: + if dim == 0: + return aten.slice.Tensor(scale, 0, start, end, step) + else: + return scale + + # Block-wise quantization (2D scale) if scale.shape == data_shape: return aten.slice.Tensor(scale, dim, start, end, step) @@ -158,6 +169,12 @@ def _slice_scale_for_dimension( # Slice away as normal return aten.slice.Tensor(scale, dim, start, end, step) else: + # Error on Step > 1 + if step > 1: + raise NotImplementedError( + "Slicing with step > 1 is not implemented for scale tensors." + ) + # There is blocking in this dimension # Calculate which scale elements correspond to the sliced data scale_start = start // block_size_for_dim if start is not None else None @@ -167,12 +184,6 @@ def _slice_scale_for_dimension( else None ) - # Error on Step > 1 - if step > 1: - raise NotImplementedError( - "Slicing with step > 1 is not implemented for scale tensors." - ) - return aten.slice.Tensor(scale, dim, scale_start, scale_end, 1) diff --git a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py index ded7020263..622d69ae09 100644 --- a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py +++ b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py @@ -125,10 +125,6 @@ def from_hp( output_dtype=torch.int8, ) - if tuple(block_size) != w.shape and len(scale.shape) == 1: - # per-row - pass - return cls( int_data, scale, @@ -158,7 +154,7 @@ def dequantize(self, output_dtype: Optional[torch.dtype] = None) -> torch.Tensor @implements(aten.linear.default) @implements_torch_function(torch.nn.functional.linear) def _(func, types, args, kwargs): - """quantization: dynamic, weight-only int8 quantization""" + """INT8 quantization: dynamic activation or weight-only""" activation_tensor, weight_tensor, bias = ( args[0], args[1], @@ -220,24 +216,9 @@ def _(func, types, args, kwargs): end = self.shape[dim] sliced_qdata = aten.slice.Tensor(self.qdata, dim, start, end, step) - - if self.scale.numel() == 1: - # Per-tensor quantization - scale doesn't change - sliced_scale = self.scale - elif self.scale.ndim == 1: - # Per-row: 1D scale - only slice if dim=0 - if dim == 0: - sliced_scale = aten.slice.Tensor(self.scale, 0, start, end, step) - else: - sliced_scale = self.scale - elif dim < self.scale.ndim and self.scale.shape[dim] > 1: - # Block-wise quantization - need to slice the scale appropriately - sliced_scale = aten.slice.Tensor(self.scale, dim, start, end, step) - else: - # Block-wise quantization with different dimensions - sliced_scale = _slice_scale_for_dimension( - self.scale, self.qdata.shape, dim, start, end, step - ) + sliced_scale = _slice_scale_for_dimension( + self.scale, self.qdata.shape, dim, start, end, step + ) block_size = list(self.block_size) for i in range(len(block_size)): From 9f1b6c9c98820893e78ea22dc5f01310313d4ece Mon Sep 17 00:00:00 2001 From: youn17 Date: Fri, 31 Oct 2025 18:29:49 +0900 Subject: [PATCH 4/7] add 3D support to api, build linear variants test --- .../workflows/int8/test_int8_tensor.py | 69 ++++++++++++------ torchao/quantization/quant_api.py | 4 +- .../quantize_/workflows/int8/int8_tensor.py | 73 ++++++++++++++----- 3 files changed, 102 insertions(+), 44 deletions(-) diff --git a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py index 6e225ee108..fb091a0e22 100644 --- a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py +++ b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py @@ -75,13 +75,48 @@ def test_creation_and_attributes(self): torch.all(tensor.qdata >= -128) and torch.all(tensor.qdata <= 127) ) - @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) + @common_utils.parametrize("dtype", [torch.bfloat16, torch.float32]) + @common_utils.parametrize("compile", [True, False]) + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) @common_utils.parametrize( "sizes", [ - ((128,), 256, 128), + ((128,), 256, 128), # 2D + ((32, 128), 64, 256), # 3D ], ) + def test_int8_linear_variants( + self, + dtype: torch.dtype, + config, + compile: bool, + sizes: tuple, + ): + """Test linear operation supports including shape and compile""" + M, N, K = sizes + input_tensor = torch.randn(*M, K, dtype=dtype, device="cuda") + model = ToyTwoLinearModel(K, N, K, dtype=dtype, device="cuda").eval() + model_q = copy.deepcopy(model) + + quantize_(model_q, config) + + if compile: + model_q = torch.compile(model_q, fullgraph=True) + + output_fp = model(input_tensor) + output_quantized = model_q(input_tensor) + + assert compute_error(output_fp, output_quantized) > 20, ( + f"Quantization error is too high got a SQNR of {compute_error(output_fp, output_quantized)}" + ) + + @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) @common_utils.parametrize( "config", [ @@ -89,6 +124,13 @@ def test_creation_and_attributes(self): Int8WeightOnlyConfig(version=2), ], ) + @common_utils.parametrize( + "sizes", + [ + ((128,), 256, 128), # 2D + ((32, 128), 64, 256), # 3D + ], + ) def test_int8_linear_quantization_accuracy( self, dtype: torch.dtype, @@ -100,16 +142,16 @@ def test_int8_linear_quantization_accuracy( input_tensor = torch.randn(*M, K, dtype=dtype, device="cuda") # Create a linear layer - m = ToyTwoLinearModel(K, N, K).eval().to(dtype).to("cuda") + m = ToyTwoLinearModel(K, N, K, dtype=dtype, device="cuda").eval() m_q = copy.deepcopy(m) # Quantize quantize_(m_q, config) - output_original = m(input_tensor) + output_fp = m(input_tensor) output_quantized = m_q(input_tensor) - error = compute_error(output_original, output_quantized) + error = compute_error(output_fp, output_quantized) assert error > 20, ( f"Quantization quality is too low, SQNR: {error}dB (expected > {20}dB)" ) @@ -135,23 +177,6 @@ def test_per_row_scale_shape(self, dtype, config): else: self.assertEqual(linear.weight.scale.numel(), 1) - @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) - @common_utils.parametrize("has_bias", [True, False]) - def test_weight_only_linear_with_bias(self, dtype, has_bias): - """Test weight-only quantization with and without bias""" - K, N = 128, 64 - linear = torch.nn.Linear(K, N, bias=has_bias, dtype=dtype, device="cuda") - input_tensor = torch.randn(self.batch_size, K, dtype=dtype, device="cuda") - - output_fp = linear(input_tensor) - - quantize_(linear, Int8WeightOnlyConfig(version=2)) - output_q = linear(input_tensor) - - self.assertEqual(output_q.shape, output_fp.shape) - error = compute_error(output_fp, output_q) - self.assertGreater(error, 20) - @common_utils.parametrize( "config", [ diff --git a/torchao/quantization/quant_api.py b/torchao/quantization/quant_api.py index b042d723c1..6da83a40a0 100644 --- a/torchao/quantization/quant_api.py +++ b/torchao/quantization/quant_api.py @@ -1354,7 +1354,7 @@ class Int8WeightOnlyConfig(AOBaseConfig): group_size: Optional[int] = None set_inductor_config: bool = True - version: int = 2 + version: int = 1 def __post_init__(self): torch._C._log_api_usage_once("torchao.quantization.Int8WeightOnlyConfig") @@ -1524,7 +1524,7 @@ class Int8DynamicActivationInt8WeightConfig(AOBaseConfig): weight_only_decode: bool = False granularity: Optional[Union[PerRow, PerTensor]] = PerRow() set_inductor_config: bool = True - version: int = 2 + version: int = 1 def __post_init__(self): torch._C._log_api_usage_once( diff --git a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py index 622d69ae09..a6e747e7e1 100644 --- a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py +++ b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py @@ -10,7 +10,10 @@ import torch from torch.utils._python_dispatch import return_and_correct_aliasing -from torchao.float8.inference import _slice_scale_for_dimension +from torchao.float8.inference import ( + _slice_scale_for_dimension, + preprocess_scale, +) from torchao.kernel import int_scaled_matmul from torchao.quantization.quant_primitives import ( MappingType, @@ -46,7 +49,7 @@ class Int8Tensor(TorchAOBaseTensor): int8 quantized tensor with plain layout Tensor Attributes: - qdata: (N, K) int8 quantized weight data + qdata: (N, K) or (B, N, K) int8 quantized weight data (2D or 3D) scale: scale factors for dequantization Non-Tensor Attributes: @@ -103,8 +106,8 @@ def from_hp( block_size: list[int], act_quant_kwargs: Optional[QuantizeTensorToInt8Kwargs] = None, ): - if w.dim() != 2 or len(block_size) != 2: - raise ValueError("Expected 2D tensor and block_size length 2") + if w.dim() not in [2, 3] or len(block_size) != w.dim(): + raise ValueError("Expected 2D or 3D tensor with same block_size length") scale, zero_point = choose_qparams_affine( input=w, @@ -165,41 +168,66 @@ def _(func, types, args, kwargs): f"Expected weight to be Int8Tensor, got {type(weight_tensor)}" ) + # Store original shape for reshaping result + original_weight_shape = weight_tensor.qdata.shape + + # Reshape 3D weights to 2D: (B, N, K) -> (B*N, K) + if weight_tensor.qdata.dim() == 3: + w_q_2d = weight_tensor.qdata.reshape(-1, original_weight_shape[-1]) + w_scale_2d = ( + weight_tensor.scale.reshape(-1) + if weight_tensor.scale.numel() > 1 + else weight_tensor.scale + ) + else: + w_q_2d = weight_tensor.qdata + w_scale_2d = weight_tensor.scale + if weight_tensor.act_quant_kwargs is not None: if not isinstance(activation_tensor, Int8Tensor): - # Activation quantization + # Dynamic activation quantization + act_kwargs = weight_tensor.act_quant_kwargs + input_ndim = activation_tensor.ndim + + # Ensure block_size matches input tensor dimensions + if len(act_kwargs.block_size) != input_ndim: + if input_ndim == 3 and len(act_kwargs.block_size) == 2: + block_size_updated = [1] + list(act_kwargs.block_size) + else: + block_size_updated = list(act_kwargs.block_size)[-input_ndim:] + act_kwargs = QuantizeTensorToInt8Kwargs(block_size=block_size_updated) + activation_tensor = _choose_quant_func_and_quantize_tensor( - activation_tensor, weight_tensor.act_quant_kwargs + activation_tensor, act_kwargs ) - x_vals = activation_tensor.qdata - x_scales = activation_tensor.scale - w_vals_t = weight_tensor.qdata.contiguous().t() - w_scales = weight_tensor.scale - - tmp = x_vals.reshape(-1, x_vals.shape[-1]) + x_vals = activation_tensor.qdata.reshape(-1, activation_tensor.qdata.shape[-1]) + x_scales = preprocess_scale(activation_tensor.scale, x_vals.shape) + w_vals_t = w_q_2d.contiguous().t() intermediate_dtype = ( torch.float if x_scales.dtype == torch.half else x_scales.dtype ) y_dot_scaled = int_scaled_matmul( - tmp, w_vals_t, x_scales.reshape(-1, 1).to(intermediate_dtype) + x_vals, w_vals_t, x_scales.to(intermediate_dtype) ) - y_dot_scaled = y_dot_scaled.to(x_scales.dtype) + y_dot_scaled = y_dot_scaled.to(activation_tensor.scale.dtype) - result = (y_dot_scaled * w_scales).reshape( - *x_vals.shape[:-1], y_dot_scaled.shape[-1] + result = (y_dot_scaled * w_scale_2d).reshape( + *activation_tensor.shape[:-1], *original_weight_shape[:-1] ) result = result.to(activation_tensor.dtype) else: # FP × INT8 (weight-only) - w_vals_int8_t = weight_tensor.qdata.t() + w_vals_int8_t = w_q_2d.t() m = torch.mm( activation_tensor.reshape(-1, activation_tensor.shape[-1]), w_vals_int8_t.to(activation_tensor.dtype), ) - result = m * weight_tensor.scale.to(m.dtype) - result = result.reshape(*activation_tensor.shape[:-1], result.shape[-1]) + result = m * w_scale_2d.to(m.dtype) + result = result.reshape( + *activation_tensor.shape[:-1], *original_weight_shape[:-1] + ) return result + bias if bias is not None else result @@ -212,6 +240,11 @@ def _(func, types, args, kwargs): if step != 1: raise NotImplementedError("Slicing with step > 1 is not supported") + assert dim in [0, 1, 2], f"Only dim=0,1,2 are supported, got: dim={dim}" + assert self.qdata.ndim in [2, 3], ( + f"Expected qdata to have dim=2,3 got: dim={self.qdata.ndim}" + ) + if end >= self.shape[dim]: end = self.shape[dim] @@ -252,7 +285,7 @@ def _(func, types, args, kwargs): Int8Tensor( self.qdata[index], selected_scale, - self.block_size, + self.block_size[1:], self.act_quant_kwargs, self.dtype, ), From 930171719f126888ceea9b201fc2af31ffea117f Mon Sep 17 00:00:00 2001 From: youn17 Date: Fri, 31 Oct 2025 18:54:35 +0900 Subject: [PATCH 5/7] add kernel detection test case --- .../workflows/int8/test_int8_tensor.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py index fb091a0e22..83ca569af4 100644 --- a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py +++ b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py @@ -8,6 +8,7 @@ import unittest import torch +from torch._inductor.utils import run_and_get_code from torch.testing._internal import common_utils from torchao.quantization import ( @@ -255,6 +256,43 @@ def test_dequantization_accuracy(self): msg=f"Dequantization error exceeds tolerance of {0.1}", ) + def test_available_gpu_kernels(self): + """Check which GPU kernels are available""" + M, K, N = 128, 256, 512 + m = torch.nn.Sequential( + torch.nn.Linear(K, N, device="cuda", dtype=torch.bfloat16) + ) + config = Int8DynamicActivationInt8WeightConfig(version=2) + quantize_(m, config) + m = torch.compile(m) + x = torch.randn(M, K, device="cuda", dtype=torch.bfloat16) + + try: + out, code = run_and_get_code(m, x) + kernels_found = {} + + # Check for Triton kernels + if "torch.ops.triton" in code[0]: + kernels_found["triton"] = True + print("Triton kernels are available for int8 quantization") + else: + kernels_found["triton"] = False + print("Triton kernels are NOT available for int8 quantization") + + # Check for FBGEMM kernels + if "torch.ops.fbgemm" in code[0]: + kernels_found["fbgemm"] = True + print("FBGEMM kernels are available for int8 quantization") + else: + kernels_found["fbgemm"] = False + print("FBGEMM kernels are NOT available for int8 quantization") + + # Just log what we found, don't fail the test + print(f"Available kernels for int8 quantization: {kernels_found}") + + except Exception as e: + print(f"Could not check available kernels: {e}") + if __name__ == "__main__": common_utils.run_tests() From caaba7a5dcd51ffadccb620094fe89376c0e9604 Mon Sep 17 00:00:00 2001 From: youn17 Date: Fri, 31 Oct 2025 19:05:19 +0900 Subject: [PATCH 6/7] refactor kernel test --- .../workflows/int8/test_int8_tensor.py | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py index 83ca569af4..1ea3b22948 100644 --- a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py +++ b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py @@ -267,31 +267,24 @@ def test_available_gpu_kernels(self): m = torch.compile(m) x = torch.randn(M, K, device="cuda", dtype=torch.bfloat16) - try: - out, code = run_and_get_code(m, x) - kernels_found = {} - - # Check for Triton kernels - if "torch.ops.triton" in code[0]: - kernels_found["triton"] = True - print("Triton kernels are available for int8 quantization") - else: - kernels_found["triton"] = False - print("Triton kernels are NOT available for int8 quantization") - - # Check for FBGEMM kernels - if "torch.ops.fbgemm" in code[0]: - kernels_found["fbgemm"] = True - print("FBGEMM kernels are available for int8 quantization") - else: - kernels_found["fbgemm"] = False - print("FBGEMM kernels are NOT available for int8 quantization") - - # Just log what we found, don't fail the test - print(f"Available kernels for int8 quantization: {kernels_found}") - - except Exception as e: - print(f"Could not check available kernels: {e}") + out, code = run_and_get_code(m, x) + kernels = {} + + # Check for Triton kernels + if "torch.ops.triton" in code[0]: + kernels["triton"] = True + print("Triton kernels are available for int8 quantization") + else: + kernels["triton"] = False + print("Triton kernels are NOT available for int8 quantization") + + # Check for FBGEMM kernels + if "torch.ops.fbgemm" in code[0]: + kernels["fbgemm"] = True + print("FBGEMM kernels are available for int8 quantization") + else: + kernels["fbgemm"] = False + print("FBGEMM kernels are NOT available for int8 quantization") if __name__ == "__main__": From 0f51ee66c3cc6d9ffd9cab86b766d22ca29c00e7 Mon Sep 17 00:00:00 2001 From: younn17 Date: Tue, 4 Nov 2025 22:47:31 +0900 Subject: [PATCH 7/7] update linear variant, kernel detection test - Configs are updated to global variants --- .../workflows/int8/test_int8_tensor.py | 137 +++++++----------- .../quantize_/workflows/int8/int8_tensor.py | 18 ++- 2 files changed, 65 insertions(+), 90 deletions(-) diff --git a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py index 1ea3b22948..a7411cc967 100644 --- a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py +++ b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py @@ -16,9 +16,6 @@ Int8WeightOnlyConfig, quantize_, ) -from torchao.quantization.quantize_.workflows.int8.int8_tensor import ( - Int8Tensor, -) from torchao.quantization.utils import compute_error from torchao.testing.utils import TorchAOIntegrationTestCase @@ -56,7 +53,7 @@ class TestInt8Tensor(TorchAOIntegrationTestCase): def setUp(self): super().setUp() - self.test_shape = (4, 3) + self.test_shape = (32, 20) self.dtype = torch.bfloat16 self.batch_size = 32 @@ -66,9 +63,26 @@ def setUp(self): self.bias = torch.randn(self.test_shape[0], dtype=self.dtype) self.block_size = list(self.test_shape) - def test_creation_and_attributes(self): + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + def test_creation_and_attributes(self, config): """Test tensor creation, dtypes, and ranges""" - tensor = Int8Tensor.from_hp(self.weight_fp, self.block_size) + linear = torch.nn.Linear( + self.test_shape[1], + self.test_shape[0], + bias=False, + dtype=self.dtype, + device="cuda", + ) + linear.weight.data = self.weight_fp.cuda() + quantize_(linear, config) + + tensor = linear.weight self.assertEqual(tensor.shape, self.test_shape) self.assertEqual(tensor.qdata.dtype, torch.int8) @@ -117,46 +131,6 @@ def test_int8_linear_variants( f"Quantization error is too high got a SQNR of {compute_error(output_fp, output_quantized)}" ) - @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) - @common_utils.parametrize( - "config", - [ - Int8DynamicActivationInt8WeightConfig(version=2), - Int8WeightOnlyConfig(version=2), - ], - ) - @common_utils.parametrize( - "sizes", - [ - ((128,), 256, 128), # 2D - ((32, 128), 64, 256), # 3D - ], - ) - def test_int8_linear_quantization_accuracy( - self, - dtype: torch.dtype, - sizes: tuple, - config, - ): - """Test quantization preserves reasonable accuracy""" - M, N, K = sizes - input_tensor = torch.randn(*M, K, dtype=dtype, device="cuda") - - # Create a linear layer - m = ToyTwoLinearModel(K, N, K, dtype=dtype, device="cuda").eval() - m_q = copy.deepcopy(m) - - # Quantize - quantize_(m_q, config) - - output_fp = m(input_tensor) - output_quantized = m_q(input_tensor) - - error = compute_error(output_fp, output_quantized) - assert error > 20, ( - f"Quantization quality is too low, SQNR: {error}dB (expected > {20}dB)" - ) - @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) @common_utils.parametrize( "config", @@ -218,36 +192,42 @@ def test_slice(self, config, device, dtype): with self.assertRaises(NotImplementedError): _ = dummy.weight[::2] - def test_index_select(self): - """test that `x_0 = x[0]` works when `x` is a 2D `Int8Tensor`.""" + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + def test_index_select(self, config): + """test that `x_0 = x[0]` works when `x` is a 2D quantized tensor.""" N, K = 256, 512 x = torch.randn(N, K, device="cuda", dtype=torch.bfloat16) - x_int8 = Int8Tensor.from_hp(x, block_size=[N, K]) + linear = torch.nn.Linear(K, N, bias=False, dtype=torch.bfloat16, device="cuda") + linear.weight.data = x + quantize_(linear, config) + + x_int8 = linear.weight x_int8_0 = x_int8[0] torch.testing.assert_close( x_int8.dequantize()[0], x_int8_0.dequantize(), atol=0, rtol=0 ) - def test_invalid_input_handling(self): - """Test input validation with specific error types""" - invalid_tensor = torch.randn(5) - incompatible_block_size = [1] - - with self.assertRaises( - ValueError, msg="Should reject incompatible tensor dimensions" - ): - Int8Tensor.from_hp(invalid_tensor, incompatible_block_size) - - with self.assertRaises( - ValueError, msg="Should reject mismatched block size dimensions" - ): - Int8Tensor.from_hp(self.weight_fp, [1]) - - def test_dequantization_accuracy(self): + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + def test_dequantization_accuracy(self, config): """Test dequantization accuracy separately""" - test_data = torch.tensor([[1.0, -1.0]], dtype=torch.bfloat16) - tensor = Int8Tensor.from_hp(test_data, [1, 2]) + test_data = torch.tensor([[1.0, -1.0]], dtype=torch.bfloat16, device="cuda") + linear = torch.nn.Linear(2, 1, bias=False, dtype=torch.bfloat16, device="cuda") + linear.weight.data = test_data + quantize_(linear, config) + tensor = linear.weight dequantized = tensor.dequantize() self.assertEqual(dequantized.shape, test_data.shape) self.assertLess( @@ -268,23 +248,14 @@ def test_available_gpu_kernels(self): x = torch.randn(M, K, device="cuda", dtype=torch.bfloat16) out, code = run_and_get_code(m, x) - kernels = {} - - # Check for Triton kernels - if "torch.ops.triton" in code[0]: - kernels["triton"] = True - print("Triton kernels are available for int8 quantization") - else: - kernels["triton"] = False - print("Triton kernels are NOT available for int8 quantization") + has_triton = "triton" in code[0].lower() # Trition + has_fbgemm = "fbgemm" in code[0].lower() # FB-GEMM + has_int_mm = "_int_mm" in code[0] # Int8 MatMul - # Check for FBGEMM kernels - if "torch.ops.fbgemm" in code[0]: - kernels["fbgemm"] = True - print("FBGEMM kernels are available for int8 quantization") - else: - kernels["fbgemm"] = False - print("FBGEMM kernels are NOT available for int8 quantization") + self.assertTrue( + has_triton or has_fbgemm or has_int_mm, + f"No int8 quantization kernels found. has_triton={has_triton}, has_fbgemm={has_fbgemm}, has_int_mm={has_int_mm}", + ) if __name__ == "__main__": diff --git a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py index a6e747e7e1..11633ebc24 100644 --- a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py +++ b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py @@ -143,10 +143,11 @@ def dequantize(self, output_dtype: Optional[torch.dtype] = None) -> torch.Tensor output_dtype = self.dtype qdata_fp = self.qdata.to(output_dtype) - # Reshape scale to broadcast if granularity is block-wise - scale_expanded = _maybe_expand_scale_to_tensor_shape( - self.scale, self.qdata.shape - ) + scale = self.scale + while scale.ndim < qdata_fp.ndim: + scale = scale.unsqueeze(-1) + + scale_expanded = _maybe_expand_scale_to_tensor_shape(scale, qdata_fp.shape) return qdata_fp * scale_expanded.to(output_dtype) @@ -276,16 +277,19 @@ def _(func, types, args, kwargs): self, dim, index = args assert dim == 0, f"Only dim=0 supported, got {dim}" - selected_scale = self.scale if self.scale.ndim == 0 else self.scale[index] + selected_qdata = self.qdata[index] + selected_scale = _slice_scale_for_dimension( + self.scale, self.qdata.shape, dim, index, index + 1, step=1 + ).squeeze(0) return return_and_correct_aliasing( func, args, kwargs, Int8Tensor( - self.qdata[index], + selected_qdata, selected_scale, - self.block_size[1:], + [selected_qdata.shape[-1]], self.act_quant_kwargs, self.dtype, ),