diff --git a/docs/source/quantization_overview.rst b/docs/source/quantization_overview.rst index f5c82bfe5f..df0a924b11 100644 --- a/docs/source/quantization_overview.rst +++ b/docs/source/quantization_overview.rst @@ -5,7 +5,7 @@ First we want to lay out the torchao stack:: Quantization Algorithms/Flows: weight only/dynamic/static quantization, hqq, awq, gptq etc. --------------------------------------------------------------------------------------------- - Quantized Tensors (derived dtypes): Int4Tensor, Int4PreshuffledTensor, Float8Tensor + Quantized Tensors (derived dtypes): Int4Tensor, Int4PreshuffledTensor, Int8Tensor, Float8Tensor --------------------------------------------------------------------------------------------- Quantization Primitive Ops/Efficient Kernels: matmul, quantize, dequantize --------------------------------------------------------------------------------------------- @@ -88,6 +88,8 @@ So in general we structure Tensor subclasses by dervied dtpype and packing forma - scaled int4 - preshuffled (special format to optimize for loading) - float8 act + int4 weight dynamic quantization and int4 weight only quantization + * - Int8Tensor + - plain .. note:: We don't have granularity specific tensor subclasses, i.e. no Float8RowwiseTensor or Float8BlockwiseTensor, all granularities are implemented in the same Tensor, we typically use a general `block_size` attribute to distinguish between different granularities, and each Tensor is allowed to support only a subset of all possible granularity options. diff --git a/test/quantization/quantize_/workflows/int8/test_int8_tensor.py b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py new file mode 100644 index 0000000000..a7411cc967 --- /dev/null +++ b/test/quantization/quantize_/workflows/int8/test_int8_tensor.py @@ -0,0 +1,262 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD 3-Clause license found in the +# LICENSE file in the root directory of this source tree. + +import copy +import unittest + +import torch +from torch._inductor.utils import run_and_get_code +from torch.testing._internal import common_utils + +from torchao.quantization import ( + Int8DynamicActivationInt8WeightConfig, + Int8WeightOnlyConfig, + quantize_, +) +from torchao.quantization.utils import compute_error +from torchao.testing.utils import TorchAOIntegrationTestCase + + +# TODO: Refactor after https://github.com/pytorch/ao/pull/2729 is merged +class ToyTwoLinearModel(torch.nn.Module): + def __init__( + self, + input_dim, + hidden_dim, + output_dim, + has_bias=False, + dtype=None, + device=None, + ): + super().__init__() + self.dtype = dtype + self.device = device + self.linear1 = torch.nn.Linear( + input_dim, hidden_dim, bias=has_bias, dtype=dtype, device=device + ) + self.linear2 = torch.nn.Linear( + hidden_dim, output_dim, bias=has_bias, dtype=dtype, device=device + ) + + def forward(self, x): + x = self.linear1(x) + x = self.linear2(x) + return x + + +@unittest.skipIf(not torch.cuda.is_available(), "Need CUDA available") +@common_utils.instantiate_parametrized_tests +class TestInt8Tensor(TorchAOIntegrationTestCase): + def setUp(self): + super().setUp() + + self.test_shape = (32, 20) + self.dtype = torch.bfloat16 + self.batch_size = 32 + + torch.manual_seed(42) + self.weight_fp = torch.randn(*self.test_shape, dtype=self.dtype) + self.input_fp = torch.randn(*self.test_shape, dtype=self.dtype) + self.bias = torch.randn(self.test_shape[0], dtype=self.dtype) + self.block_size = list(self.test_shape) + + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + def test_creation_and_attributes(self, config): + """Test tensor creation, dtypes, and ranges""" + linear = torch.nn.Linear( + self.test_shape[1], + self.test_shape[0], + bias=False, + dtype=self.dtype, + device="cuda", + ) + linear.weight.data = self.weight_fp.cuda() + quantize_(linear, config) + + tensor = linear.weight + + self.assertEqual(tensor.shape, self.test_shape) + self.assertEqual(tensor.qdata.dtype, torch.int8) + self.assertTrue( + torch.all(tensor.qdata >= -128) and torch.all(tensor.qdata <= 127) + ) + + @common_utils.parametrize("dtype", [torch.bfloat16, torch.float32]) + @common_utils.parametrize("compile", [True, False]) + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + @common_utils.parametrize( + "sizes", + [ + ((128,), 256, 128), # 2D + ((32, 128), 64, 256), # 3D + ], + ) + def test_int8_linear_variants( + self, + dtype: torch.dtype, + config, + compile: bool, + sizes: tuple, + ): + """Test linear operation supports including shape and compile""" + M, N, K = sizes + input_tensor = torch.randn(*M, K, dtype=dtype, device="cuda") + model = ToyTwoLinearModel(K, N, K, dtype=dtype, device="cuda").eval() + model_q = copy.deepcopy(model) + + quantize_(model_q, config) + + if compile: + model_q = torch.compile(model_q, fullgraph=True) + + output_fp = model(input_tensor) + output_quantized = model_q(input_tensor) + + assert compute_error(output_fp, output_quantized) > 20, ( + f"Quantization error is too high got a SQNR of {compute_error(output_fp, output_quantized)}" + ) + + @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + def test_per_row_scale_shape(self, dtype, config): + """Test per-row quantization maintains 1D scale""" + N, K = 64, 128 + linear = torch.nn.Linear(K, N, bias=False, dtype=dtype, device="cuda") + quantize_(linear, config) + + # Dynamic: per-row (1D scale [N]), Weight-only: per-tensor (scalar) + if isinstance(config, Int8DynamicActivationInt8WeightConfig): + self.assertEqual(linear.weight.scale.shape, (N,)) + self.assertEqual(linear.weight.scale.ndim, 1) + else: + self.assertEqual(linear.weight.scale.numel(), 1) + + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + @common_utils.parametrize("device", ["cpu", "cuda"]) + @common_utils.parametrize("dtype", [torch.bfloat16, torch.float16]) + def test_slice(self, config, device, dtype): + """Test tensor slicing""" + tensor_size = 256 + slice_sizes = (64, 128) + + dummy = torch.nn.Linear( + tensor_size, tensor_size, bias=False, dtype=dtype, device=device + ) + quantize_(dummy, config) + + weight1 = dummy.weight.clone().narrow(0, 0, slice_sizes[0]) + weight2 = dummy.weight.clone().narrow(1, 0, slice_sizes[1]) + + self.assertEqual(weight1.qdata, dummy.weight.qdata.narrow(0, 0, slice_sizes[0])) + self.assertEqual(weight2.qdata, dummy.weight.qdata.narrow(1, 0, slice_sizes[1])) + + # Int8DynamicActivationInt8WeightConfig uses per-row (PerRow) + # Int8WeightOnlyConfig uses per-tensor (PerTensor) + if isinstance(config, Int8DynamicActivationInt8WeightConfig): + # PerRow: dim 0 slicing affects scale, dim 1 doesn't + self.assertEqual( + weight1.scale, dummy.weight.scale.narrow(0, 0, slice_sizes[0]) + ) + self.assertEqual(weight2.scale, dummy.weight.scale) + else: + # PerTensor: scale unchanged by slicing + self.assertEqual(weight1.scale, dummy.weight.scale) + self.assertEqual(weight2.scale, dummy.weight.scale) + with self.assertRaises(NotImplementedError): + _ = dummy.weight[::2] + + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + def test_index_select(self, config): + """test that `x_0 = x[0]` works when `x` is a 2D quantized tensor.""" + N, K = 256, 512 + x = torch.randn(N, K, device="cuda", dtype=torch.bfloat16) + linear = torch.nn.Linear(K, N, bias=False, dtype=torch.bfloat16, device="cuda") + linear.weight.data = x + quantize_(linear, config) + + x_int8 = linear.weight + x_int8_0 = x_int8[0] + torch.testing.assert_close( + x_int8.dequantize()[0], x_int8_0.dequantize(), atol=0, rtol=0 + ) + + @common_utils.parametrize( + "config", + [ + Int8DynamicActivationInt8WeightConfig(version=2), + Int8WeightOnlyConfig(version=2), + ], + ) + def test_dequantization_accuracy(self, config): + """Test dequantization accuracy separately""" + test_data = torch.tensor([[1.0, -1.0]], dtype=torch.bfloat16, device="cuda") + linear = torch.nn.Linear(2, 1, bias=False, dtype=torch.bfloat16, device="cuda") + linear.weight.data = test_data + quantize_(linear, config) + + tensor = linear.weight + dequantized = tensor.dequantize() + self.assertEqual(dequantized.shape, test_data.shape) + self.assertLess( + torch.abs(dequantized - test_data).max().item(), + 0.1, + msg=f"Dequantization error exceeds tolerance of {0.1}", + ) + + def test_available_gpu_kernels(self): + """Check which GPU kernels are available""" + M, K, N = 128, 256, 512 + m = torch.nn.Sequential( + torch.nn.Linear(K, N, device="cuda", dtype=torch.bfloat16) + ) + config = Int8DynamicActivationInt8WeightConfig(version=2) + quantize_(m, config) + m = torch.compile(m) + x = torch.randn(M, K, device="cuda", dtype=torch.bfloat16) + + out, code = run_and_get_code(m, x) + has_triton = "triton" in code[0].lower() # Trition + has_fbgemm = "fbgemm" in code[0].lower() # FB-GEMM + has_int_mm = "_int_mm" in code[0] # Int8 MatMul + + self.assertTrue( + has_triton or has_fbgemm or has_int_mm, + f"No int8 quantization kernels found. has_triton={has_triton}, has_fbgemm={has_fbgemm}, has_int_mm={has_int_mm}", + ) + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/torchao/float8/inference.py b/torchao/float8/inference.py index f15d38576c..da58b54269 100644 --- a/torchao/float8/inference.py +++ b/torchao/float8/inference.py @@ -140,7 +140,18 @@ def _slice_scale_for_dimension( """ aten = torch.ops.aten - # Unsupported case for now, this would be 1 scale per data element + # Per-tensor quantization (scalar scale) + if scale.numel() == 1: + return scale + + # Per-row quantization (1D scale) + if scale.ndim == 1: + if dim == 0: + return aten.slice.Tensor(scale, 0, start, end, step) + else: + return scale + + # Block-wise quantization (2D scale) if scale.shape == data_shape: return aten.slice.Tensor(scale, dim, start, end, step) @@ -158,6 +169,12 @@ def _slice_scale_for_dimension( # Slice away as normal return aten.slice.Tensor(scale, dim, start, end, step) else: + # Error on Step > 1 + if step > 1: + raise NotImplementedError( + "Slicing with step > 1 is not implemented for scale tensors." + ) + # There is blocking in this dimension # Calculate which scale elements correspond to the sliced data scale_start = start // block_size_for_dim if start is not None else None @@ -167,12 +184,6 @@ def _slice_scale_for_dimension( else None ) - # Error on Step > 1 - if step > 1: - raise NotImplementedError( - "Slicing with step > 1 is not implemented for scale tensors." - ) - return aten.slice.Tensor(scale, dim, scale_start, scale_end, 1) diff --git a/torchao/quantization/__init__.py b/torchao/quantization/__init__.py index 87e011c57b..05172063ba 100644 --- a/torchao/quantization/__init__.py +++ b/torchao/quantization/__init__.py @@ -97,6 +97,7 @@ Int4PreshuffledTensor, Int4Tensor, Int4TilePackedTo4dTensor, + Int8Tensor, IntxOpaqueTensor, IntxUnpackedToInt8Tensor, ) @@ -168,6 +169,7 @@ "IntxOpaqueTensor", "IntxUnpackedToInt8Tensor", "Int4TilePackedTo4dTensor", + "Int8Tensor", "Float8Tensor", "Int4OpaqueTensor", # smooth quant - subject to change diff --git a/torchao/quantization/quant_api.py b/torchao/quantization/quant_api.py index ae8210a41a..6da83a40a0 100644 --- a/torchao/quantization/quant_api.py +++ b/torchao/quantization/quant_api.py @@ -81,6 +81,7 @@ Int4PreshuffledTensor, Int4Tensor, Int4TilePackedTo4dTensor, + Int8Tensor, IntxChooseQParamsAlgorithm, IntxOpaqueTensor, IntxPackingFormat, @@ -1353,6 +1354,7 @@ class Int8WeightOnlyConfig(AOBaseConfig): group_size: Optional[int] = None set_inductor_config: bool = True + version: int = 1 def __post_init__(self): torch._C._log_api_usage_once("torchao.quantization.Int8WeightOnlyConfig") @@ -1363,22 +1365,30 @@ def __post_init__(self): def _int8_weight_only_quantize_tensor(weight, config): - mapping_type = MappingType.SYMMETRIC - target_dtype = torch.int8 - eps = torch.finfo(torch.float32).eps - zero_point_dtype = torch.int64 - group_size = config.group_size - if group_size is None: - group_size = weight.shape[-1] - block_size = tuple([1 for x in range(weight.dim() - 1)] + [group_size]) - new_weight = to_affine_quantized_intx( - weight, - mapping_type, - block_size, - target_dtype, - eps=eps, - zero_point_dtype=zero_point_dtype, - ) + if config.version == 1: + warnings.warn( + "Config Deprecation: version 1 of Int8WeightOnlyConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2752 for more details" + ) + mapping_type = MappingType.SYMMETRIC + target_dtype = torch.int8 + eps = torch.finfo(torch.float32).eps + zero_point_dtype = torch.int64 + group_size = config.group_size + if group_size is None: + group_size = weight.shape[-1] + block_size = tuple([1 for x in range(weight.dim() - 1)] + [group_size]) + new_weight = to_affine_quantized_intx( + weight, + mapping_type, + block_size, + target_dtype, + eps=eps, + zero_point_dtype=zero_point_dtype, + ) + else: + assert config.version == 2, f"Unexpected version: {config.version}" + block_size = [weight.shape[0], weight.shape[1]] + new_weight = Int8Tensor.from_hp(weight, block_size=block_size) return new_weight @@ -1506,12 +1516,15 @@ class Int8DynamicActivationInt8WeightConfig(AOBaseConfig): in original precision during decode operations. set_inductor_config: bool = True - If True, adjusts `torchinductor` settings to recommended values for better performance with this quantization scheme. + version (int): the version of the config, version 1 is using AffineQuantizedTensor that we plan to deprecate/split, version 2 is using Int8Tensor """ layout: Optional[Layout] = PlainLayout() act_mapping_type: Optional[MappingType] = MappingType.SYMMETRIC weight_only_decode: bool = False + granularity: Optional[Union[PerRow, PerTensor]] = PerRow() set_inductor_config: bool = True + version: int = 1 def __post_init__(self): torch._C._log_api_usage_once( @@ -1543,9 +1556,6 @@ def _int8_dynamic_activation_int8_weight_quantize_tensor(weight, config): mapping_type = MappingType.SYMMETRIC weight_zero_point_domain = ZeroPointDomain.NONE - def get_weight_block_size(x): - return tuple([1 for _ in range(x.dim() - 1)] + [x.shape[-1]]) - target_dtype = torch.int8 eps = torch.finfo(torch.float32).eps zero_point_dtype = torch.int64 @@ -1559,19 +1569,43 @@ def get_weight_block_size(x): else: input_quant_func = _int8_asymm_per_token_quant - block_size = get_weight_block_size(weight) - new_weight = to_affine_quantized_intx( - weight, - mapping_type, - block_size, - target_dtype, - eps=eps, - zero_point_dtype=zero_point_dtype, - _layout=layout, - zero_point_domain=weight_zero_point_domain, - ) - new_weight = to_linear_activation_quantized(new_weight, input_quant_func) - return new_weight + if isinstance(config.granularity, PerTensor): + # Tensor granularity + block_size = weight.shape + else: + # Per row granularity + block_size = tuple([1 for _ in range(weight.dim() - 1)] + [weight.shape[-1]]) + + if config.version == 1: + warnings.warn( + "Config Deprecation: version 1 of Int8DynamicActivationInt8WeightConfig is deprecated and will no longer be supported in a future release, please use version 2, see https://github.com/pytorch/ao/issues/2752 for more details" + ) + quantized_weight = to_affine_quantized_intx( + weight, + mapping_type, + block_size, + target_dtype, + eps=eps, + zero_point_dtype=zero_point_dtype, + _layout=layout, + zero_point_domain=weight_zero_point_domain, + ) + quantized_weight = to_linear_activation_quantized( + quantized_weight, input_quant_func + ) + else: + from torchao.quantization.quantize_.workflows.int8.int8_tensor import ( + QuantizeTensorToInt8Kwargs, + ) + + assert config.version == 2, f"Unexpected version: {config.version}" + quantized_weight = Int8Tensor.from_hp( + weight, + block_size, + act_quant_kwargs=QuantizeTensorToInt8Kwargs(block_size=block_size), + ) + + return quantized_weight @register_quantize_module_handler(Int8DynamicActivationInt8WeightConfig) diff --git a/torchao/quantization/quantize_/common/quantize_tensor_kwargs.py b/torchao/quantization/quantize_/common/quantize_tensor_kwargs.py index 0adc8c786d..44dd09ff62 100644 --- a/torchao/quantization/quantize_/common/quantize_tensor_kwargs.py +++ b/torchao/quantization/quantize_/common/quantize_tensor_kwargs.py @@ -39,7 +39,9 @@ def _choose_quant_func_and_quantize_tensor( """ from torchao.quantization.quantize_.workflows import ( Float8Tensor, + Int8Tensor, QuantizeTensorToFloat8Kwargs, + QuantizeTensorToInt8Kwargs, ) if isinstance(quant_kwargs, QuantizeTensorToFloat8Kwargs): @@ -52,5 +54,11 @@ def _choose_quant_func_and_quantize_tensor( quant_kwargs.hp_value_ub, quant_kwargs.kernel_preference, ) + elif isinstance(quant_kwargs, QuantizeTensorToInt8Kwargs): + return Int8Tensor.from_hp( + tensor, + quant_kwargs.block_size, + act_quant_kwargs=quant_kwargs, + ) raise NotImplementedError(f"Quant kwargs not supported: {quant_kwargs}") diff --git a/torchao/quantization/quantize_/workflows/__init__.py b/torchao/quantization/quantize_/workflows/__init__.py index 4307637f8e..3891f28dbe 100644 --- a/torchao/quantization/quantize_/workflows/__init__.py +++ b/torchao/quantization/quantize_/workflows/__init__.py @@ -20,6 +20,10 @@ Int4Tensor, ) from .int4.int4_tile_packed_to_4d_tensor import Int4TilePackedTo4dTensor +from .int8.int8_tensor import ( + Int8Tensor, + QuantizeTensorToInt8Kwargs, +) from .intx.intx_choose_qparams_algorithm import IntxChooseQParamsAlgorithm from .intx.intx_opaque_tensor import ( IntxOpaqueTensor, @@ -37,6 +41,8 @@ "Int4MarlinSparseTensor", "Int4PlainInt32Tensor", "Int4TilePackedTo4dTensor", + "Int8Tensor", + "QuantizeTensorToInt8Kwargs", "Float8Tensor", "QuantizeTensorToFloat8Kwargs", "Int4OpaqueTensor", diff --git a/torchao/quantization/quantize_/workflows/int8/int8_tensor.py b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py new file mode 100644 index 0000000000..11633ebc24 --- /dev/null +++ b/torchao/quantization/quantize_/workflows/int8/int8_tensor.py @@ -0,0 +1,300 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD 3-Clause license found in the +# LICENSE file in the root directory of this source tree. + +from dataclasses import dataclass +from typing import Optional + +import torch +from torch.utils._python_dispatch import return_and_correct_aliasing + +from torchao.float8.inference import ( + _slice_scale_for_dimension, + preprocess_scale, +) +from torchao.kernel import int_scaled_matmul +from torchao.quantization.quant_primitives import ( + MappingType, + _maybe_expand_scale_to_tensor_shape, + choose_qparams_affine, + quantize_affine, +) +from torchao.quantization.quantize_.common import ( + QuantizeTensorKwargs, + _choose_quant_func_and_quantize_tensor, +) +from torchao.utils import TorchAOBaseTensor, fill_defaults + +__all__ = ["Int8Tensor", "QuantizeTensorToInt8Kwargs"] + +aten = torch.ops.aten + + +@dataclass +class QuantizeTensorToInt8Kwargs(QuantizeTensorKwargs): + """Tensor kwargs for creating int8 tensor (either activation or weight) + + Args: + block_size (list[int]): block size for quantization granularity + # TODO: Static quantization support using `static_scale`, `static_zero_point` + """ + + block_size: list[int] + + +class Int8Tensor(TorchAOBaseTensor): + """ + int8 quantized tensor with plain layout + + Tensor Attributes: + qdata: (N, K) or (B, N, K) int8 quantized weight data (2D or 3D) + scale: scale factors for dequantization + + Non-Tensor Attributes: + block_size: block size for quantization granularity + act_quant_kwargs: flags for dynamic activation quantization + """ + + tensor_data_names = ["qdata", "scale"] + tensor_attribute_names = ["block_size"] + optional_tensor_attribute_names = [ + "act_quant_kwargs", + "dtype", + ] + + def __new__( + cls: type, + qdata: torch.Tensor, + scale: torch.Tensor, + block_size: list[int], + act_quant_kwargs=None, + dtype=None, + ): + kwargs = { + "device": qdata.device, + "dtype": dtype or scale.dtype, + "requires_grad": False, + } + return torch.Tensor._make_wrapper_subclass(cls, qdata.shape, **kwargs) + + def __init__( + self, + qdata: torch.Tensor, + scale: torch.Tensor, + block_size: list[int], + act_quant_kwargs=None, + dtype=None, + ): + super().__init__() + self.qdata = qdata + self.scale = scale + self.block_size = block_size + self.act_quant_kwargs = act_quant_kwargs + + def __repr__(self): + return ( + f"{self.__class__.__name__}({self.act_quant_kwargs=}, {self.qdata=}, {self.scale=}, " + f"{self.block_size=}, {self.shape=}, {self.device=}, {self.dtype=})" + ) + + @classmethod + def from_hp( + cls, + w: torch.Tensor, + block_size: list[int], + act_quant_kwargs: Optional[QuantizeTensorToInt8Kwargs] = None, + ): + if w.dim() not in [2, 3] or len(block_size) != w.dim(): + raise ValueError("Expected 2D or 3D tensor with same block_size length") + + scale, zero_point = choose_qparams_affine( + input=w, + mapping_type=MappingType.SYMMETRIC, + block_size=block_size, + target_dtype=torch.int8, + quant_min=-128, + quant_max=127, + scale_dtype=w.dtype, + zero_point_dtype=torch.int8, + ) + + int_data = quantize_affine( + w, + block_size=block_size, + scale=scale, + zero_point=zero_point, + output_dtype=torch.int8, + ) + + return cls( + int_data, + scale, + block_size, + act_quant_kwargs=act_quant_kwargs, + dtype=w.dtype, + ) + + def dequantize(self, output_dtype: Optional[torch.dtype] = None) -> torch.Tensor: + """Dequantize int8 tensor to floating point""" + + if output_dtype is None: + output_dtype = self.dtype + + qdata_fp = self.qdata.to(output_dtype) + scale = self.scale + while scale.ndim < qdata_fp.ndim: + scale = scale.unsqueeze(-1) + + scale_expanded = _maybe_expand_scale_to_tensor_shape(scale, qdata_fp.shape) + return qdata_fp * scale_expanded.to(output_dtype) + + +implements = Int8Tensor.implements +implements_torch_function = Int8Tensor.implements_torch_function + + +@implements(aten.linear.default) +@implements_torch_function(torch.nn.functional.linear) +def _(func, types, args, kwargs): + """INT8 quantization: dynamic activation or weight-only""" + activation_tensor, weight_tensor, bias = ( + args[0], + args[1], + args[2] if len(args) > 2 else None, + ) + + assert isinstance(weight_tensor, Int8Tensor), ( + f"Expected weight to be Int8Tensor, got {type(weight_tensor)}" + ) + + # Store original shape for reshaping result + original_weight_shape = weight_tensor.qdata.shape + + # Reshape 3D weights to 2D: (B, N, K) -> (B*N, K) + if weight_tensor.qdata.dim() == 3: + w_q_2d = weight_tensor.qdata.reshape(-1, original_weight_shape[-1]) + w_scale_2d = ( + weight_tensor.scale.reshape(-1) + if weight_tensor.scale.numel() > 1 + else weight_tensor.scale + ) + else: + w_q_2d = weight_tensor.qdata + w_scale_2d = weight_tensor.scale + + if weight_tensor.act_quant_kwargs is not None: + if not isinstance(activation_tensor, Int8Tensor): + # Dynamic activation quantization + act_kwargs = weight_tensor.act_quant_kwargs + input_ndim = activation_tensor.ndim + + # Ensure block_size matches input tensor dimensions + if len(act_kwargs.block_size) != input_ndim: + if input_ndim == 3 and len(act_kwargs.block_size) == 2: + block_size_updated = [1] + list(act_kwargs.block_size) + else: + block_size_updated = list(act_kwargs.block_size)[-input_ndim:] + act_kwargs = QuantizeTensorToInt8Kwargs(block_size=block_size_updated) + + activation_tensor = _choose_quant_func_and_quantize_tensor( + activation_tensor, act_kwargs + ) + + x_vals = activation_tensor.qdata.reshape(-1, activation_tensor.qdata.shape[-1]) + x_scales = preprocess_scale(activation_tensor.scale, x_vals.shape) + w_vals_t = w_q_2d.contiguous().t() + intermediate_dtype = ( + torch.float if x_scales.dtype == torch.half else x_scales.dtype + ) + + y_dot_scaled = int_scaled_matmul( + x_vals, w_vals_t, x_scales.to(intermediate_dtype) + ) + y_dot_scaled = y_dot_scaled.to(activation_tensor.scale.dtype) + + result = (y_dot_scaled * w_scale_2d).reshape( + *activation_tensor.shape[:-1], *original_weight_shape[:-1] + ) + result = result.to(activation_tensor.dtype) + else: + # FP × INT8 (weight-only) + w_vals_int8_t = w_q_2d.t() + m = torch.mm( + activation_tensor.reshape(-1, activation_tensor.shape[-1]), + w_vals_int8_t.to(activation_tensor.dtype), + ) + result = m * w_scale_2d.to(m.dtype) + result = result.reshape( + *activation_tensor.shape[:-1], *original_weight_shape[:-1] + ) + + return result + bias if bias is not None else result + + +@implements(aten.slice.Tensor) +def _(func, types, args, kwargs): + """Slice operation for Int8Tensor""" + self, dim, start, end, step = fill_defaults(args, 5, [0, None, None, 1]) + + if step != 1: + raise NotImplementedError("Slicing with step > 1 is not supported") + + assert dim in [0, 1, 2], f"Only dim=0,1,2 are supported, got: dim={dim}" + assert self.qdata.ndim in [2, 3], ( + f"Expected qdata to have dim=2,3 got: dim={self.qdata.ndim}" + ) + + if end >= self.shape[dim]: + end = self.shape[dim] + + sliced_qdata = aten.slice.Tensor(self.qdata, dim, start, end, step) + sliced_scale = _slice_scale_for_dimension( + self.scale, self.qdata.shape, dim, start, end, step + ) + + block_size = list(self.block_size) + for i in range(len(block_size)): + block_size[i] = min(block_size[i], sliced_qdata.shape[i]) + + return return_and_correct_aliasing( + func, + args, + kwargs, + Int8Tensor( + sliced_qdata, + sliced_scale, + block_size, + self.act_quant_kwargs, + dtype=self.dtype, + ), + ) + + +@implements(aten.select.int) +def _(func, types, args, kwargs): + self, dim, index = args + assert dim == 0, f"Only dim=0 supported, got {dim}" + + selected_qdata = self.qdata[index] + selected_scale = _slice_scale_for_dimension( + self.scale, self.qdata.shape, dim, index, index + 1, step=1 + ).squeeze(0) + + return return_and_correct_aliasing( + func, + args, + kwargs, + Int8Tensor( + selected_qdata, + selected_scale, + [selected_qdata.shape[-1]], + self.act_quant_kwargs, + self.dtype, + ), + ) + + +Int8Tensor.__module__ = "torchao.quantization" +torch.serialization.add_safe_globals([Int8Tensor, QuantizeTensorToInt8Kwargs])