Skip to content

vllm.model_executor.layers.quantization.input_hadamard

Block Hadamard rotation of linear-layer inputs, independent of the weight format.

Rotating each contiguous block of the input dimension by an orthonormal Hadamard matrix H on both operands leaves the product unchanged, (x H)(w H)^T = x w^T, while spreading outliers across each block, which shrinks the error of block-scaled quantization (MXFP4, MXFP8, ...).

Classes:

Functions:

InputHadamardLinearMethod

Bases: LinearMethodBase

Wraps a linear method and rotates the layer input by an orthonormal block Hadamard matrix before the wrapped apply, in the manner of CompressedTensorsLinearTransformMethod.

With rotate_weight, the weight ([out, in]) is rotated the same way right before the wrapped method processes it, for example before online quantization. Otherwise the checkpoint stores the weight already rotated, together with an int8 input_rotation tensor holding the unnormalized (+-1) Sylvester matrix, which is checked and then dropped.

The activation rotation runs in PyTorch unless the wrapped method takes it over through LinearMethodBase.fuse_input_hadamard.

Source code in vllm/model_executor/layers/quantization/input_hadamard.py 
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
@register_weight_loader_v2_supported_method
class InputHadamardLinearMethod(LinearMethodBase):
    """Wraps a linear method and rotates the layer input by an orthonormal
    block Hadamard matrix before the wrapped `apply`, in the manner of
    `CompressedTensorsLinearTransformMethod`.

    With ``rotate_weight``, the weight (``[out, in]``) is rotated the same way
    right before the wrapped method processes it, for example before online
    quantization. Otherwise the checkpoint stores the weight already rotated,
    together with an int8 ``input_rotation`` tensor holding the unnormalized
    (+-1) Sylvester matrix, which is checked and then dropped.

    The activation rotation runs in PyTorch unless the wrapped method takes it
    over through `LinearMethodBase.fuse_input_hadamard`.
    """

    def __init__(
        self, quant_method: LinearMethodBase, size: int, rotate_weight: bool
    ) -> None:
        self.quant_method = quant_method
        self.size = size
        self.rotate_weight = rotate_weight

    @property
    def uses_meta_device(self) -> bool:  # type: ignore[override]
        return self.quant_method.uses_meta_device

    def create_weights(
        self,
        layer: torch.nn.Module,
        input_size_per_partition: int,
        output_partition_sizes: list[int],
        input_size: int,
        output_size: int,
        params_dtype: torch.dtype,
        **extra_weight_attrs,
    ):
        if input_size_per_partition % self.size != 0:
            raise ValueError(
                f"Input size per partition {input_size_per_partition} is not "
                f"divisible by the Hadamard size {self.size}."
            )
        # This class is registered for weight_loader_v2; hand v1 to wrapped
        # methods that are not, as CompressedTensorsLinearTransformMethod does.
        if self.quant_method.__class__.__name__ not in WEIGHT_LOADER_V2_SUPPORTED:
            extra_weight_attrs["weight_loader"] = layer.weight_loader
        self.quant_method.create_weights(
            layer,
            input_size_per_partition,
            output_partition_sizes,
            input_size,
            output_size,
            params_dtype,
            **extra_weight_attrs,
        )
        if not self.rotate_weight:
            input_rotation = torch.nn.Parameter(
                torch.zeros(self.size, self.size, dtype=torch.int8),
                requires_grad=False,
            )
            set_weight_attrs(
                input_rotation, {"weight_loader": self._load_input_rotation}
            )
            layer.register_parameter("input_rotation", input_rotation)

    def _load_input_rotation(
        self, param: torch.nn.Parameter, loaded_weight: torch.Tensor, *args, **kwargs
    ) -> None:
        """Every shard of a fused layer carries the same rotation, and it must
        be the Sylvester matrix `apply` rotates by."""
        hadamard = hadamard_matrix(self.size, loaded_weight.device)
        if not torch.equal(loaded_weight.float(), (hadamard * self.size**0.5).round()):
            raise NotImplementedError(
                "Only the Sylvester Hadamard input rotation is supported; random "
                "or trained rotations are not."
            )
        param.data.copy_(loaded_weight)

    def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
        # Online methods may be called again after quantizing; rotate once.
        weight: torch.Tensor = layer.weight
        if not getattr(layer, "_already_called_process_weights_after_loading", False):
            if self.rotate_weight:
                hadamard = hadamard_matrix(self.size, weight.device)
                weight.data.copy_(block_hadamard_rotate(weight.data, hadamard))
            else:
                input_rotation: torch.Tensor = layer.input_rotation
                if not input_rotation.any():
                    raise ValueError(
                        "The checkpoint did not provide the input_rotation of an "
                        "online-rotated layer."
                    )
                del layer.input_rotation
        self.quant_method.process_weights_after_loading(layer)
        layer.input_hadamard_fused = self.quant_method.fuse_input_hadamard(
            layer, self.size
        )
        if layer.input_hadamard_fused:
            logger.info_once(
                "%s rotates its input by the %d-wide Hadamard matrix itself.",
                type(self.quant_method).__name__,
                self.size,
            )
        else:
            layer.input_hadamard = hadamard_matrix(self.size, layer.weight.device)

    def apply(
        self,
        layer: torch.nn.Module,
        x: torch.Tensor,
        bias: torch.Tensor | None = None,
    ) -> torch.Tensor:
        if not layer.input_hadamard_fused:
            hadamard: torch.Tensor = layer.input_hadamard
            x = block_hadamard_rotate(x, hadamard)
        return self.quant_method.apply(layer, x, bias)

_load_input_rotation(param, loaded_weight, *args, **kwargs)

Every shard of a fused layer carries the same rotation, and it must be the Sylvester matrix apply rotates by.

Source code in vllm/model_executor/layers/quantization/input_hadamard.py 
113
114
115
116
117
118
119
120
121
122
123
124
def _load_input_rotation(
    self, param: torch.nn.Parameter, loaded_weight: torch.Tensor, *args, **kwargs
) -> None:
    """Every shard of a fused layer carries the same rotation, and it must
    be the Sylvester matrix `apply` rotates by."""
    hadamard = hadamard_matrix(self.size, loaded_weight.device)
    if not torch.equal(loaded_weight.float(), (hadamard * self.size**0.5).round()):
        raise NotImplementedError(
            "Only the Sylvester Hadamard input rotation is supported; random "
            "or trained rotations are not."
        )
    param.data.copy_(loaded_weight)

block_hadamard_rotate(x, hadamard)

Rotate each contiguous hadamard.shape[0]-wide block of the last dimension of x by hadamard, computed in fp32 and returned in x.dtype.

Source code in vllm/model_executor/layers/quantization/input_hadamard.py 
39
40
41
42
43
44
45
def block_hadamard_rotate(x: torch.Tensor, hadamard: torch.Tensor) -> torch.Tensor:
    """Rotate each contiguous ``hadamard.shape[0]``-wide block of the last
    dimension of ``x`` by ``hadamard``, computed in fp32 and returned in
    ``x.dtype``."""
    size = hadamard.shape[0]
    rotated = x.float().unflatten(-1, (-1, size)) @ hadamard
    return rotated.flatten(-2).to(x.dtype)

hadamard_matrix(size, device) cached

Orthonormal Sylvester Hadamard matrix, scipy.linalg.hadamard(size) scaled by 1 / sqrt(size), in fp32. size must be a power of two.

Source code in vllm/model_executor/layers/quantization/input_hadamard.py 
27
28
29
30
31
32
33
34
35
36
@functools.cache
def hadamard_matrix(size: int, device: torch.device) -> torch.Tensor:
    """Orthonormal Sylvester Hadamard matrix, ``scipy.linalg.hadamard(size)``
    scaled by ``1 / sqrt(size)``, in fp32. ``size`` must be a power of two."""
    if size <= 0 or size & (size - 1):
        raise ValueError(f"Hadamard size must be a power of two, got {size}")
    h = torch.ones(1, 1, dtype=torch.float32, device=device)
    while h.shape[0] < size:
        h = torch.cat([torch.cat([h, h], 1), torch.cat([h, -h], 1)], 0)
    return h / size**0.5