from __future__ import annotations import functools import math import operator from typing import Any, TYPE_CHECKING import sympy import torch # NOTE: other files rely on the imports below from torch._dynamo import callback as compilation_callback # noqa: F401 from torch._inductor.runtime.cache_dir_utils import ( # noqa: F401 cache_dir, default_cache_dir, triton_cache_dir, ) if TYPE_CHECKING: from collections.abc import Hashable from .triton_compat import Config def conditional_product(*args: int) -> int: return functools.reduce(operator.mul, [x for x in args if x]) def ceildiv(number: int, denom: int) -> int: return -(number // -denom) def is_power_of_2(n: int) -> bool: """Returns whether n = 2 ** m for some integer m.""" return n > 0 and n & n - 1 == 0 def next_power_of_2(n: int) -> int: """Return the smallest power of 2 greater than or equal to n""" if isinstance(n, sympy.Integer): n = int(n) if n <= 0: return 1 return 1 << (n - 1).bit_length() def last_power_of_2(n: int) -> int: """Return the largest power of 2 less than or equal to n""" next_pow2 = next_power_of_2(n) return next_pow2 // 2 if next_pow2 > n else next_pow2 def get_num_bytes(*args: torch.Tensor, num_in_out_args: int = 0) -> int: """ Return the total number of bytes the arguments of tensor type takes. For in/out args, tensor sizes are counted twice: once for reading and once for writing. The first num_in_out_args arguments are in out tensors. """ return sum( arg.numel() * arg.element_size() * (1 + int(i < num_in_out_args)) for i, arg in enumerate(args) if isinstance(arg, torch.Tensor) ) def triton_config_to_hashable(cfg: Config) -> Hashable: """ Convert triton config to a tuple that can uniquely identify it. We can use the return value as a dictionary key. """ # pyrefly: ignore [missing-attribute] items = sorted(cfg.kwargs.items()) # pyrefly: ignore [missing-attribute] items.append(("num_warps", cfg.num_warps)) # pyrefly: ignore [missing-attribute] items.append(("num_stages", cfg.num_stages)) return tuple(items) def validate_triton_config(cfg: Config) -> None: # [Note: Triton pre_hook in inductor] # pre-hook is a lambda function, which we don't attempt to serialize. # right now, if a pre-hook is attached to the config, it will not be saved; # and then it won't be used when the config is loaded from cache. # So we assert - if we do get a pre_hook, it might get ignored after caching. assert getattr(cfg, "pre_hook", None) is None, ( "triton configs with pre_hooks not supported" ) def create_bandwidth_info_str( ms: float, num_gb: float, gb_per_s: float, prefix: str = "", suffix: str = "", color: bool = True, ) -> str: info_str = f"{prefix}{ms:.3f}ms \t{num_gb:.3f} GB \t {gb_per_s:7.2f}GB/s{suffix}" slow = ms > 0.012 and gb_per_s < 650 return red_text(info_str) if color and slow else info_str def get_max_y_grid() -> int: return 65535 try: import colorama HAS_COLORAMA = True except ModuleNotFoundError: HAS_COLORAMA = False colorama = None # type: ignore[assignment] if HAS_COLORAMA: def _color_text(msg: str, color: str) -> str: # pyrefly: ignore [missing-attribute] return getattr(colorama.Fore, color.upper()) + msg + colorama.Fore.RESET else: def _color_text(msg: str, color: str) -> str: return msg def green_text(msg: str) -> str: return _color_text(msg, "green") def yellow_text(msg: str) -> str: return _color_text(msg, "yellow") def red_text(msg: str) -> str: return _color_text(msg, "red") def blue_text(msg: str) -> str: return _color_text(msg, "blue") def get_first_attr(obj: Any, *attrs: str) -> Any: """ Return the first available attribute or throw an exception if none is present. """ for attr in attrs: if hasattr(obj, attr): return getattr(obj, attr) raise AssertionError(f"{obj} does not has any of the attributes: {attrs}") dynamo_timed = torch._dynamo.utils.dynamo_timed # type: ignore[has-type] def triton_hash_to_path_key(key: str) -> str: # In early versions of Triton, the hash is directly used in the path name. # Later, the hash is converted to base64 before being used in the path name. # Later, the base64 conversion was replaced to the base32 # # This code tries to import _base64 and falls back to _base32 if _base64 is unavailable. # # To handle this, try to import the to-base64-conversion function. # If it exists, use it; otherwise, try using _base32; if both are unavailable, use the hash directly. try: from triton.runtime.cache import _base64 return _base64(key) except Exception: try: from triton.runtime.cache import _base32 return _base32(key) except Exception: return key def compile_mps_shader(source: str) -> Any: """ Compiles shader source but raise more actionable error message when needed """ try: return torch.mps.compile_shader(source) except SyntaxError as err: raise SyntaxError(f"failed to compile {source} with {err.msg}") from err def torch_dtype_to_jax_runtime(dtype: torch.dtype) -> Any: """ Map PyTorch dtype to actual JAX dtype object at runtime. This helper is used in generated Pallas kernels at runtime to convert PyTorch dtypes to JAX dtype objects (not string representations). Args: dtype: PyTorch dtype to convert Returns: JAX dtype object (e.g., jnp.float32 object itself) """ import jax.numpy as jnp # pyrefly: ignore [import-error, missing-import] dtype_map = { torch.float32: jnp.float32, torch.float64: jnp.float64, torch.float16: jnp.float16, torch.bfloat16: jnp.bfloat16, torch.int32: jnp.int32, torch.int64: jnp.int64, torch.int16: jnp.int16, torch.int8: jnp.int8, torch.uint8: jnp.uint8, torch.bool: jnp.bool_, torch.complex64: jnp.complex64, torch.complex128: jnp.complex128, } if dtype not in dtype_map: raise ValueError(f"Unsupported dtype for JAX conversion: {dtype}") return dtype_map[dtype] def torch_dtype_to_jax(dtype: torch.dtype) -> str: """ Map PyTorch dtype to JAX dtype expression string. This helper is used at compile time in codegen to generate JAX dtype expressions for Pallas kernels. Args: dtype: PyTorch dtype to convert Returns: JAX dtype expression as string (e.g., "jnp.float32") """ jax_dtype = torch_dtype_to_jax_runtime(dtype) dtype_name = jax_dtype.__name__ if dtype_name == "bool": dtype_name = "bool_" return f"jnp.{dtype_name}" def pallas_partial_reduce(reduce_fn: Any, v: Any, pw_numel: int, red_numel: int) -> Any: """ Helper for partial reductions in Pallas kernels. Reorders axes and reduces, returning result with keepdims-style shape for proper in-kernel broadcasting. Args: reduce_fn: The reduction function to apply (e.g., jnp.sum, jnp.max) v: The input array to reduce pw_numel: The number of pointwise elements red_numel: The number of reduction elements Returns: Reduced array with keepdims-style shape """ import jax.numpy as jnp # pyrefly: ignore [import-error, missing-import] shape = tuple(v.shape) # Find contiguous axes whose product = red_numel (search from right) red_axes = None for i in range(len(shape) - 1, -1, -1): prod = 1 for j in range(i, -1, -1): prod *= shape[j] if prod == red_numel: red_axes = list(range(j, i + 1)) break if red_axes is not None: break if red_axes is None: red_axes = [len(shape) - 1] # Build output shape with 1s for reduced dimensions (keepdims style) out_shape = tuple(1 if i in red_axes else s for i, s in enumerate(shape)) # Move pointwise axes to front, reduction axes to back pw_axes = [i for i in range(len(shape)) if i not in red_axes] reordered = jnp.moveaxis(v, pw_axes, list(range(len(pw_axes)))) result = reduce_fn(reordered.reshape(pw_numel, red_numel), axis=-1) return result.reshape(out_shape) def pallas_gpu_pad_inputs(inputs: list[Any], alignment: int = 128) -> list[Any]: """Flatten and pad each input JAX array to a multiple of alignment.""" import jax.numpy as jnp # pyrefly: ignore [import-error, missing-import] padded = [] for inp in inputs: flat = inp.flatten() orig_size = flat.size aligned_size = ((orig_size + alignment - 1) // alignment) * alignment if orig_size != aligned_size: padded.append(jnp.pad(flat, (0, aligned_size - orig_size))) else: padded.append(flat) return padded def pallas_gpu_align_output_specs( out_shapes: tuple[Any, ...], out_dtypes: tuple[Any, ...], alignment: int = 128, ) -> tuple[tuple[Any, ...], list[bool]]: """Build aligned output ShapeDtypeStruct specs for GPU kernels. Returns (aligned_specs, is_scalar_output) where is_scalar_output[i] is True when the i-th output is scalar and should not be padded/unpadded. """ import jax # pyrefly: ignore [import-error, missing-import] aligned_specs = [] is_scalar = [] for shape, dtype in zip(out_shapes, out_dtypes): numel = math.prod(shape) if numel <= 1: aligned_specs.append(jax.ShapeDtypeStruct(shape, dtype)) is_scalar.append(True) else: aligned_numel = ((numel + alignment - 1) // alignment) * alignment aligned_specs.append(jax.ShapeDtypeStruct((aligned_numel,), dtype)) is_scalar.append(False) return tuple(aligned_specs), is_scalar def pallas_gpu_unpad_results( results: Any, orig_shapes: tuple[Any, ...], is_scalar_output: list[bool] | None = None, ) -> Any: """Remove padding from GPU kernel results and reshape to original shapes. If is_scalar_output is None, all outputs are treated as non-scalar. """ if not isinstance(results, tuple): results = (results,) unpadded = [] for i, (res, shape) in enumerate(zip(results, orig_shapes)): if is_scalar_output is not None and is_scalar_output[i]: unpadded.append(res) else: orig_numel = math.prod(shape) unpadded.append(res[:orig_numel].reshape(shape)) return unpadded[0] if len(unpadded) == 1 else tuple(unpadded)