`.typings/mlx/nn/layers/positional_encoding.pyi`

81 lines
"""
This type stub file was generated by pyright.
"""

from typing import Optional

import mlx.core as mx
from base import Module

class RoPE(Module):
    """Implements the rotary positional encoding.

    The traditional implementation rotates consecutive pairs of elements in the
    feature dimension while the default implementation rotates pairs with
    stride half the feature dimensions for efficiency.

    For more details see `RoFormer: Enhanced Transformer with Rotary Position
    Embedding <https://arxiv.org/abs/2104.09864>`_.

    Args:
        dims (int): The feature dimensions to be rotated. If the input feature
            is larger than dims then the rest is left unchanged.
        traditional (bool, optional): If set to ``True`` choose the traditional
            implementation which is slightly less efficient. Default: ``False``.
        base (float, optional): The base used to compute angular frequency for
            each dimension in the positional encodings. Default: ``10000``.
        scale (float, optional): The scale used to scale the positions. Default: ``1.0``.
    """
    def __init__(
        self, dims: int, traditional: bool = ..., base: float = ..., scale: float = ...
    ) -> None: ...
    def __call__(self, x, offset: int = ...) -> mx.array: ...

class SinusoidalPositionalEncoding(Module):
    r"""Implements sinusoidal positional encoding.

    For more details see the paper `Attention Is All You Need
    <https://arxiv.org/abs/1706.03762>`_.

    Args:
        dims (int): The dimensionality of the resulting positional embeddings.
        min_freq (float, optional): The minimum frequency expected. Default:
            ``0.0001``.
        max_freq (float, optional): The maximum frequency expected. Default:
            ``1``.
        scale (float, optional): A multiplicative scale for the embeddings.
            Default: ``sqrt(2/dims)``.
        cos_first (bool, optional): If ``True`` embed using ``[cos(x); sin(x)]``
            instead of the reverse. Default: ``False``.
        full_turns (bool, optional): If ``True`` multiply the frequencies with
            :math:`2\pi`. Default: ``False``.
    """
    def __init__(
        self,
        dims: int,
        min_freq: float = ...,
        max_freq: float = ...,
        scale: Optional[float] = ...,
        cos_first: bool = ...,
        full_turns: bool = ...,
    ) -> None: ...
    def __call__(self, x: mx.array) -> mx.array: ...

class ALiBi(Module):
    _alibi_mask_key = ...
    _alibi_mask = ...
    @classmethod
    def create_alibi_matrix(
        cls,
        q_sequence_length: int,
        k_sequence_length: int,
        num_heads: int,
        offset: int,
        dtype=...,
    ) -> mx.array | None: ...
    @staticmethod
    def create_alibi_slope(num_heads: int) -> mx.array: ...
    def __call__(
        self, attention_scores: mx.array, offset=..., mask=...
    ) -> mx.array: ...