← back to Atmos Game

assets/gen/.venv/lib/python3.12/site-packages/fsspec/__pycache__/caching.cpython-312.pyc

362 lines

�

c�bj��	�l�UddlmZddlZddlZddlZddlZddlZddlZddlmZddl	m
Z
ddlmZm
Z
ddlmZddlmZddlmZmZmZmZmZmZerddlZdd	lmZed
�Zned
�Zed�Zej<d�Ze
e e ge!fZ"e
e#e e fge!fZ$Gd
�d�Z%Gd�de%�Z&Gd�de%�Z'Gd�de%�Z(Gd�de%�Z)Gd�de%�Z*Gd�de%�Z+Gd�de%�Z,Gd�deeef�Z-Gd�d e%�Z.de%iZ/d!e0d"<d$d%d#�Z1e%e&e*e'e)e(e+e,e.f	D]
Z2e1e2��y)&�)�annotationsN)�OrderedDict)�Callable)�Future�ThreadPoolExecutor)�groupby)�
itemgetter)�
TYPE_CHECKING�Any�ClassVar�Generic�
NamedTuple�TypeVar)�	ParamSpec�P�Tzfsspec.cachingc�H�eZdZUdZdZded<dd�Zdd�Zd
d�Zdd�Z	dd	�Z
y
)�	BaseCacheagPass-though cache: doesn't keep anything, calls every time

    Acts as base class for other cachers

    Parameters
    ----------
    blocksize: int
        How far to read ahead in numbers of bytes
    fetcher: func
        Function of the form f(start, end) which gets bytes from remote as
        specified
    size: int
        How big this file is
    �none�
ClassVar[str]�namec�f�||_d|_||_||_d|_d|_d|_y�Nr)�	blocksize�nblocks�fetcher�size�	hit_count�
miss_count�total_requested_bytes)�selfrrrs    �e/Users/macstudio3/Projects/atmos-game/assets/gen/.venv/lib/python3.12/site-packages/fsspec/caching.py�__init__zBaseCache.__init__4s4��"�����������	�������%&��"�c�t�|�d}|�|j}||jk\s||k\ry|j||�S)Nrr$)rr�r!�start�stops   r"�_fetchzBaseCache._fetch>s@���=��E��<��9�9�D��D�I�I���$����|�|�E�4�(�(r$c�.�d|_d|_d|_y)zAReset hit and miss counts for a more ganular report e.g. by file.rN)rrr �r!s r"�_reset_statszBaseCache._reset_statsGs��������%&��"r$c	��|jdk(r|jdk(ryd|j�d|j�d|j�d|j�d�	S)z2Return a formatted string of the cache statistics.r�z , z: z hits, z	 misses, z total requested bytes)rrrr r+s r"�
_log_statszBaseCache._log_statsMs^���>�>�Q��4�?�?�a�#7���T�Y�Y�K�r�$�.�.�!1�����8I��SW�Sm�Sm�Rn�oE�F�	Fr$c���d|jj�d|j�d|j�d|j�d|j
�d|j�d|j�d�S)	Nz

        <z:
            block size  :   z
            block count :   z
            file size   :   z
            cache hits  :   z
            cache misses:   z$
            total requested bytes: z
>
        )�	__class__�__name__rrrrrr r+s r"�__repr__zBaseCache.__repr__Ts���
�
�.�.�
!�
!�	"�#�!�^�^�,�-�!�\�\�N�+�!�Y�Y�K�(�!�^�^�,�-�!�_�_�-�.$�$(�$>�$>�#?�@	��	r$N�r�intr�Fetcherrr5�return�None�r'�
int | Noner(r:r7�bytes�r7r8)r7�str)r2�
__module__�__qualname__�__doc__r�__annotations__r#r)r,r/r3�r$r"rr"s-��
�!�D�-� �'�)�'�F�
r$rc�j��eZdZdZdZ			d													d	�fd�
Zd
d�Zdd�Zdd�Zd
d�Z	�xZ
S)�	MMapCachea�memory-mapped sparse file cache

    Opens temporary file, which is filled blocks-wise when data is requested.
    Ensure there is enough disc space in the temporary location.

    This cache method might only work on posix

    Parameters
    ----------
    blocksize: int
        How far to read ahead in numbers of bytes
    fetcher: Fetcher
        Function of the form f(start, end) which gets bytes from remote as
        specified
    size: int
        How big this file is
    location: str
        Where to create the temporary file. If None, a temporary file is
        created using tempfile.TemporaryFile().
    blocks: set[int]
        Set of block numbers that have already been fetched. If None, an empty
        set is created.
    multi_fetcher: MultiFetcher
        Function of the form f([(start, end)]) which gets bytes from remote
        as specified. This function is used to fetch multiple blocks at once.
        If not specified, the fetcher function is used instead.
    �mmapc���t�|�|||�|�
t�n||_||_||_|j
�|_y�N)�superr#�set�blocks�location�
multi_fetcher�	_makefile�cache)r!rrrrKrJrLr1s       �r"r#zMMapCache.__init__�sB���	����G�T�2�%�~�c�e�6��� ��
�*����^�^�%��
r$c�4�ddl}ddl}|jdk(r
t�S|j�)t
jj|j�s�|j� |j�}t�|_
nt|jd�}|j|jdz
�|jd�|j�nt|jd�}|j|j�|j�S)Nrzwb+��1zr+b)rE�tempfiler�	bytearrayrK�os�path�exists�
TemporaryFilerIrJ�open�seek�write�flush�fileno)r!rErR�fds    r"rMzMMapCache._makefile�s������9�9��>��;���=�=� ������t�}�}�(E��}�}�$��+�+�-��!�e����$�-�-��/���G�G�D�I�I��M�"��H�H�T�N��H�H�J��d�m�m�U�+�B��t�y�y�����d�i�i�0�0r$c� ��tjd|�d|���|�d}|��j}|�jk\s||k\ry|�jz}|�jz}t	||dz�}�fd�|D�}�xj
t
�fd�|D��z
c_g}tt|�d��	�D]�\}}	tttd�|	��}	|	d�jz}
t|	d
�jz�jz�j�}�xj||
z
z
c_
tjd|	d�d|	d
�d|
�d|�d
�	�|j|
|f��jj!|	��xj"t%|	�z
c_��|s�j&||S�j(rmtjd|���t�j)|��D]7\}}
||\}
}tjd|
�d|���|
�j&|
|�9nD|D]?\}
}tjd|
�d|����j+|
|��j&|
|�A�j&||S)NzMMap cache fetching �-rr$rPc3�@�K�|]}|�jvs�|���y�wrG�rJ��.0�ir!s  �r"�	<genexpr>z#MMapCache._fetch.<locals>.<genexpr>�s�����?�;�a�!�4�;�;�*>��;����c3�@�K�|]}|�jvs�d���y�w)rPNrarbs  �r"rez#MMapCache._fetch.<locals>.<genexpr>�s�����I��A��T�[�[�8H�a��rfc��|d|dz
S)NrrPrB)�xs r"�<lambda>z"MMapCache._fetch.<locals>.<lambda>�s���1���!��r$)�key�����zMMap get blocks z (�)zMMap copy block (zMMap get block ()�logger�debugrr�ranger�sumr�	enumerate�tuple�mapr	�minr �appendrJ�updater�lenrNrLr)r!r'�end�start_block�	end_block�block_range�need�ranges�_�_blocks�sstart�send�idx�rs`             r"r)zMMapCache._fetch�sj������+�E�7�!�C�5�9�:��=��E��;��)�)�C��D�I�I���#����t�~�~�-���4�>�>�)�	��K��Q��7��?�;�?�����#�I��I�I�I����"�)�D�/�7L�M�J�A�w��C�
�1�
�w�7�8�G��Q�Z�$�.�.�0�F��w�r�{�T�^�^�3�d�n�n�D�d�i�i�P�D�
�&�&�$��-�7�&��L�L�"�7�1�:�,�a����}�B�v�h�a��v�Q�O�
�
�M�M�6�4�.�)�
�K�K���w�'��O�O�s�7�|�+�O�%N�(��:�:�e�C�(�(�����L�L�+�F�8�4�5�#�D�$6�$6�v�$>�?���Q�%�c�{�������0����$��@�A�*+��
�
�6�$�'�@�
!'�������/��x�q���?�@�*.�,�,�v�t�*D��
�
�6�$�'�!'��z�z�%��$�$r$c�@�|jj�}|d=|S)NrN)�__dict__�copy�r!�states  r"�__getstate__zMMapCache.__getstate__�s ���
�
�"�"�$���'�N��r$c�d�|jj|�|j�|_yrG)r�rwrMrNr�s  r"�__setstate__zMMapCache.__setstate__�s"���
�
���U�#��^�^�%��
r$)NNN)rr5rr6rr5rKz
str | NonerJzset[int] | NonerLzMultiFetcher | Noner7r8)r7zmmap.mmap | bytearray�r'r:ryr:r7r;�r7�dict[str, Any]�r�r�r7r8)r2r>r?r@rr#rMr)r�r��
__classcell__�r1s@r"rDrDasx����8�D� $�"&�-1�
&��
&��
&��	
&�
�
&� �

&�+�
&�
�
&�1�,<%�|�&r$rDc�0��eZdZdZdZd�fd�Zdd�Z�xZS)�ReadAheadCachea!Cache which reads only when we get beyond a block of data

    This is a much simpler version of BytesCache, and does not attempt to
    fill holes in the cache or keep fragments alive. It is best suited to
    many small reads in a sequential order (e.g., reading lines from a file).
    �	readaheadc�R��t�|�|||�d|_d|_d|_y)Nr$r)rHr#rNr'ry�r!rrrr1s    �r"r#zReadAheadCache.__init__�s)���
����G�T�2���
���
���r$c�^�|�d}|�||jkDr|j}||jk\s||k\ry||z
}||jk\rM||jkr>|xjdz
c_|j||jz
||jz
S|j|cxkr|jkrOnnL|xj
dz
c_|j||jz
d}|t
|�z}|j}n|xj
dz
c_d}t|j||jz�}|xj||z
z
c_	|j||�|_||_|jt
|j�z|_||jd|zS�Nrr$rP)rr'ryrrNrrxrurr r)r!r'ry�l�parts     r"r)zReadAheadCache._fetch�se���=��E��;�#��	�	�/��)�)�C��D�I�I���#����%�K���D�J�J��3�$�(�(�?��N�N�a��N��:�:�e�d�j�j�0�3����3C�D�D�
�Z�Z�5�
+�4�8�8�
+��O�O�q� �O��:�:�e�d�j�j�0�2�3�D�
��T��N�A��H�H�E�
�O�O�q� �O��D��$�)�)�S�4�>�>�1�2���"�"�c�E�k�1�"��\�\�%��-��
���
��:�:��D�J�J��/����d�j�j��!�n�$�$r$r4r��r2r>r?r@rr#r)r�r�s@r"r�r��s�����D��%r$r�c�0��eZdZdZdZd�fd�Zdd�Z�xZS)�FirstChunkCachez�Caches the first block of a file only

    This may be useful for file types where the metadata is stored in the header,
    but is randomly accessed.
    �firstc�D��||kDr|}t�|�|||�d|_yrG)rHr#rNr�s    �r"r#zFirstChunkCache.__init__&s(����t���I�
����G�T�2�#'��
r$c�n�|xsd}||jkDrtjd�yt||j�}||jk�r&|j
��|xjdz
c_||jkDr@|xj|z
c_|jd|�}|d|j|_||dS|jd|j�|_|xj|jz
c_|j
||}||jkDrA|xj||jz
z
c_||j|j|�z
}|xjdz
c_	|S|xjdz
c_|xj||z
z
c_|j||�S)Nrz,FirstChunkCache: requested start > file sizer$rP)
rrnrorurrNrr rr)r!r'ry�datar�s     r"r)zFirstChunkCache._fetch-sh���
����4�9�9���L�L�G�H���#�t�y�y�!���4�>�>�!��z�z�!����1�$������'��.�.�#�5�.��<�<��3�/�D�!%�&6����!7�D�J����<�'�!�\�\�!�T�^�^�<��
��*�*�d�n�n�<�*��:�:�e�C�(�D��T�^�^�#��*�*�c�D�N�N�.B�B�*�����T�^�^�S�9�9���N�N�a��N��K��O�O�q� �O��&�&�#��+�5�&��<�<��s�+�+r$r4r�r�r�s@r"r�r�s�����D�(�,r$r�c���eZdZdZdZ	d
									d�fd�
Zd�Zdd�Zd
d�Zdd�Z	d�fd�Z
										dd	�Z�xZS)�
BlockCachea�
    Cache holding memory as a set of blocks.

    Requests are only ever made ``blocksize`` at a time, and are
    stored in an LRU cache. The least recently accessed block is
    discarded when more than ``maxblocks`` are stored.

    Parameters
    ----------
    blocksize : int
        The number of bytes to store in each block.
        Requests are only ever made for ``blocksize``, so this
        should balance the overhead of making a request against
        the granularity of the blocks.
    fetcher : Callable
    size : int
        The total size of the file being cached.
    maxblocks : int
        The maximum number of blocks to cache for. The maximum memory
        use for this cache is then ``blocksize * maxblocks``.
    �
blockcachec����t�|�|||�tj||z�|_||_t
j|�|j�|_	yrG)
rHr#�math�ceilr�	maxblocks�	functools�	lru_cache�_fetch_block�_fetch_block_cached�r!rrrr�r1s     �r"r#zBlockCache.__init__dsR���	����G�T�2��y�y��	�!1�2���"���#A�9�#6�#6�y�#A�$�BS�BS�#T�� r$c�6�|jj�S�z�
        The statistics on the block cache.

        Returns
        -------
        NamedTuple
            Returned directly from the LRU Cache used internally.
        �r��
cache_infor+s r"r�zBlockCache.cache_infol����'�'�2�2�4�4r$c�$�|j}|d=|S)Nr��r�r�s  r"r�zBlockCache.__getstate__ws���
�
���'�(��r$c��|jj|�tj|d�|j�|_y)Nr�)r�rwr�r�r�r�r�s  r"r�zBlockCache.__setstate__|s<���
�
���U�#�#J�9�#6�#6�u�[�7I�#J����$
�� r$c��|�d}|�|j}||jk\s||k\ry|j||||jz|dz
|jz�Sr�)r�_read_cacher)r!r'rys   r"r)zBlockCache._fetch�sa���=��E��;��)�)�C��D�I�I���#�������3�����/�#��'�d�n�n�1L�
�	
r$c�@��||jkDrtd|�d|j�d���||jz}||jz}|xj||z
z
c_|xjdz
c_t
j
d|�t�|�!||�}|S)�=
        Fetch the block of data for `block_number`.
        �'block_number=�(' is greater than the number of blocks (rmrPzBlockCache fetching block %d)	r�
ValueErrorrr rrn�inforHr))r!�block_numberr'ry�block_contentsr1s     �r"r�zBlockCache._fetch_block�s�����$�,�,�&�� ���/)�)-����a�9��
�
�t�~�~�-���d�n�n�$���"�"�c�E�k�1�"����1������2�L�A�����s�3���r$c	��||jz}||jz}|dk(r|j}|xjdz
c_||k(r|j|�}|||S|j|�|dg}|jt	|jt|dz|���|j
|j|�d|�dj|�S)��
        Read from our block cache.

        Parameters
        ----------
        start, end : int
            The start and end byte positions.
        start_block_number, end_block_number : int
            The start and end block numbers.
        rrPNr$�rrr��extendrtrprv�join�	r!r'ry�start_block_number�end_block_number�	start_pos�end_pos�block�outs	         r"r�zBlockCache._read_cache�s����D�N�N�*�	�����&���a�<��n�n�G����!����!1�1��3�3�4F�G�E���7�+�+��+�+�,>�?�	�
�K�L�C�

�J�J���,�,��,�q�0�2B�C��
�
�J�J�t�/�/�0@�A�(�7�K�L��8�8�C�=� r$�� �
rr5rr6rr5r�r5r7r8r�r�r�)r�r5r7r;�
r'r5ryr5r�r5r�r5r7r;)
r2r>r?r@rr#r�r�r�r)r�r�r�r�s@r"r�r�Ks�����,�D�MO�U��U�'.�U�69�U�FI�U�	
�U�	5��

�

��$(!��(!�"�(!�8;�(!�OR�(!�	�(!r$r�c�Z��eZdZUdZdZded<	d									d	�fd�
Zd
d�Zdd�Z�xZ	S)�
BytesCacheaKCache which holds data in a in-memory bytes object

    Implements read-ahead by the block size, for semi-random reads progressing
    through the file.

    Parameters
    ----------
    trim: bool
        As we read more data, whether to discard the start of the buffer when
        we are more than a blocksize ahead of it.
    r;rrc�`��t�|�|||�d|_d|_d|_||_y)Nr$)rHr#rNr'ry�trim)r!rrrr�r1s     �r"r#zBytesCache.__init__�s2���	����G�T�2���
�!%��
�#�����	r$c���|�d}|�|j}||jk\s||k\ry|j�c||jk\rT|j�H||jkr9||jz
}|xjdz
c_|j|||z|z
S|j
r$t
|j||j
z�}n|}||k(s||jkDry|j�||jkrh|j�||jkDrM|xj||z
z
c_|xjdz
c_|j||�|_||_�n�|j�J�|j�J�|xjdz
c_||jkr�|j�|j|z
|j
kDr8|xj||z
z
c_|j||�|_||_�n4|xj|j|z
z
c_|j||j�}||_||jz|_n�|j��||jkDr�|j|jkDrn�||jz
|j
kDr7|xj||z
z
c_|j||�|_||_nR|xj||jz
z
c_|j|j|�}|j|z|_|jt|j�z|_||jz
}|j|||z|z
}|jrq|j|jz
|j
dzz}|dkDrC|xj|j
|zz
c_|j|j
|zd|_|Sr�)rr'ryrrNrrur rrrxr�)r!r'ry�offset�bend�newr��nums        r"r)zBytesCache._fetch�sJ���=��E��;��)�)�C��D�I�I���#����J�J�"�����#����$��d�h�h���T�Z�Z�'�F��N�N�a��N��:�:�f�v��|�e�';�<�<��>�>��t�y�y�#����"6�7�D��D��5�=�E�D�I�I�-���J�J��%�$�*�*�"4��H�H���d�h�h��
�&�&�$��,�6�&��O�O�q� �O����e�T�2�D�J��D�J��:�:�)�)�)��8�8�'�'�'��O�O�q� �O��t�z�z�!��8�8�#�t�x�x�#�~����'F��.�.�$��,�>�.�!%���e�T�!:�D�J�!&�D�J��.�.�$�*�*�u�2D�D�.��,�,�u�d�j�j�9�C�!&�D�J�!$�t�z�z�!1�D�J����%�$����/��8�8�d�i�i�'���4�8�8�^�d�n�n�4��.�.�$��,�>�.�!%���e�T�!:�D�J�!&�D�J��.�.�$����/�A�.��,�,�t�x�x��6�C�!%���c�!1�D�J��:�:��D�J�J��/�������#���j�j��&�3�,��"6�7���9�9��8�8�d�j�j�(�d�n�n�q�.@�A�C��Q�w��
�
�d�n�n�s�2�2�
�!�Z�Z�����(<�(>�?��
��
r$c�,�t|j�SrG)rxrNr+s r"�__len__zBytesCache.__len__,s���4�:�:��r$)T)
rr5rr6rr5r��boolr7r8r�)r7r5)
r2r>r?r@rrAr#r)r�r�r�s@r"r�r��sT���
�"�D�-�!�IM����'.��69��AE��	
��G�Rr$r�c�X��eZdZUdZdZded<				d									d�fd�
Zd	d�Z�xZS)
�AllBytesz!Cache entire contents of the file�allrrc����t�|�|||�|�P|xjdz
c_|xj|jz
c_|jd|j�}||_y)NrPr)rHr#rr rrr�)r!rrrr�r1s     �r"r#zAllBytes.__init__5sY���	����G�T�2��<��O�O�q� �O��&�&�$�)�)�3�&��<�<��4�9�9�-�D���	r$c�J�|xjdz
c_|j||S)NrP)rr�r&s   r"r)zAllBytes._fetchCs!�����!����y�y��t�$�$r$)NNNN)
rr:rzFetcher | Nonerr:r�zbytes | Noner7r8r9)	r2r>r?r@rrAr#r)r�r�s@r"r�r�0sX���+��D�-��!%�"&��!���� ���	�
��
�
�%r$r�c����eZdZUdZdZded<		d											d�fd�
Zed��Zejd��Zed��Z
e
jd	��Z
d
d
�Z�xZS)�KnownPartsOfAFilea�
    Cache holding known file parts.

    Parameters
    ----------
    blocksize: int
        How far to read ahead in numbers of bytes
    fetcher: func
        Function of the form f(start, end) which gets bytes from remote as
        specified
    size: int
        How big this file is
    data: dict
        A dictionary mapping explicit `(start, stop)` file-offset tuples
        with known bytes.
    strict: bool, default True
        Whether to fetch reads that go beyond a known byte-range boundary.
        If `False`, any read that ends outside a known part will be zero
        padded. Note that zero padding will not be used for reads that
        begin outside a known byte-range.
    �partsrrc����t�|�|||�||_|r�t|j	��}|dg}|j|d�g}	|ddD]m\}
}|d\}}
|
|
k(r&||f|d<|	dxx|j|
|f�z
cc<�9|j
|
|f�|	j
|j|
|f���ott||	��|_	yi|_	y)NrrPrl)
rHr#�strict�sorted�keys�poprv�dict�zipr�)r!rrrr�r�r�old_offsets�offsetsrJr'r(�start0�stop0r1s              �r"r#zKnownPartsOfAFile.__init__as����	����G�T�2����� �����-�K�"�1�~�&�G��h�h�{�1�~�.�/�F�*�1�2����t� '���
����E�>�#)�4�.�G�B�K��2�J�$�(�(�E�4�=�"9�9�J��N�N�E�4�=�1��M�M�$�(�(�E�4�=�"9�:� /��S��&�1�2�D�I��D�Ir$c�:�td�|jD��S)Nc3�2K�|]}|d|dz
���y�w)rPrNrB)rcrs  r"rez)KnownPartsOfAFile.size.<locals>.<genexpr>�s����2�	�1�1�Q�4�!�A�$�;�	�s�)rqr�r+s r"rzKnownPartsOfAFile.sizes���2��	�	�2�2�2r$c��yrGrB�r!�values  r"rzKnownPartsOfAFile.size����r$c�,�t|j�SrG)rxr�r+s r"rzKnownPartsOfAFile.nblocks�s���4�9�9�~�r$c��yrGrBr�s  r"rzKnownPartsOfAFile.nblocks�r�r$c���tjd||�|�d}|�|j}|xj||z
z
c_d}d}d}t	|j
�D�]T\}}||cxkr|krGnnD||cxkr|kr9nn6||z
}|xjdz
c_|j
||f|||z|z
cS||krn�|r7||kDr2|xjdz
c_|jrt�|d||z
zz
}||cxkr|kr:nn7|xjdz
c_||z
}|j
||f|||z|z
}d}nu||kr/||kDr*|xjdz
c_||j
||fz
}nA||cxkr|kr6nn3|xjdz
c_||j
||fd||z
z}|cS|}��W|xjdz
c_|r|js
|d||z
zz}|St�)NzKnown parts request %s %srr$FrP�T)
rnrorr r�r�rrr�r�)	r!r'r(r��started�loc_old�loc0�loc1�offs	         r"r)zKnownPartsOfAFile._fetch�s������0�%��>��=��E��<��9�9�D��"�"�d�U�l�2�"������� ����+�J�D�$���$��$�4�4�+?�4�+?��d�l�����!�#���y�y�$���.�s�S�4�Z�%�5G�H�H��t�|���4�'�>����1�$���;�;�$�$��w�$��.�1�1���u�#�t�#����!�#���d�l���i�i��t��-�c�C�$�J��4F�G�������$��+����!�#���t�y�y�$���.�.����%��%����!�#���D�I�I�t�T�l�3�M�d�T�k�B�B���
��G�;,�<	
���1����4�;�;���4�'�>�2�2�C��J��r$)NF)rr5rr6rr5r�z#dict[tuple[int, int], bytes] | Noner�r�rrr9)
r2r>r?r@rrAr#�propertyr�setterrr)r�r�s@r"r�r�Hs�����,"�D�-�!�59��
������	�
2���
���<�3��3�
�[�[�
��
�����
�^�^�
��
�,r$r�c�P�eZdZdZGd�de�Zd
dd�Zdd�Zd
d�Zdd�Z	dd�Z
y	)�UpdatableLRUzh
    Custom implementation of LRU cache that allows updating keys

    Used by BackgroundBlockCache
    c�6�eZdZUded<ded<ded<ded<y)�UpdatableLRU.CacheInfor5�hits�misses�maxsize�currsizeN)r2r>r?rArBr$r"�	CacheInfor�s���	������
r$r	c��tj�|_||_||_d|_d|_tj�|_	yr)
�collectionsr�_cache�_func�	_max_size�_hits�_misses�	threading�Lock�_lock)r!�func�max_sizes   r"r#zUpdatableLRU.__init__�s<��+6�+B�+B�+D�����
�!�����
�����^�^�%��
r$c�^�|rtd|j�����|j5||jvrH|jj	|�|xj
dz
c_|j|cddd�S	ddd�|j|i|��}|j5||j|<|xjdz
c_t|j�|jkDr|jjd��ddd�|S#1swY��xYw#1swY|SxYw)Nz Got unexpected keyword argument rPF��last)�	TypeErrorr�rr�move_to_endrr
rrxr�popitem)r!�args�kwargs�results    r"�__call__zUpdatableLRU.__call__�s�����>�v�{�{�}�o�N�O�O�
�Z�Z��t�{�{�"����'�'��-��
�
�a��
��{�{�4�(�	�Z�"������T�,�V�,��
�Z�Z� &�D�K�K����L�L�A��L��4�;�;��$�.�.�0����#�#��#�/�	��
��Z����
�s�A
D�)A#D"�D�"D,c�b�|j5||jvcddd�S#1swYyxYwrG)rr)r!rs  r"�
is_key_cachedzUpdatableLRU.is_key_cached�s��
�Z�Z��4�;�;�&��Z�Z�s�%�.c���|j5||j|<t|j�|jkDr|jj	d��ddd�y#1swYyxYw)NFr)rrrxrr)r!rrs   r"�add_keyzUpdatableLRU.add_key�sK��
�Z�Z� &�D�K�K����4�;�;��$�.�.�0����#�#��#�/��Z�Z�s�AA$�$A-c���|j5|j|jt|j�|j
|j��cddd�S#1swYyxYw)N)rrrr)rr	rrxrrrr+s r"r�zUpdatableLRU.cache_info�sE��
�Z�Z��>�>�����T�[�[�)��Z�Z��|�|�	"���Z�Z�s�AA�A&N)�)rzCallable[P, T]rr5r7r8)rzP.argsrzP.kwargsr7r)rrr7r�)rrrrr7r8�r7r)r2r>r?r@rr	r#rr!r#r�rBr$r"rr�s,����J��&��&'�0�r$rc���eZdZUdZdZded<	d									d
�fd�
Zdd�Zdd�Zdd�Z	dd	�Z
dd�fd
�
Z										dd�Z�xZ
S)�BackgroundBlockCachea�
    Cache holding memory as a set of blocks with pre-loading of
    the next block in the background.

    Requests are only ever made ``blocksize`` at a time, and are
    stored in an LRU cache. The least recently accessed block is
    discarded when more than ``maxblocks`` are stored. If the
    next block is not in cache, it is loaded in a separate thread
    in non-blocking way.

    Parameters
    ----------
    blocksize : int
        The number of bytes to store in each block.
        Requests are only ever made for ``blocksize``, so this
        should balance the overhead of making a request against
        the granularity of the blocks.
    fetcher : Callable
    size : int
        The total size of the file being cached.
    maxblocks : int
        The maximum number of blocks to cache for. The maximum memory
        use for this cache is then ``blocksize * maxblocks``.
    �
backgroundrrc���t�|�|||�tj||z�|_||_t
|j|�|_td��|_
d|_d|_tj�|_y)NrP��max_workers)rHr#r�r�rr�rr�r�r�_thread_executor�_fetch_future_block_number�
_fetch_futurerr�_fetch_future_lockr�s     �r"r#zBackgroundBlockCache.__init__ss���	����G�T�2��y�y��	�!1�2���"���#/��0A�0A�9�#M�� � 2�q� A���6:��'�37���"+�.�.�"2��r$c�6�|jj�Sr�r�r+s r"r�zBackgroundBlockCache.cache_info#r�r$c�<�|j}|d=|d=|d=|d=|d=|S)Nr�r-r.r/r0r�r�s  r"r�z!BackgroundBlockCache.__getstate__.s<���
�
���'�(��$�%��.�/��/�"��&�'��r$c���|jj|�t|j|d�|_td��|_d|_d|_tj�|_y)Nr�rPr+)r�rwrr�r�rr-r.r/rrr0r�s  r"r�z!BackgroundBlockCache.__setstate__7sZ���
�
���U�#�#/��0A�0A�5��CU�#V�� � 2�q� A���*.��'�!���"+�.�.�"2��r$c��|�d}|�|j}||jk\s||k\ry||jz}||jz}d}d}|j5|j��|j�J�|jj�rbtjd�|jj|jj�|j�d|_d|_nKt||jcxkxr|knc�}|r&|j}|j}d|_d|_ddd�|�?tjd�|jj|j�|�t||dz�D]}|j|��|dz}	|j5|j�]|	|jkrN|jj|	�s3|	|_|jj!|j"|	d�|_ddd�|j%||||��S#1swY��xYw#1swY�+xYw)Nrr$z3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.rP�async)r�r�)rrr0r/r.�donernr�r�r#rr�rprr!r-�submitr�r�)
r!r'ryr�r��fetch_future_block_number�fetch_future�	must_joinr��end_block_plus_1s
          r"r)zBackgroundBlockCache._fetch?sL���=��E��;��)�)�C��D�I�I���#���#�d�n�n�4���$�.�.�0��$(�!���
�
$�
$��!�!�-��6�6�B�B�B��%�%�*�*�,��K�K� U�V��,�,�4�4��*�*�1�1�3�T�5T�5T��7;�D�3�)-�D�&�!%�*��:�:�,�+�,�!�I�
!�59�4S�4S�1�'+�'9�'9��;?��7�-1��*�7%�<�#��K�K�B�C��$�$�,�,��#�#�%�'@�
�
"�"4�6F��6J�K�L��$�$�\�2�L�
,�a�/��
�
$�
$��"�"�*�$����4��0�0�>�>�?O�P�2B��/�%)�%:�%:�%A�%A��%�%�'7��&��"�%������1�-�	 �
�	
�o%�
$��X%�
$�s�C"H>�7A*I�>I�Ic�B��||jkDrtd|�d|j�d���||jz}||jz}tj	d||�|xj
||z
z
c_|xjdz
c_t�|�!||�}|S)r�r�r�rmz!BlockCache fetching block (%s) %drP)	rr�rrnr�r rrHr))r!r��log_infor'ryr�r1s      �r"r�z!BackgroundBlockCache._fetch_block�s�����$�,�,�&�� ���/)�)-����a�9��
�
�t�~�~�-���d�n�n�$�����7��<�P��"�"�c�E�k�1�"����1�������s�3���r$c	��||jz}||jz}|xjdz
c_||k(r|j|�}|||S|j|�|dg}|jt	|jt|dz|���|j
|j|�d|�dj|�S)r�rPNr$r�r�s	         r"r�z BackgroundBlockCache._read_cache�s����D�N�N�*�	�����&��	
���!����!1�1��,�,�-?�@�E���7�+�+��+�+�,>�?�	�
�K�L�C�

�J�J���,�,��,�q�0�2B�C��
�
�J�J�t�/�/�0@�A�(�7�K�L��8�8�C�=� r$r�r�r&r�r<r�)�sync)r�r5r=r=r7r;r�)r2r>r?r@rrAr#r�r�r�r)r�r�r�r�s@r"r(r(�s�����2'�D�-�&�MO�3��3�'.�3�69�3�FI�3�	
�3�	5��3�J
�X�$(!��(!�"�(!�8;�(!�OR�(!�	�(!r$r(z!dict[str | None, type[BaseCache]]�cachesc�r�|j}|s |tvrtd|�dt|����|t|<y)z�'Register' cache implementation.

    Parameters
    ----------
    clobber: bool, optional
        If set to True (default is False) - allow to overwrite existing
        entry.

    Raises
    ------
    ValueError
    zCache with name z is already known: N)rr@r�)�cls�clobberrs   r"�register_cacherD�s=���8�8�D��t�v�~��+�D�8�3F�v�d�|�n�U�V�V��F�4�Lr$)F)rBztype[BaseCache]rCr�r7r8)3�
__future__rrr��loggingr�rTrr�collections.abcr�concurrent.futuresrr�	itertoolsr�operatorr	�typingr
rrr
rrrE�typing_extensionsrrr�	getLoggerrnr5r;r6�list�MultiFetcherrrDr�r�r�r�r�r�rr(r@rArD�crBr$r"�<module>rQst��"�����	��#�$�9���M�M���+��#��A����A��C�L��
��	�	�+�	,��
�C��:�u�$�
%����c�3�h��(�%�/�0��<�<�~K&�	�K&�\+%�Y�+%�\+,�i�+,�\}!��}!�@b��b�J%�y�%�0s�	�s�l9�7�1�a�4�=�9�xK!�9�K!�`	�)�-��)���(�
��������

�A��1��

r$