o i @sTUddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z ddl m Z ddlmZddlmZdd lmZmZmZmZmZdd lmZmZddlZddlZdd lmZmZm Z dd l!m"Z"d dl#m$Z$d dl%m&Z&m'Z'd dl(m)Z)m*Z*m+Z+m,Z,d dl-m.Z.m/Z/d dl0m1Z1m2Z2m3Z3d dl4m5Z5m6Z6m7Z7m8Z8d dl9m:Z:e;dej<Z=e GdddZ>e?ej@ZAeeBd<GdddeAZCGdddej@eCdZDGdddejEjFZGGd d!d!ejEjFZHd"eId#eIfd$d%ZJd&eId#eIfd'd(ZKd)eId*eeLd#efd+d,ZMd-eDd)eId.eId/eNd0eIf d1d2ZOd:d4ejPd5eNd#eQfd6d7ZRd8d9ZSdS);N)deque) ExitStack)deepcopy) dataclassfield)datetime)chain)Path)AnyIteratorNoReturnOptionalUnion)quoteunquote)_DEFAULT_CALLBACK NoOpCallback TqdmCallback) isfilelike) constants)CommitOperationCopyCommitOperationDelete)EntryNotFoundErrorHfHubHTTPErrorRepositoryNotFoundErrorRevisionNotFoundError) hf_hub_urlhttp_get)HfApiLastCommitInfoRepoFile)HFValidationErrorhf_raise_for_status http_backoffhttp_stream_backoff)md5zy (^refs\/convert\/\w+) # `refs/convert/parquet` revisions | (^refs\/pr\/\d+) # PR revisions c@sXeZdZUdZeed<eed<eed<eed<edddZeeed <d efd d Z dS) HfFileSystemResolvedPathzUData structure containing information about a resolved Hugging Face file system path. repo_typerepo_idrevision path_in_repoNF)defaultrepr _raw_revisionreturncCs~tj|jd|j}|jr|d|jd|jdS|jtj kr4|dt |jd|jdS|d|jdS)N@/) rREPO_TYPES_URL_PREFIXESgetr(r)r.r+rstripr*DEFAULT_REVISION safe_revision)self repo_pathr:d/var/www/html/karishye-ai-python/venv/lib/python3.10/site-packages/huggingface_hub/hf_file_system.py unresolve4s  "z"HfFileSystemResolvedPath.unresolve) __name__ __module__ __qualname____doc__str__annotations__rr.r r<r:r:r:r;r'(s r' _cached_basecs(eZdZdZfddZddZZS)_Cacheda Metaclass for caching HfFileSystem instances according to the args. This creates an additional reference to the filesystem, which prevents the filesystem from being garbage collected when all *user* references go away. A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* be made for a filesystem instance to be garbage collected. This is a slightly modified version of `fsspec.spec._Cached` to improve it. In particular in `_tokenize` the pid isn't taken into account for the `fs_token` used to identify cached instances. The `fs_token` logic is also robust to defaults values and the order of the args. Finally new instances reuse the states from sister instances in the main thread. cstj|i|i|_dSN)super__init___cache)clsargskwargs __class__r:r;rGRs z_Cached.__init__c Os|dd}|j|tg|Ri|}|j|tjg|Ri|}|s7|jr7||jvr7||_|j|St j |g|Ri|}|sc|jrc||jvrc|j| }| D] \}} t ||| qX||_||_||_|jry|sy||_||j|<|S)Nskip_instance_cacheF)pop _tokenize threading get_ident main_threadidentcachablerH_latesttype__call___get_instance_stateitemssetattr _fs_token_ storage_argsstorage_options) rIrJrKskipfs_tokenfs_token_main_threadobjinstance_stateattr state_valuer:r:r;rXZs$     z_Cached.__call__)r=r>r?r@rGrX __classcell__r:r:rLr;rDBs rDcseZdZdZdZdZdddddeedee edfdee ffd d Z e d e d efd dZ dededeed ee eeffddZdNdedeed efddZdNdeed dfddZ   dOdededeedee d df ddZdNdedeed dfddZ  dPded!e d"ee deed df d#d$Z dQded&e d'e deed eeeeeefff d(d)Z  dRded!e d'e deed*e d"ee f d+d,Zded eeeeeeefffd-d. Zded eeffd/d0 Z  dSded"ee d1e d&e d'e deed eeeeeeeeffffd2d3ZdNd4ed5edeed dfd6d7Z ded e!fd8d9Z"dTded'e deed eeeffd:d;Z#dd?Z%d@dAZ&ded efdBdCZ'e(dfdUfdDdE Z)e*dFdGZ+dHdIZ,dJdKZ-dLdMZ.Z/S)V HfFileSystema% Access a remote Hugging Face Hub repository as if were a local file system. > [!WARNING] > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. Args: token (`str` or `bool`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . Usage: ```python >>> from huggingface_hub import HfFileSystem >>> fs = HfFileSystem() >>> # List files >>> fs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> fs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with fs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` r0hfN)endpointtoken block_sizerirjrkcsHtj|i||p tj|_||_t||d|_||_i|_ i|_ dS)N)rirj) rFrGrENDPOINTrirjr_apirk_repo_and_revision_exists_cachedircache)r8rirjrkrJr^rLr:r;rGs  zHfFileSystem.__init__threading_identr/cs\dptjd<dd<fddtD|||f}tt|}|S)zDeterministic token for cachingrirjci|]}||qSr:r:).0keyrKr:r; z*HfFileSystem._tokenize..)r4rrlsortedr&rAencode hexdigest)rIrprJrK tokenize_argshr:rtr;rPs  zHfFileSystem._tokenizer(r)r*c Cs|||f|jvrqz |jj|||tjdWnKttfy=}zd|f|j|||f<d|f|j||df<WYd}~n8d}~wty`}zd|f|j|||f<d|j||df<WYd}~nd}~wwd|j|||f<d|j||df<|j|||fS)N)r*r(timeoutF)TN)rnrm repo_inforHF_HUB_ETAG_TIMEOUTrr"r)r8r(r)r*er:r:r;_repo_and_revision_exists"   z%HfFileSystem._repo_and_revision_existpathcCspdttdttdttfdd}||}|std|dddtjvr@d|vr2td|dd \}}tj|}ntj }| ddkrd |vr|d d \}}d|vrt |}|d ury|d | fvryt d |d}| }n |dd \}}nd }|t||}||||\} } | st|| nd }d|dd d } d|dd d } |dd} d|dd d }| }| }||||\} } | st| ttfr| }|}||||\} }| st|| n1t|| n+|}d }d |vr|d d \}}|t||}nd }||||\} }| s%td|d ur,|ntj}t|||||dS)a Resolve a Hugging Face file system path into its components. Args: path (`str`): Path to resolve. revision (`str`, *optional*): The revision of the repo to resolve. Defaults to the revision specified in the path. Returns: [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. Raises: `ValueError`: If path contains conflicting revision information. `NotImplementedError`: If trying to list repositories. revision_in_pathr*r/cSs:|dur|dur||krtd|d|d|S|}|S)NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.) ValueError)rr*r:r:r;%_align_revision_in_path_with_revisionszHHfFileSystem.resolve_path.._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r2rrr1Nr0)r.)r rA_strip_protocolNotImplementedErrorsplitrr3valuesREPO_TYPES_MAPPINGREPO_TYPE_MODELcountSPECIAL_REFS_REVISION_REGEXsearchgroupsublstriprr_raise_file_not_foundjoin isinstancerr"r6r')r8rr*rr(r)rmatchr+repo_and_revision_existerrrepo_id_with_namespacepath_in_repo_with_namespacerepo_id_without_namespacepath_in_repo_without_namespace_r:r:r; resolve_pathsx        zHfFileSystem.resolve_pathcCs|s|j|jdS||}|}|r'|j|d||}|s|jsE|j|j|j dfd|j|j|j |j fddSdS)ac Clear the cache for a given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). Args: path (`str`, *optional*): Path to clear from cache. If not provided, clear the entire cache. N) roclearrnrr<rO_parentr+r(r)r*)r8r resolved_pathr:r:r;invalidate_cache1s   zHfFileSystem.invalidate_cacherbmodeHfFileSystemFilecKsj|dur|n|j}|dur||d<d|vrtd|dkr)t||f||d|St||f||d|S)Nrkaz/Appending to remote files is not yet supported.r)rr*)rkrHfFileSystemStreamFiler)r8rrr*rkrKr:r:r;_openMszHfFileSystem._openc KsR|j||d}|jj|j|j|j|j|j|d|dd|j | ddS)Nr*commit_messagecommit_description)r+r)rjr(r*rrr) rrm delete_filer+r)rjr(r*r4rr<)r8rr*rKrr:r:r;_rm_s zHfFileSystem._rmF recursivemaxdepthc sj||d}j||||d}fdd|D}dd|D} d|d} | |r+dnd 7} | |d ur9d |dnd 7} jj|j|jj| |j|d | |d dj | dd S)a Delete files from a repository. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). > [!WARNING] > Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. r)rrr*cs"g|] }|s|jqSr:)isdirrr+)rrrr8r:r; "z#HfFileSystem.rm..cSsg|]}t|dqS))r+)r)rrr+r:r:r;rrvzDelete  z recursively r0Nz up to depth rr)r)r(rj operationsr*rrr) r expand_pathrm create_commitr)r(rjr*r4rr<) r8rrrr*rKrpaths paths_in_reporrr:rr;rmls"   zHfFileSystem.rmTdetailrefreshc s|j|d}|z|jf||d|}Wn4tyM|js(td|j|f||d|}fdd|D}t|dkrKtdYnw|rR|Sdd|DS)a List the contents of a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). > [!WARNING] > Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `list[Union[str, dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). r)rr*Ncg|] }|dkr|qSnamer:rrorr:r;rz#HfFileSystem.ls..rcSsg|]}|dqSrr:rr:r:r;rs)rr<_ls_treerr+rrlen)r8rrrr*rKroutr:rr;lss    zHfFileSystem.ls expand_infoc sj||d}|}t|j|j|jd|jd}g} |jvr|sj|} | | g} |rwdt fdd| D} | rw| \} |dusN|kru| djvr]| | dnj| d} | | | fdd| D| s@g}|rfd d| D}|r| s|r|rt j | |}|d s||ks|t| |vr|d n||durt|dd }||8}fd d| D} tjD]}|d r߈j|dqψjd| j|d |||d | Sjj|j|j|||j|jd}|D]U}|d |j }t|tr(||jd|j|j |j!|j"d}n |dd|j#|j!d}|d}j$|g ||t|dd |dusX|kr]| |q | S)Nrr0r+r.rcs g|] }|ddkr|fqSrW directoryr:rr path_infodepthr:r;rs z)HfFileSystem._ls_tree..rcs$g|]}|ddkrd|fqS)rWrrr:rrr:r;rs   cs&g|]}|ddur|dqS) last_commitNr)rrrr:r;rs&r2cs"g|] }|dds|qS)rr2) startswithr) common_pathr:r;rrT)rrr*rr)rexpandr*r(filersizerWblob_idlfsrsecurityrrrrrWtree_idr)%rr<r'r(r)r*r.roextendrpopleftappendosr commonprefixendswithrr5rrrlistrrOrrmlist_repo_treer+rr!rrrrrr setdefault)r8rrrr*rrr root_pathrcached_path_infosdirs_not_in_dircache dirs_to_visitdir_infodirs_not_expanded common_prefixcommon_path_depth cached_pathtreer cache_pathcache_path_info parent_pathr:)rrr8r;rs       ,   zHfFileSystem._ls_treec/s>|j||dd}tj|g|Ri|EdHdS)a Return all files below the given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). Args: path (`str`): Root path to list files from. Returns: `Iterator[tuple[str, list[str], list[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. r*rN)rr4r<rFwalk)r8rrJrKrLr:r;r8s $zHfFileSystem.walkc s,|j||dd}tj|fi|S)ah Find files by glob-matching. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). Args: path (`str`): Path pattern to match. Returns: `list[str]`: List of paths matching the pattern. r*r)rr4r<rFglobr8rrKrLr:r;rHs zHfFileSystem.globwithdirsc s"|dur |dkr td|j||d}|}z|j|fd||j|d|Wn-tyUz|j|fd|i|dd krD|iiniWn tyRiYnwYn+w|s`d d Dn|j|fd|ji|} | dd krw| gnd dDt} |s| Sfdd| DS)a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[list[str], dict[str, dict[str, Any]]]`: List of paths or dict of file information. Nrzmaxdepth must be at least 1rT)rrr*rr*rWrcSsg|] }|ddkr|qSrr:rr:r:r;rrz%HfFileSystem.find..rcSsi|]}|d|qSrr:rr:r:r;rurvz%HfFileSystem.find..crqr:r:)rrrrr:r;rurv) rrr<rr*rinfoFileNotFoundErrorrw) r8rrrrrr*rKrrnamesr:rr;findXs@      zHfFileSystem.findpath1path2c Ks,|j||d}|j||d}|j|jko|j|jk}|rEd|d|}|jj|j|j|j|d||ddt|j|j|jdgdn?|j |d |jd } | } Wd n1s]wYd|d|}|jj | |j|j|j |j|j|d||dd |j |d |j |d d S) a Copy a file within or between repositories. > [!WARNING] > Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rzCopy z to rrr0)src_path_in_repor+ src_revision)r)r(r*rrrrNpath_or_fileobjr+r)rjr(r*rrr)rr(r)rmrr*r4rr+openread upload_filerjrr<) r8rrr*rKresolved_path1resolved_path2 same_reporfcontentr:r:r;cp_filesF     zHfFileSystem.cp_filecKs*|j|fii|ddi}|ddS)ai Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last commit date of the file. rTrdater)r8rrKrr:r:r;modifieds zHfFileSystem.modifiedc  s|j|d}||dd}|js=dddd|r<|jj|j|j|jdd }idt |j |j |j d d nd| }|sP||jvrP||||jvrlfd d |j|D} | shtd| d|s|dus||rوrوddur|jj|j|j||j|jd} | std| d} t|j|j|jd|jd} t| tr| d| j| jd| j| j| j| jdn| d| jdd| j| jd|sهfdddDdusJS)a Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). > [!WARNING] > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance. Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rrFrrN)rrrWr)r(r*)oidtitler )rrcrrr:rrr:r;rrz%HfFileSystem.info..r)rr*r(r0rr2rrrcrqr:r:)rrkrr:r;ru=rvz%HfFileSystem.info..)rrrW)rr<r4r+rmlist_repo_commitsr)r(r*r commit_idr created_atrrorrget_paths_infor'r.rr!rrrrrrr) r8rrr*rKrrrrout1 paths_inforrr:)rrr;rs            zHfFileSystem.infocKs<z|ddr |||j|fi|WdSYdS)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). > [!WARNING] > Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFT)r4rrrr:r:r;existsAs  zHfFileSystem.existscCs*z ||ddkWStyYdSw)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. rWrF)rOSErrorr8rr:r:r;rZs   zHfFileSystem.isdircCs"z ||ddkWSYdS)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. rWrFr rr:r:r;isfilels zHfFileSystem.isfilecCsB||}t|j|j|j|j|jd}||r|ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. )r(r*riz /resolve/z/tree/r) rrr)r+r(r*rirreplace)r8rrurlr:r:r;r~s  zHfFileSystem.urlc sb|d}t|dh}t|ttfrt|dkr)tj||f||d|St |r0|}n| |r>t j |dddSt|t tfrPt j t j|ddd}|dur]t|d}d}|} |j||d } |j||d d } || z2tt| j| j| j| j|jd ||| d|jt|tr|jndd | | W|r|!dSdS|r|!ww) aC Copy single remote file to local. > [!WARNING] > Note: When possible, use `HfApi.hf_hub_download()` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. r*r)callbackoutfileT)exist_okNFwbrrr)r*filenamer(ri)r temp_filedisplayed_filename expected_size resume_sizeheaders _tqdm_bar)"r4setkeysrrrrrFget_filerrrmakedirsrAr rdirnamertellrrset_sizerrr)r*r+r(rirm_build_hf_headerstqdmseekclose) r8rpathlpathrrrKr*unhandled_kwargs close_file initial_posresolve_remote_pathr$rLr:r;r*sR       zHfFileSystem.get_filecCtd)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.rrr:r:r; transactions zHfFileSystem.transactioncCr9)z@Begin write transaction for deferring files, non-context versionr:r;rr:r:r;start_transactionszHfFileSystem.start_transactioncCstt||j|j|ffSrE) make_instancerWr]r^rYrr:r:r; __reduce__s zHfFileSystem.__reduce__cCst|jt|jdS)N)rorn)rrornrr:r:r;rYsz HfFileSystem._get_instance_staterE)rNN)FNN)TFN)FFNFN)NFFFN)FNr/N)0r=r>r?r@ root_markerprotocolr rArboolintrG classmethodrPtuple Exceptionrr'rrrrrrdictr rrr rrrrrr rrrrrrr*propertyr<r=r?rYrfr:r:rLr;rgts%   _  / + ,v ?8(bD   rg) metaclasscseZdZddededeeffdd ZfddZd ed ed e fd d Z dddZ dde d dfddZ dfdd Zd efddZZS)rNfsrr*c stz |j||d|_Wnty'}zd|ddvr"t|d|d}~wwtj||jfi||dS)Nrwrr0B. Make sure the repository and revision exist before writing data.)rrrr4rFrGr<)r8rKrr*rKrrLr:r;rGszHfFileSystemFile.__init__cst|dsdStSNr)hasattrrF__del__rrLr:r;rPs  zHfFileSystemFile.__del__startendr/cCsjdd|d|di|jj}t|jj|jj|jj|jj|jj d}t d||t j d}t ||jS)Nrangebytes=-rr GET)r&r|)rKrmr/rrr)r*r+r(rir$rHF_HUB_DOWNLOAD_TIMEOUTr#r)r8rQrRr&rrr:r:r; _fetch_range s zHfFileSystemFile._fetch_rangecCstjddd|_dS)Nzhffs-F)prefixdelete)tempfileNamedTemporaryFiler"rr:r:r;_initiate_uploadsz!HfFileSystemFile._initiate_uploadFfinalc Cs|jd|j}|j||rN|j|jjj|jj |j j |j j |jj |j j|j j|jd|jddt|jj |jj|j ddSdS)Nrrrrr)bufferr1rr"writer2rKrmrrrr+r)rjr(r*rKr4rremoverr<)r8r_blockr:r:r; _upload_chunk s(        zHfFileSystemFile._upload_chunkr cs|jdkr;|dus |dkr;|jdkr;|jj|jddd}|}|jt|7_|WdS1s6wYt|S)zRead remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rNr r)rk)rlocrKrrrrrF)r8lengthrrrLr:r;r5s$ zHfFileSystemFile.readcC|j|jSrErKrrrr:r:r;rBzHfFileSystemFile.urlrEr@)Fr )r=r>r?rgrAr rGrPrDbytesrYr^rCrdrrrfr:r:rLr;rs    rc seZdZ    ddedededeed ed ef fd d Zdd edefddZd defddZ defddZ fddZ ddZ ddZ ZS)!rrNrnonerKrrr*rk cache_typec s|dkr td||dkrtd|d|vr"td|dz |j||d|_Wn"tyN}zd|d d vrDt|d |WYd}~nd}~ww|jdd |_tj||jf|||d |d|_ |t |_ dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got rlz?HfFileSystemStreamFile only supports cache_type='none' but got rLz;HfFileSystemStreamFile only supports reading but got mode=''rrr0rM)rrrrkrm) rrrrr4r<detailsrFrGresponser _exit_stack) r8rKrrr*rkrmrKrrLr:r;rGGs:    zHfFileSystemStreamFile.__init__rewhencecCs2|dkr |dkr dS||jkr|dkrdStd)NrrzCannot seek streaming HF file)rer)r8rersr:r:r;r1gs zHfFileSystemStreamFile.seekr rfcCs|jdur |d} z|jdurWdSt|j|}|jt|7_|WStyB|jdur7|j|r:|d}Ynwq )zRead the remote file. If the file is already open, we reuse the connection. Otherwise, open a new connection and read from it. If reading the stream fails, we retry with a new connection. NFT)rq_open_connection _partial_readrerrGr2)r8rf retried_oncerr:r:r;rns&      zHfFileSystemStreamFile.readr/cCrgrErhrr:r:r;rrizHfFileSystemStreamFile.urlcs"t|dsdS|jtSrN)rOrrr2rFrPrrLr:r;rPs   zHfFileSystemStreamFile.__del__cCst|j|j|j|j|jjffSrE)reopenrKrr blocksizecacherrr:r:r;r?sz!HfFileSystemStreamFile.__reduce__c Cst|jj|jj|jj|jj|jjd}|jj }|j dkr'd|j d|d<|j t d||dtjd|_zt|jWd Sty[}z|jjd krVd |_WYd }~d Sd }~ww) z%Open a connection to the remote file.r rrTrURangerV)iiii)r&retry_on_status_codesr|iN)rrr)r*r+r(rKrirmr/rerr enter_contextr%rrWrqr#r status_code)r8rr&rr:r:r;rus8    z'HfFileSystemStreamFile._open_connection)rNrrl)rrj)r=r>r?rgrAr rDrGr1rrrPr?rurfr:r:rLr;rFs0  rr*r/cCst|r|St|SrE)rr safe_quoterr:r:r;r7sr7scCs t|ddS)Nr0)safe)r)rr:r:r;rs rrrcCsN|}t|tr |d}nt|tr|d}n t|tr"|d}t||)Nz (repository not found)z (revision not found)z (invalid repository id))rrrr"r)rrmsgr:r:r;rs       rrKrrkrmcCs|j||||dS)Nro)r)rKrrrkrmr:r:r;rxsrxr rqrfcCst}|dkr td|dkrdS|dkr%|D]}||qt|S|j|dD]}||t||krBt|d|Sq+t|S)zb Read up to `length` bytes from a streamed response. If length == -1, read until EOF. r zlength must be -1 or >= 0rrt) chunk_sizeN) bytearrayr iter_bytesrrkr)rqrfbufchunkr:r:r;rvs    rvcCs0||i|}|D] \}}t|||q |SrE)rZr[)rIrJrKrcrKrdrer:r:r;r>sr>rj)Trrer\rQ collectionsr contextlibrcopyr dataclassesrrr itertoolsrpathlibr typingr r r r r urllib.parserrfsspechttpxfsspec.callbacksrrr fsspec.utilsrr0r _commit_apirrerrorsrrrr file_downloadrrhf_apirr r!utilsr"r#r$r%utils.insecure_hashlibr&compileVERBOSErr'rWAbstractFileSystemrCrBrDrgspecAbstractBufferedFilerrrAr7rrGrrDrxResponserkrvr>r:r:r:r;s`           2 Lq