o
    iq                     @   s  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZmZmZmZmZmZmZmZ ddlZddlmZmZmZmZmZmZmZmZ dd	lm Z m!Z! d
dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( e(ddZ)G dd de*eZ+e	ddG dd dZ,e-dZ.h dZ/h dZ0h dZ1e-dZ2dddddZ3i Z4e5e1D ]Z6e6e4e6< e6 de4e6 d< e6 de4e6 d< qed e4e*e7d!Z8e	ddG d"d# d#Z9eeee f Z:eee;e* f Z<d$eee;e f d%e9fd&d'Z=d$eee;e f d%dfd(d)Z>d*ed%eee*ef ee*eeef f f fd+d,Z?d*ed-e@d%eee: e<f fd.d/ZAd0e*d1e<d%eeee eBge@f fd2d3ZCd4ee: d-e@d1e<d%eee*ef  fd5d6ZDd4ee: d-e@d1e<d%dfd7d8ZEd4ee: d-e@d1e<d%dfd9d:ZFd4ee: d-e@d1e<d%dfd;d<ZGd=eeB d>e*d?eBd%e@fd@dAZHdBe*d%eIee:geIedCf f e@f fdDdEZJd*edFeKe* d%e,fdGdHZLe)M ddIde+jNdIddfdJeee* ejOdKdf dLee@ejOdMdf dNeeeKe*  ejOdOdPdQdf dRee+ejOdSdf dTee@ejOdUdVdWdf dXeee8 ejOdYdf dZeeeP ejOd[df d%dfd\d]ZQe)M 		I	IdxdFeeKe* ejRd^df dJeee* ejOdKdf d_ee@ejOd`dadbdf dcee@ejOdddf d%df
dedfZSe)M 		I	IdxdJeee* ejOdKdf d_ee@ejOd`dadbdf dcee@ejOdddf d%dfdgdhZTe)M e$jUddddIdIdfdie#dje$dke%dJeee* ejOdldf dmeee* ejOdndf doee@ejOdpdqdf dree@ejOdsdtdf due&d%dfdvdwZVdS )yzHContains the 'hf cache' command group with cache management subcommands.    N)defaultdict)	dataclass)Enum)	AnnotatedAnyCallableDictListMappingOptionalTuple   )ANSICachedRepoInfoCachedRevisionInfoCacheNotFoundHFCacheInfo_format_sizescan_cache_dirtabulate)parse_duration
parse_size   )	RepoIdArgRepoTypeOptRevisionOptTokenOpt
get_hf_apityper_factoryzManage local cache directory.)helpc                   @   s   e Zd ZdZdZdZdS )OutputFormattablejsoncsvN)__name__
__module____qualname__r!   r"   r#    r'   r'   _/var/www/html/karishye-ai-python/venv/lib/python3.10/site-packages/huggingface_hub/cli/cache.pyr    1   s    r    T)frozenc                   @   s>   e Zd ZU ee ed< eeee f ed< e	edf ed< dS )_DeletionResolution	revisionsselected.missingN)
r$   r%   r&   	frozensetstr__annotations__dictr   r   tupler'   r'   r'   r(   r*   7   s   
 r*   zA^(?P<key>[a-zA-Z_]+)\s*(?P<op>==|!=|>=|<=|>|<|=)\s*(?P<value>.+)$>   <=>!=<=>=>   refssizetypeaccessedmodified>   namer:   r<   r=   z.^(?P<key>[a-zA-Z_]+)(?::(?P<order>asc|desc))?$descasc)r<   r=   r:   r>   z:asc_ascz:desc_descSortOptions)r;   modulec                   @   s*   e Zd ZU dZeed< eed< eed< dS )CacheDeletionCountsz>Simple counters summarizing cache deletions for CLI messaging.
repo_countpartial_revision_counttotal_revision_countN)r$   r%   r&   __doc__intr0   r'   r'   r'   r(   rE   V   s
   
 rE   selected_by_reporeturnc                 C   sd   d}d}d}|   D ]\}}|t|7 }t|t|jkr'|d7 }|t|7 }q
|| }t|||S )z(Summarize deletions across repositories.r   r   )itemslenr+   rE   )rK   rF   total_revisionsrevisions_in_full_reposrepor+   rG   r'   r'   r(   summarize_deletionsc   s   rR   c              	   C   s   t |  dd dD ]M}|j d|j }t | | dd d}t|t|jkr1td| d q
td| d |D ]}d	t |jpFd
}td|j	 d| d|j
  q;q
dS )zBPretty-print selected cache revisions during confirmation prompts.c                 S      | j | j fS N	repo_typerepo_idlowerrQ   r'   r'   r(   <lambda>w       z0print_cache_selected_revisions.<locals>.<lambda>key/c                 S      | j S rT   commit_hashrevr'   r'   r(   rZ   y         - z (entire repo): z
(detached)z      z [z] N)sortedkeysrV   rW   rN   r+   printjoinr9   ra   size_on_disk_str)rK   rQ   repo_keyr+   revisionr9   r'   r'   r(   print_cache_selected_revisionsu   s    ro   hf_cache_infoc                 C   sL   i }i }| j D ]}|j }|||< |jD ]}||f||j < qq||fS )zPCreate lookup tables so CLI commands can resolve repo ids and revisions quickly.)reposcache_idrX   r+   ra   )rp   repo_lookuprevision_lookuprQ   rm   rn   r'   r'   r(   build_cache_index   s   


ru   include_revisionsc                C   s   g }i }t | jdd d}|D ]+}tdd |jD ||< |r3t |jdd dD ]	}|||f q(q||df q|rI|jdd d ||fS |jd	d d ||fS )
z;Flatten cache metadata into rows consumed by `hf cache ls`.c                 S   rS   rT   rU   rY   r'   r'   r(   rZ      r[   z'collect_cache_entries.<locals>.<lambda>r\   c                 S   s   h | ]
}|j D ]}|qqS r'   )r9   ).0rn   refr'   r'   r(   	<setcomp>   s    z(collect_cache_entries.<locals>.<setcomp>c                 S   r_   rT   r`   rb   r'   r'   r(   rZ      rd   Nc                 S   s&   | d j | d d ur| d jfS dfS )Nr   r    )rr   ra   entryr'   r'   r(   rZ      s
   c                 S   s
   | d j S )Nr   )rr   r{   r'   r'   r(   rZ      s   
 )rh   rq   r.   r+   appendsort)rp   rv   entriesrepo_refs_mapsorted_reposrQ   rn   r'   r'   r(   collect_cache_entries   s"   r   exprr   c                    s  t |  }|std|  d|d |d|d tvr8td d|  dtt d	tvrLtd
 d|  dtt d	dkr[t	fddS dv rzt
 dtdtt dtdtffdd}|S dkr  dkrtd ddtdtt dtdtf fdd}|S dkrtd d	dtdtt dtdtffdd}|S )zqConvert a `hf cache ls` filter expression into the yes/no test we apply to each cache entry before displaying it.zInvalid filter expression: 'z'.r]   opvaluezUnsupported operator 'z' in filter ''. Must be one of .zUnsupported filter key '' in 'r:   c                    s"   t |d ur|j S | j S rT   )_compare_numericsize_on_diskrQ   rn   _)r   size_thresholdr'   r(   rZ      s    z&compile_cache_filter.<locals>.<lambda>>   r<   r=   rQ   rn   nowrL   c                    s>    dkr| j n	|d ur|jn| j}|d u rdS t|| S )Nr<   F)last_accessedlast_modifiedr   )rQ   rn   r   	timestamp)r]   r   secondsr'   r(   _time_filter   s   z*compile_cache_filter.<locals>._time_filterr;   r4   z/Only '=' is supported for 'type' filters. Got 'r   c                    s   | j   kS rT   )rV   rX   r   )expectedr'   r(   _type_filter   s   z*compile_cache_filter.<locals>._type_filterz.Only '=' is supported for 'refs' filters. Got c                    s2   |d ur|j n | t } dd |D v S )Nc                 S   s   g | ]}|  qS r'   )rX   )rw   rx   r'   r'   r(   
<listcomp>       z>compile_cache_filter.<locals>._refs_filter.<locals>.<listcomp>)r9   getr.   rX   )rQ   rn   r   r9   )r   	value_rawr'   r(   _refs_filter   s   z*compile_cache_filter.<locals>._refs_filter)_FILTER_PATTERNmatchstrip
ValueErrorgrouprX   _ALLOWED_OPERATORSlist_FILTER_KEYSr   r   r   r   r   floatbool)r   r   r   r   r   r   r'   )r   r]   r   r   r   r   r   r(   compile_cache_filter   s6   
  &"$r   r   c                C   s   g }| D ]=\}}|r'|du rq|j |j|jt|j|j|j|jt|j	d}n|j |j|j|j|jt|
|t d}|| q|S )zGNormalize cache entries into serializable records for JSON/CSV exports.NrW   rV   rn   snapshot_pathr   r   r   r9   rW   rV   r   r   r   r9   )rW   rV   ra   r/   r   r   r   r   rh   r9   r   r.   r}   )r   rv   r   payloadrQ   rn   recordr'   r'   r(   _build_cache_export_payload   s.   r   c                   s   | s|rdnd}t | dS |rg d}dd | D }ng d} fdd| D }t t||d	 d
d | D }t|}|rQtdd | D }tdd | D }	ntdd |D }tdd |D }	d| d| dt|	 d}
t t|
 dS )zBRender cache entries as a table and show a human-readable summary.zNo cached revisions found.zNo cached repositories found.N)IDREVISIONSIZELAST_MODIFIEDREFSc              
   S   s@   g | ]\}}|d ur|j |j|jd|jdt|jgqS )N   rg   )rr   ra   rl   rjustlast_modified_strrk   rh   r9   )rw   rQ   rn   r'   r'   r(   r     s    
z-print_cache_entries_table.<locals>.<listcomp>)r   r   LAST_ACCESSEDr   r   c                    sD   g | ]\}}|j |jd |jpd|jdt |t gqS )r   rz   rg   )	rr   rl   r   last_accessed_strr   rk   rh   r   r.   rw   rQ   r   r   r'   r(   r   (  s    
)headersc                 S   s   h | ]\}}|qS r'   r'   r   r'   r'   r(   ry   5  r   z,print_cache_entries_table.<locals>.<setcomp>c                 s   s     | ]\}}|d urdV  qd S )Nr   r'   rw   r   rn   r'   r'   r(   	<genexpr>8  s    z,print_cache_entries_table.<locals>.<genexpr>c                 s   s"    | ]\}}|d ur|j V  qd S rT   r   r   r'   r'   r(   r   9       c                 s   s    | ]}t |jV  qd S rT   )rN   r+   rw   rQ   r'   r'   r(   r   ;  s    c                 s       | ]}|j V  qd S rT   r   r   r'   r'   r(   r   <      z
Found z repo(s) for a total of z revision(s) and z	 on disk.)rj   r   rN   sumr   r   bold)r   rv   r   messager   
table_rowsunique_reposrF   revision_count
total_sizesummaryr'   r   r(   print_cache_entries_table  s.   
r   c                C   s0   t | ||d}tj|tjdd tjd dS )z7Dump cache entries as JSON for scripting or automation.rv   r   r   )indent
N)r   r"   dumpsysstdoutwrite)r   rv   r   r   r'   r'   r(   print_cache_entries_jsonB  s   r   c          	      C   s   t | ||d}ttj}|rg d}ng d}|| |s!dS |D ]X}|d }|rV|dd|dd|d	d|d
d|d|d|d|rRd|ndg}n |dd|dd|d|d|d|rsd|ndg}|| q#dS )z@Export cache entries as CSV rows with the shared payload format.r   r   r   Nr9   rW   rz   rV   rn   r   r   r   r   rg   )r   r#   writerr   r   writerowr   rk   )	r   rv   r   recordsr   r   r   r9   rowr'   r'   r(   print_cache_entries_csvK  s:   







r   leftr   rightc                 C   sT   | du rdS | |k| |k| |k| |k | |k| |kd}||vr&t d| || S )z)Evaluate numeric comparisons for filters.NF)r4   r6   r5   r3   r8   r7   z)Unsupported numeric comparison operator: )r   )r   r   r   comparisonsr'   r'   r(   r   |  s   	r   	sort_expr.c                    s   t |   }|std|  d|d  |d} tvr3td  d|  dtt d|r7|nt  }|d	k}d
t	dt
tdf f fdd}||fS )zConvert a `hf cache ls` sort expression into a key function for sorting entries.

    Returns:
        A tuple of (key_function, reverse_flag) where reverse_flag indicates whether
        to sort in descending order (True) or ascending order (False).
    zInvalid sort expression: 'z5'. Expected format: 'key' or 'key:asc' or 'key:desc'.r]   orderzUnsupported sort key 'r   r   r   r?   r|   rL   .c                    s   | \}} dkr|j  }|fS  dkr$|d ur|j}|fS |j}|fS  dkr8|jd ur3|j}|fS d}|fS  dkr]|d urP|jd urK|j}|fS d}|fS |jd urX|jnd}|fS td  )Nr>   r:   r<   g        r=   zUnsupported sort key: )rr   rX   r   r   r   r   )r|   rQ   rn   r   r\   r'   r(   	_sort_key  s.   
z%compile_cache_sort.<locals>._sort_key)_SORT_PATTERNr   r   rX   r   r   
_SORT_KEYSr   _SORT_DEFAULT_ORDER
CacheEntryr2   r   )r   r   explicit_orderr   reverser   r'   r\   r(   compile_cache_sort  s   
 r   targetsc                 C   s   t | \}}tt}t }g }|D ]V}| }|sq| }	td|	rE||	}
|
du r3|| q|
\}}|| 	| |	|j
 q||	}|du rT|| q|jD ]}|| 	| |	|j
 qWqdd | D }tt||t|dS )z8Resolve the deletion targets into a deletion resolution.z[0-9a-fA-F]{40}Nc                 S   s   i | ]	\}}|t |qS r'   )r.   )rw   rQ   revsr'   r'   r(   
<dictcomp>  s    z-_resolve_deletion_targets.<locals>.<dictcomp>r+   r,   r-   )ru   r   setr   rX   re	fullmatchr   r}   addra   r+   rM   r*   r.   r2   )rp   r   rs   rt   r,   r+   r-   
raw_targettargetloweredr   rQ   rn   matched_repofrozen_selectedr'   r'   r(   _resolve_deletion_targets  s@   




r   F	cache_dirz9Cache directory to scan (defaults to Hugging Face cache).r+   zCInclude revisions in the output instead of aggregated repositories.filterz-fz--filterzZFilter entries (e.g. 'size>1GB', 'type=model', 'accessed>7d'). Can be used multiple times.formatzOutput format.quietz-qz--quietz-Print only IDs (repo IDs or revision hashes).r~   a  Sort entries by key. Supported keys: 'accessed', 'modified', 'name', 'size'. Append ':asc' or ':desc' to explicitly set the order (e.g., 'modified:asc'). Defaults: 'accessed', 'modified', 'size' default to 'desc' (newest/biggest first); 'name' defaults to 'asc' (alphabetical).limitzSLimit the number of results returned. Returns only the top N entries after sorting.c              
      s  zt | }W n ty# } ztdt|j  tjdd|d}~ww |p'g }	t||d\}
zfdd|	D }W n tyO } zt	t||d}~ww t

 |D ]  fdd|
D }
qV|rzt|j\}}|
j||d	 W n ty } zt	t||d}~ww |dur|d
k rt	d| d|
d| }
|r|
D ]\}}t|dur|jn|j qdS tjttjttjti}|| |
|dS )z&List cached repositories or revisions.Cache directory not found: r   codeN)rv   c                    s   g | ]}t | qS r'   )r   )rw   r   r   r'   r(   r   1  s    zls.<locals>.<listcomp>c                    s$   g | ]} |d  |d r|qS )r   r   r'   )rw   r|   )fnr   r'   r(   r   7  s   $ )r]   r   r   z&Limit must be a positive integer, got r   r   )r   r   rj   r/   r   typerExitr   r   BadParametertimer   r   r~   ra   rr   r    r!   r   r"   r   r#   r   )r   r+   r   r   r   r~   r   rp   excfiltersr   
filter_fnssort_key_fnr   rQ   rn   
formattersr'   )r   r   r   r(   ls  sN   5r  zQOne or more repo IDs (e.g. model/bert-base-uncased) or revision hashes to delete.yesz-yz--yeszSkip confirmation prompt.dry_runz,Preview deletions without removing anything.c              
   C   s  zt |}W n ty" } ztdt|j  tjddd}~ww t|| }|jr<td |jD ]	}td|  q2t	|j
dkrMtd tjdd|jt|j
 }t|j}	g }
|	jrh|
|	j d	 |	jrt|
|	j d
 |
s|
|	j d
 d|
}td| d|j d t|j |rtd dS |stjdddstd dS |  t|j}	td|	j d|	j d|j d dS )z(Remove cached repositories or revisions.r   r   r   Nz2Could not find the following targets in the cache:re   r   zNothing to delete.z repo(s)z revision(s)z and About to delete z totalling r   Dry run: no files were deleted.zProceed with deletion?F)defaultzDeletion cancelled.Deleted z repo(s) and z revision(s); freed )r   r   rj   r/   r   r   r   r   r-   rN   r+   delete_revisionsrh   rR   r,   rF   r}   rG   rH   rk   expected_freed_size_strro   confirmexecute)r   r   r  r  rp   r   
resolutionr|   strategycountssummary_partssummary_textr'   r'   r(   rmT  sL   





r  c              
   C   sF  zt | }W n ty" } ztdt|j  tjddd}~ww i }t }|jD ]}t	dd |j
D }|s:q+|||< |dd |D  q+t|dkrUtd	 dS tt	||d
d}	|jt|	j
 }
t|}td|j d|
j d t| |rtd dS |stdstd dS |
  td|j d|
j d dS )z)Remove detached revisions from the cache.r   r   r   Nc                 s   s"    | ]}t |jd kr|V  qdS )r   N)rN   r9   rw   rn   r'   r'   r(   r     r   zprune.<locals>.<genexpr>c                 s   r   rT   r`   r  r'   r'   r(   r     r   r   z2No unreferenced revisions found. Nothing to prune.r'   r   r  z unreferenced revision(s) (z total).r  zProceed?zPruning cancelled.r	  z! unreferenced revision(s); freed r   )r   r   rj   r/   r   r   r   r   rq   r.   r+   updaterN   r*   r
  rh   rR   rH   r  ro   r  r  )r   r  r  rp   r   r,   r+   rQ   detachedr  r  r  r'   r'   r(   prune  sJ   
r  rW   rV   rn   zXCache directory to use when verifying files from cache (defaults to Hugging Face cache).	local_dirz?If set, verify files under this directory instead of the cache.fail_on_missing_filesz--fail-on-missing-filesz?Fail if some files exist on the remote but are missing locally.fail_on_extra_filesz--fail-on-extra-fileszLFail if some files exist locally but are not present on the remote revision.tokenc                 C   s  |dur|durt d tjddt|d}|j| t|dr"|jnt|||||d}	d}
t|	j	}|rYt d	 |	j	D ]}t d
|d  d|d  d|d  d|d   q<d}
|	j
r|rrt d |	j
D ]	}t d
|  qed}
nt|	j
 d}t d|  |	jr|rt d |	jD ]	}t d
|  qd}
nt|	j d}t d|  |	j}|
dkrt d|  d|j d| d t d|	j  tj|
dt d|	j d|  d|j d|  t d dS ) a  Verify checksums for a single repo revision from cache or a local directory.

    Examples:
      - Verify main revision in cache: `hf cache verify gpt2`
      - Verify specific revision: `hf cache verify gpt2 --revision refs/pr/1`
      - Verify dataset: `hf cache verify karpathy/fineweb-edu-100b-shuffle --repo-type dataset`
      - Verify local dir: `hf cache verify deepseek-ai/DeepSeek-OCR --local-dir /path/to/repo`
    NzCCannot pass both --local-dir and --cache-dir. Use one or the other.r   r   )r  r   )rW   rV   rn   r  r   r  r   u;   ❌ Checksum verification failed for the following file(s):re   pathz: expected r   z (	algorithmz), got actualr   z1Missing files (present remotely, absent locally):zM remote file(s) are missing locally. Use --fail-on-missing-files for details.u   ⚠️  z/Extra files (present locally, absent remotely):zV local file(s) do not exist on the remote repo. Use --fail-on-extra-files for details.u   ❌ Verification failed for 'z' (z) in r   z  Revision: u   ✅ Verified z file(s) for 'z  All checksums match.)rj   r   r   r   verify_repo_checksumshasattrr   r/   r   
mismatchesmissing_pathsrN   extra_pathsverified_pathrn   checked_count)rW   rV   rn   r   r  r  r  r  apiresult	exit_codehas_mismatchesmpwarningverified_locationr'   r'   r(   verify  sX   *
	

2

$r.  )NFF)WrI   r#   r"   r   r   r   collectionsr   dataclassesr   enumr   typingr   r   r   r   r	   r
   r   r   r   utilsr   r   r   r   r   r   r   r   utils._parsingr   r   
_cli_utilsr   r   r   r   r   r   	cache_clir/   r    r*   compiler   r   r   r   r   r   _sort_options_dictrh   r]   r$   rC   rE   r   r.   RepoRefsMaprR   ro   ru   r   r   r   r   r   r   r   r   r   r2   r   r   r   commandr!   OptionrJ   r  Argumentr  r  modelr.  r'   r'   r'   r(   <module>   sP  ((
 








A
 
2
	1,6,	
#,2aLC
