o
    i                  	   @   s$  d dl Z d dlZd dlZd dlZd dlmZ ejr!d dlmZm	Z	 dZ
eeZdedefddZd	d
dedefddZd	d
dededeje fddZd	d
dedeje defddZd	ddedefddZd	ddededeje fddZd	ddedeje defddZdedefddZdS )    N)	Tokenizer)AsyncClientClient
tokenizersmodelreturnc                 C   s   t  d|  S )N:)TOKENIZER_CACHE_KEY)r    r
   k/var/www/html/karishye-ai-python/venv/lib/python3.10/site-packages/cohere/manually_maintained/tokenizers.pytokenizer_cache_key   s   r   cor   c              
   C   s   |  t|}|dur|S | j|j}|std| zt|}td| d| d W n t	yI } zt
d|  W Y d}~nd}~ww t|}t|j}| t|| |S z9Returns a HF tokenizer from a given tokenizer config URL.Nz!No tokenizer URL found for model z Downloading tokenizer for model z
. Size is z MBs.z0Failed to get the size of the tokenizer config: )
_cache_getr   modelsgettokenizer_url
ValueError_get_tokenizer_config_sizeloggerinfo	Exceptionwarnrequestsr   from_strtext
_cache_setr   r   	tokenizerr   sizeeresponser
   r
   r   get_hf_tokenizer   s"   
r"   r   c                 C   s   t | |}|j|ddjS )-Encodes a given text using a local tokenizer.Fadd_special_tokens)r"   encodeidsr   r   r   r   r
   r
   r   local_tokenize+   s   
r)   tokensc                 C   s   t | |}||S )7Decodes a given list of tokens using a local tokenizer.)r"   decoder   r   r*   r   r
   r
   r   local_detokenize1   s   

r.   r   c              
      s   |  t|}|dur|S | j|I dH j}|s!td| zt|}td| d| d W n t	yM } zt
d|  W Y d}~nd}~ww t dtj|I dH }t|j}| t|| |S r   )r   r   r   r   r   r   r   r   r   r   r   asyncioget_event_looprun_in_executorr   r   r   r   r   r   r
   r
   r   async_get_hf_tokenizer7   s$   r2   c                    s"   t | |I dH }|j|ddjS )r#   NFr$   )r2   r&   r'   r(   r
   r
   r   async_local_tokenizeP   s   r3   c                    s   t | |I dH }||S )r+   N)r2   r,   r-   r
   r
   r   async_local_detokenizeV   s   
r4   r   c                 C   sJ   t | }d }dD ]}|j|}|r nq	tttt|d d dS )N)zx-goog-stored-content-lengthzContent-Lengthi      )r   headheadersr   roundinttypingcast)r   head_responser   headerr
   r
   r   r   \   s   
r   )r/   loggingr:   r   r   r   TYPE_CHECKINGcohere.clientr   r   r	   	getLogger__name__r   strr   r"   Listr9   r)   Sequencer.   r2   r3   r4   floatr   r
   r
   r
   r   <module>   s"    
    