
    *i$                        S SK Jr  S SKrS SKJrJrJr  S SKrS SKJr   " S S\R                  5      r
 " S S\
5      r " S	 S
\
5      rg)    )annotationsN)ListOptionalAny)r   c                      \ rS rSr\R
                   S       SS jj5       r\R
                    S       S	S jj5       rSrg)
BaseLlamaTokenizer   c                    [         e)zTokenize the text into tokens.

Args:
    text: The utf-8 encoded string to tokenize.
    add_bos: Whether to add a beginning of sequence token.
    special: Whether to tokenize special tokens.
NotImplementedErrorselftextadd_bosspecials       W/var/www/html/ai-backend/venv/lib/python3.13/site-packages/llama_cpp/llama_tokenizer.pytokenizeBaseLlamaTokenizer.tokenize   s
     "!    Nc                    [         e)zDetokenize the tokens into text.

Args:
    tokens: The list of tokens to detokenize.
    prev_tokens: The list of previous tokens. Offset mapping will be performed if provided.
    special: Whether to detokenize special tokens.
r   r   tokensprev_tokensr   s       r   
detokenizeBaseLlamaTokenizer.detokenize   s
     "!r    TTr   bytesr   boolr   r    return	List[int]NFr   r"   r   zOptional[List[int]]r   r    r!   r   )	__name__
__module____qualname____firstlineno__abcabstractmethodr   r   __static_attributes__r   r   r   r   r      s    AE
"
"$(
":>
"	
" 
" 	 ,0	"" )" 	"
 
" "r   r   c                      \ rS rSrS
S jr S       SS jjr  S       SS jjr S       SS jjrSS jr\	SS j5       r
S	rg)LlamaTokenizer-   c                &    UR                   U l         g N_model)r   llamas     r   __init__LlamaTokenizer.__init__.   s    llr   c                6    U R                   R                  XUS9$ )Nr   r   )r2   r   r   s       r   r   LlamaTokenizer.tokenize1   s     {{##D7#KKr   Nc                4    U R                   R                  XS9$ )N)r   )r2   r   r   s       r   r   LlamaTokenizer.detokenize6   s     {{%%f%>>r   c                >    U R                  UR                  SSS9X#S9$ )Nutf-8ignoreerrorsr7   )r   encoder   s       r   r@   LlamaTokenizer.encode>   s,     }}KKK17  
 	
r   c                @    U R                  U5      R                  SSS9$ )Nr<   r=   r>   )r   decode)r   r   s     r   rC   LlamaTokenizer.decodeE   s!    v&--gh-GGr   c                8    U " [         R                  " USS95      $ )NT)
model_path
vocab_only)	llama_cppLlama)clspaths     r   from_ggml_fileLlamaTokenizer.from_ggml_fileH   s    9??dtDEEr   r1   )r3   zllama_cpp.Llamar   r   r#   r$   )r   strr   r    r   r    r!   r"   )r   r"   r!   rN   )rK   rN   r!   z'LlamaTokenizer')r%   r&   r'   r(   r4   r   r   r@   rC   classmethodrL   r+   r   r   r   r-   r-   -   s    # BFLL$(L:>L	L ,0	?? )? 	?
 
? @D

"&
8<
	
H F Fr   r-   c                  p    \ rS rSrSS jr S	       S
S jjr  S       SS jjr\SS j5       rSr	g)LlamaHFTokenizerM   c                    Xl         g r0   hf_tokenizer)r   rU   s     r   r4   LlamaHFTokenizer.__init__N   s    (r   c                R    U R                   R                  UR                  SSS9US9$ )Nr<   r=   r>   )add_special_tokens)rU   r@   rC   r   s       r   r   LlamaHFTokenizer.tokenizeQ   s4       ''KKK1g ( 
 	
r   Nc                "   U(       + nUb_  U R                   R                  X!-   US9R                  SSS9nU R                   R                  X$S9R                  SSS9nU[        U5      S  $ U R                   R                  XS9R                  SSS9$ )N)skip_special_tokensr<   r=   r>   )rU   rC   r@   len)r   r   r   r   r[   r   	prev_texts          r   r   LlamaHFTokenizer.detokenizeX   s     #*k"$$++$:M , fWXf.  ))00 1 fWXf.  I())$$++ , fWXf./r   c                p     SSK Jn  UR                  US9nU " U5      $ ! [         a    [        S5      ef = f)Nr   )AutoTokenizerzsThe `transformers` library is required to use the `HFTokenizer`.You can install it with `pip install transformers`.)pretrained_model_name_or_path)transformersr`   ImportErrorfrom_pretrained)rJ   ra   r`   rU   s       r   rd    LlamaHFTokenizer.from_pretrainedl   sU    	2 %44*G 5 
 <    	F 	s    5rT   )rU   r   r   r   r#   r$   )ra   rN   r!   z'LlamaHFTokenizer')
r%   r&   r'   r(   r4   r   r   rO   rd   r+   r   r   r   rQ   rQ   M   sz    ) BF

$(
:>
	
 ,0	// )/ 	/
 
/( ! !r   rQ   )
__future__r   r)   typingr   r   r   rH   llama_cpp.llama_typesABCr   r-   rQ   r   r   r   <module>rj      sJ    " 
   &" ">F' F@+!) +!r   