*iT 4SSSKJr SSKrSSKrSSKrSSKJrJrJrJ r J r SSK J r J r Jr \ (aSSK JrJrJrJrJrJrJr Sr\R.R1S5r\cG\R4"\R6R9\R6R;\555S- O\R4"\5r\ "\\5r \"\ 5r!Sr"S r#S r$S r%S r&S r'Sr(Sr)Sr*Sr+Sr,Sr-Sr.Sr/Sr0Sr1Sr2Sr3Sr4Sr5Sr6Sr7Sr8Sr9S r:S!r;S"r\R~"\R\R\R\R5rB\R~"\R\R5rC/\ RlE\R\ RlG\ R5rHS%rIS&rJS'rKS(rLS)rM\LrNSrO\MrPS rQ\"S*\R5rS\RrT\"S+\R5rU\RrV\"S,\R5rW\RrX\"S-\R5rY\RrZ\"S.\R5r[\Rr\\Rr^\Rr_\R"\_5ra\RrbSrcS rdS reS rfS/rgS0rhS riSrjS rkS rlS rmS/rnS0roS rpS rqSrrSrsSrtSruSrvSrwSrxSrySrzSr{Sr|Sr}Sr~SrSrSrSrSrS rS!rS"rS#rS$rS1rS2rS3rS4rS5rS&rSrS =rrS=rrS=rrSrS rS rS rS/rS0rS rSrS rS rS/rSrSrS2rS6rS7rS8rS9rSrS rS rS rS rSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrSrS rS!rS"rS#rS$rS1rS2rS:rS;rSS?\GR5r\R"\5r"S@SA\GR5r\R"\5r\R~"\R\GR\R5r"SBSC\GR5rSrS rS rS r"SDSE\R5r"SFSG\GR5r"SHSI\GR5r"SJSK\GR5r\R~"S\GR\GR\R5r"SLSM\GR5r"SNSO\GR5r\R"\5r"SPSQ\GR5r"SRSS\GR5r\Rr\R"\R5r\!"ST/\5GSSUj5r\!"SV/\5GSSWj5r\!"SX/\5GSSYj5r\!"SZ/\5GSS[j5r\!"S\/S5S]5rSrS rS rS rS/rS0Gr\!"S^/S5S_5Gr\!"S`\GR/S5GSSaj5Gr\!"Sb\GR\/\V5GSScj5Gr\!"Sd\GR\/\V5GSSej5Gr\!"Sf\R"\GR5\R\/\V5GSSgj5Gr\!"Sh\V\GR/S5GSSij5Gr\!"Sj\V/S5GS Skj5Gr\!"Sl\V/S5GS Smj5Gr\!"Sn\V\/\X5GS!Soj5Gr \!"Sp\V\/\X5GS!Sqj5Gr \!"Sr\X/S5GS"Ssj5Gr \!"St/\GR5GS#Suj5Gr \!"Sv/\R5GS#Swj5rD\!"Sx/\R5GS#Syj5Gr\!"Sz/\R5GS$S{j5Gr\!"S|/\R5GS$S}j5Gr\!"S~/\R5GS$Sj5Gr\!"S/\R5GS$Sj5Gr\!"S\X/\GR&5GS%Sj5Gr\!"S\X/\GR&5GS%Sj5Gr\!"S\X/\GR&5GS%Sj5Gr\!"S\X/\GR&5GS%Sj5Gr\!"S\V/\R5GS&Sj5Gr\!"S\V/\R5GS&Sj5Gr\!"S\V/\R5GS&Sj5Gr\!"S\V/\R5GS&Sj5Gr\!"S\T/\R5GS'Sj5Gr\!"S\X/\V5GS(Sj5Gr\!"S\X/\Z5GS)Sj5Gr\!"S\X/\GR5GS%Sj5Gr\!"S\X/\\5GS*Sj5Gr \!"S\V/\T5GS+Sj5Gr!\!"S\V/\GR5GS&Sj5Gr"\!"S\V/\R5GS&Sj5Gr#\!"S\V/\R5GS&Sj5Gr$\!"S\V/\R5GS&Sj5Gr%\!"S\V/\R5GS&Sj5Gr&\!"S\V/\R5GS&Sj5Gr'\!"S\V/\R5GS&Sj5Gr(\!"S\V/\GR5GS,Sj5Gr)\!"S\V/\GR&5GS&Sj5Gr*\!"S\V\GR&/\GR5GS-Sj5Gr+\!"S\T/\GR5GS.Sj5Gr,\!"S\T/\R5GS.Sj5Gr-\!"S\V\GR\GR\R/\R5GS/Sj5Gr.\!"S\V/\R5GS&Sj5Gr/\!"S\V\R\GR\R/\R5GS0Sj5Gr0\!"S\V\R\GR\R/\R5GS0Sj5Gr1\!"S\V\GR\R/\R5GS1Sj5Gr2\!"S\V/\GRf5GS&Sj5Gr4\!"S\V\GR/\GR5GS2Sj5Gr5\!"S\V/\GRf5GS&Sj5Gr6\!"S\V/\R5GS3Sj5Gr7\!"S\V/\R5GS3Sj5Gr8\!"S\V/\R5GS&Sj5Gr9\!"S\V/\R5GS3Sj5Gr:\!"S\V/\R5GS3Sj5Gr;\!"S\GR\GR\R"\5/\GR&5GS4Sj5Gr<\!"S\V\GR/\5GS5Sj5Gr=\!"S\/S5GS6Sj5Gr>\!"S\X\\GR/\R5GS7Sj5Gr?\!"S\X\/\R5GS8Sj5Gr@\!"S\X/S5GS"Sj5GrA\!"S\X\R"\GR5\R\R\R\R/\R5GS9Sj5GrB\!"S\Z\R/S5GS:Sj5GrC\!"S\Z\b\^\^/\R5GS;Sj5GrD\!"S\Z\b\b\^\^/S5GSSj5GrG\!"S\Z\b\^\^\GR/S5GS?Sj5GrH\!"S\Z\b/\^5GS@Sj5GrI\!"S\Z\b/\^5GS@Sj5GrJ\!"S\Z/\R5GSASj5GrK\!"S\X/\R5GS%Sj5GrL\!"S\X/\R5GS%Sj5GrM\!"S\X/S5GS"Sj5GrN\!"S\X\b\^\^/\R5GSBSj5GrO\!"S\X\b\b\^\^/S5GSCSj5GrP\!"S\X\b/S5GSDSj5GrQ\!"S\X\b\^\^\^/S5GSESj5GrR\!"S\X\b\^\^\GR/S5GSFGSj5GrS\!"GS\X\b/\^5GSGGSj5GrT\!"GS\X\b/\^5GSGGSj5GrU\!"GS\X/S5GS"GSj5GrV\!"GS\X/\R5GSHGSj5GrW\!"GS \X/S5GS"GS j5GrX\!"GS \X/\R5GS%GS j5GrY\!"GS \X/\R5GS%GSj5GrZ\!"GS\X\R"\GR5\R/\R5GSIGSj5Gr\\!"GS\X\R"\GR5/\R5GSJGSj5Gr]\!"GS\X\R"\GR5\R/\R5GSKGSj5Gr^\!"GS\X\R"\GR5/\R5GSLGSj5Gr_\!"GS\X\GR\a\R\R"\R5/\R5GSMGSj5Gr`\!"GS\X\GR\a\R\R"\R5/\R5GSMGSj5Gra\!"GS\X\GR\a\R/\R5GSNGSj5Grb\!"GS\X\GR\a\R/\R5GSNGSj5Grc\!"GS\X\b/\R5GSOGS j5Grd\!"GS!\X\R"\GR5\R\b/\R5GSPGS"j5Gre\!"GS#\X\R"\GR5\R\b/\R5GSQGS$j5Grf\!"GS%\X\GR\b\a\R/\R5GSRGS&j5Grg\!"GS'\X\GR\b\a\R\R"\R5/\R5GSSGS(j5Grh\!"GS)\a\R/\5GSTGS*j5Gri\!"GS+\R\R\R/\5GSUGS,j5Grj\!"GS-\/S5GSVGS.j5Grk\!"GS/\X\/\R5GSWGS0j5Grl\!"GS1\X\/\R5GSWGS2j5Grm\!"GS3\X\R\R/S5GSXGS4j5Grn\!"GS5\X/\R5GS%GS6j5Gro\!"GS7\X/\R5GS%GS8j5Grp\!"GS9\X\R/S5GSYGS:j5Grq\!"GS;\X\R/S5GSZGS<j5Grr\!"GS=\X\R/S5GS[GS>j5Grs\!"GS?\X\C\R/S5GS\GS@j5Grt\!"GSA\X/S5GS"GSBj5Gru\!"GSC\X/\R"\GR55GS]GSDj5Grv\!"GSE\X\R/\R"\GR55GS^GSFj5Grw\!"GSG\X/\R"\GR55GS]GSHj5Grx\!"GSI\X\R/\R"\GR55GS^GSJj5Gry\!"GSK\X\b/\R"\GR55GS_GSLj5Grz\!"GSM\T\_/\GR5GS`GSNj5Gr{\!"GSO\T\_/\GR5GSaGSPj5Gr|\!"GSQ\T\_/\GR5GSbGSRj5Gr}\!"GSS\T\_/\R5GScGSTj5Gr~\!"GSU\T\_/\R5GScGSVj5Gr\!"GSW\T/\_5GSdGSXj5Gr\!"GSY\T/\_5GSdGSZj5Gr\!"GS[\T/\_5GSdGS\j5Gr\!"GS]\T/\_5GSdGS^j5Gr\!"GS_\T/\_5GSdGS`j5Gr\!"GSa\T/\_5GSdGSbj5Gr\!"GSc\T/\_5GSdGSdj5Gr\!"GSe\T/\R5GSeGSfj5Gr\!"GSg\T/\R5GSeGShj5Gr\!"GSi\T/\R5GSeGSjj5Gr\!"GSk\T/\_5GSdGSlj5Gr\!"GSm\T/\_5GSdGSnj5Gr\!"GSo\T/\_5GSdGSpj5Gr\!"GSq\T/\_5GSdGSrj5Gr\!"GSs\T/\_5GSdGStj5Gr\!"GSu\T/\_5GSdGSvj5Gr\!"GSw\T\_/\GR5GS`GSxj5Gr\!"GSy\T\_/\GR5GSaGSzj5Gr\!"GS{\T\_/\GR5GSbGS|j5Gr\!"GS}\T\_/\R5GScGS~j5Gr\!"GS\T\_/\R5GScGSj5Gr\!"GS\T/\_5GS.GSj5Gr\!"GS\T/\_5GS.GSj5Gr\!"GS\T/\_5GS.GSj5Gr\!"GS\T/\_5GS.GSj5Gr\!"GS\T/\_5GS.GSj5Gr\!"GS\T/\_5GS.GSj5Gr\!"GS\T/\_5GS.GSj5Gr\!"GS\T/\R5GSeGSj5Gr\!"GS\T/\R5GSeGSj5Gr\!"GS\T/\_5GSdGSj5Gr\!"GS\T/\_5GSdGSj5Gr\!"GS\T/\_5GSdGSj5Gr\!"GS\T/\_5GSdGSj5Gr\!"GS\T/\_5GSdGSj5Gr\!"GS\T/\_5GSdGSj5Gr\!"GS\T/\_5GSdGSj5Gr\!"GS\T\GR\R\a\R\R\R/\R5GSfGSj5Gr\!"GS\T\_\GR\R\R\R/\R5GSgGSj5Gr\!"GS\T\R"\_5\R\GR\R\R\R/\R5GShGSj5Gr\!"GS\GR\R"\5\R\R\GR\R/\R5GSiGSj5Gr\!"GS\R"\GR5\R/\R5GSjGSj5Gr\RGr"GSGS\GR5Gr"GSGS\GR5Gr\ (a\G\Gr\R"G\5Gr\R~"\GRG\5Gr\R~"SG\\_5Gr\R~"SG\\5Gr\R~"SG\5Gr\R~"G\G\5Gr\R~"SG\5GrSG\4GSG\4GSG\4GSG\4GSG\4GSG\4/G\Gl\!"GS\R"G\5G\/G\5GSkGSj5Gr\!"GSG\/\GR5GSlGSj5Gr\!"GSG\\_/S5GSmGSj5Gr\!"GSG\\/S5GSnGSj5Gr\!"GSG\/S5GSoGSj5Gr\!"GSG\/G\5GSpGSj5Gr\!"GSG\/S5GSoGSj5Gr\!"GS\/G\5GSqGSj5Gr\!"GSG\G\/S5GSrGSj5Gr\!"GSG\\R/G\5GSsGSj5Gr\!"GSG\/\GR5GStGSj5Gr\!"GSG\\R/G\5GSsGSj5Gr\!"GS/G\5GSuGSj5Gr\!"GS\GR&/G\5GSvGSj5Gr\!"GS/G\5GSuGSj5Gr\!"GS\R/G\5GSwGSj5Gr\!"GS\GR\R/G\5GSxGSj5Gr\!"GS\GR\R/G\5GSxGSj5Gr\!"GS\GR\R/G\5GSxGSj5Gr\!"GS\GR/G\5GSyGSj5Gr\!"GS\GR\GR\GR/G\5GSzGSj5Gr\!"GS\GR\GR\R\GR&/G\5GS{GSj5Gr\!"GS\GR/G\5GS|GSj5Gr\!"GS\R\GR&\GR\GR\R/G\5GS}GSj5Gr\!"GS\GR&\GR\GR/G\5GS~GSj5Gr\!"GS\T\GR\GR/G\5GSGSj5Gr\!"GS\T\GR\GR\R"\GR5\R\R"\_5\R/G\5GSGSj5Gr\!"GS\T\GR\GR\R"\GR5\R\R"\_5\R/G\5GSGSj5Gr\!"GS\R\GR\GR\GR/G\5GSGSj5Gr\!"GS\T\R\GR\GR\R\R\R"\GR5\R/G\5GSGSj5Gr\!"GS\R\R\/G\5GSGSj5Gr\!"GS\T/G\5GSGSj5Gr\!"GSG\/\GR&5GSGSj5Gr\!"GSG\\X\R/\_5GSGSj5Gr\!"GS\GR\R\GR\GR\GR/\GR5GSGSj5Gr\!"GS\GR\R\GR\GR\GR/\GR5GSGSj5Gr\!"GS/\GR5GSGSj5Gr\!"GS\R\R/S5GSGSj5Gr"GSGS\GR5Gr"GSGS\GR5Gr\!"GS\X/G\5GSGSj5Gr\!"GS\X/S5GS"GSj5Gr\!"GS\X/S5GS"GS j5Gr\!"GS G\/G\5GSGS j5Gr\!"GS G\/S5GSGS j5Gr\!"GSG\/S5GSGSj5Gr\R~"\R\R\R5Gr\!"GS\R\R/\R5GSGSj5Gr"GSGS\GR5Gr\!"GS\X\VG\/S5GSGSj5Gr\!"GS\X\R\R\R\GR\R\R/S5GSGSj5Grg() annotationsN)CallableUnionNewTypeOptional TYPE_CHECKING)load_shared_librarybyref"ctypes_function_for_shared_library) CtypesCData CtypesArray CtypesPointerCtypesVoidPointer CtypesRefCtypesPointerOrRefCtypesFuncPointerllamaLLAMA_CPP_LIB_PATHlib lialgginsggiqsgg llama_vocab_p llama_model_pllama_context_pllama_memory_tllama_kv_cache_p !"#@$%&ic\rSrSr%Sr\(aS\S'S\S'S\S'S\4S\R4S\R4/r Sr g ) llama_token_dataizUsed to store token data Attributes: id (llama_token): token id logit (float): log-odds of the token p (float): probability of the token llama_tokenidfloatlogitpN __name__ __module__ __qualname____firstlineno____doc__r__annotations__rHctypesc_float_fields___static_attributes__rMQ/var/www/html/ai-backend/venv/lib/python3.13/site-packages/llama_cpp/llama_cpp.pyrGrGsD/   { &..! fnnHrYrGc\rSrSr%Sr\(aS\S'S\S'S\S'S\S 'S\4S\R4S\R4S \R4/r S r g ) llama_token_data_arrayrBzUsed to sample tokens given logits Attributes: data (ctypes.Array[llama_token_data]): token data size (int): size of the array selected (int): index in the data array (i.e. not the token id) sorted (bool): whether the array is sortedzCtypesArray[llama_token_data]dataintsizeselectedboolsortedrMN)rOrPrQrRrSrrTllama_token_data_prUc_size_tc_int64c_boolrWrXrMrYrZr\r\sW6++    #$ ! V^^$ 6==! HrYr\c \rSrSr%Sr\(a#S\S'S\S'S\S'S \S 'S \S 'S \S'S\S'S\R4S\R"\ 54S\R"\R54S \R"\ 54S \R"\R54S\R"\R"\ 554S\R"\R54/rSrg) llama_batchi9aInput data for llama_encode/llama_decode A llama_batch object can contain input about one or many sequences The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens Attributes: n_tokens (int): number of tokens token (ctypes.Array[llama_token]): the token ids of the input (used when embd is NULL) embd (ctypes.Array[ctypes.ctypes.c_float]): token embeddings (i.e. float vector of size n_embd) (used when token is NULL) pos (ctypes.Array[ctypes.Array[llama_pos]]): the positions of the respective token in the sequence seq_id (ctypes.Array[ctypes.Array[llama_seq_id]]): the sequence to which the respective token belongs logits (ctypes.Array[ctypes.ctypes.c_int8]): if zero, the logits for the respective token will not be output r^n_tokensCtypesArray[llama_token]tokenCtypesArray[ctypes.c_float]embdz#CtypesArray[CtypesArray[llama_pos]]poszCtypesArray[ctypes.c_int]n_seq_idz&CtypesArray[CtypesArray[llama_seq_id]]seq_idzCtypesArray[ctypes.c_int8]logitsrMN)rOrPrQrRrSrrTrUc_int32POINTERrHrV llama_pos llama_seq_idc_int8rWrXrMrYrZrhrh9s  '')) 00++66** V^^$ &..-. /0 y)* V^^FNN34 6>>&..">?@ 6>>&--01HrYrhc\rSrSr%S\R 4S\R 4S\R4S\RS-4/r \ (aS\ S'S\ S'S \ S'S \ S'S r g S r g ) llama_model_kv_override_valueivval_i64val_f64val_boolval_strr@r^rJrabytesrMN) rOrPrQrRrUrec_doublerfc_charrWrrTrXrMrYrZrxrxvs\ FNN# FOO$ V]]# FMMC'( H  rYrxc\rSrSr%S\R 4S\R S-4S\4/r\ (aS\ S'S\ S'S\ S'S r g S r g ) llama_model_kv_overrideitagkeyr@valuer^r}zUnion[int, float, bool, bytes]rMN) rOrPrQrRrUc_intrrxrWrrTrXrMrYrZrrsI    #$ /0H    --rYrc<\rSrSr%Sr\(aFS\S'S\S'S\S'S\S 'S\S 'S \S 'S \S'S\S'S\S'S\S'S\S'S\S'S\S'S\S'S\R4S\R4S\R4S \R4S \R4S \R"\R54S\ 4S\R4S\R"\54S\R4S\R4S\R4S\R4S\R4/rSrg)llama_model_paramsiaParameters for llama_model Attributes: devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern n_gpu_layers (int): number of layers to store in VRAM split_mode (int): how to split the model across multiple GPUs main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights use_mmap (bool): use mmap if possible use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking)zCtypesArray[ctypes.c_void_p]devicesz-CtypesArray[llama_model_tensor_buft_override]tensor_buft_overridesr^ n_gpu_layers split_modemain_gpurl tensor_splitz(Callable[[float, ctypes.c_void_p], bool]progress_callbackctypes.c_void_pprogress_callback_user_dataz$CtypesArray[llama_model_kv_override] kv_overridesra vocab_onlyuse_mmap use_mlock check_tensorsuse_extra_buftsrMN)rOrPrQrRrSrrTrUc_void_prrrrsrVllama_progress_callbackrrfrWrXrMrYrZrrsV$--LL 11CC%44:: FOO$ &//2 ( v||$ V^^$ 78 56 &8 (?@A v}}% V]]# fmm$ &--( FMM*HrYrc<\rSrSr%Sr\(aS\S'S\S'S\S'S\S'S\S'S\S 'S\S 'S\S 'S\S 'S \S'S \S'S \S'S \S'S \S'S \S'S\S'S \S'S\S'S\S'S\S'S\S'S\S'S\S'S\S 'S\S!'S\S"'S\S#'S\S$'S\S%'S\S&'S\R4S\R4S\R4S\R4S\R4S \R4S \R4S \R4S \R4S\R4S\R4S\R4S\R4S\R4S\R4S\R4S\R4S\ 4S\R4S\R4S\R4S\4S\R4S \R4S!\R4S"\R4S#\R4S$\R4S%\R4S&\R4/rS'rg())llama_context_paramsi a[Parameters for llama_context Attributes: n_ctx (int): text context, 0 = from model n_batch (int): logical maximum batch size that can be submitted to llama_decode n_ubatch (int): physical maximum batch size n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type` pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) attention_type (int): attention type to use for embeddings rope_freq_base (float): RoPE base frequency, 0 = from model rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model yarn_attn_factor (float): YaRN magnitude scaling factor yarn_beta_fast (float): YaRN low correction dim yarn_beta_slow (float): YaRN high correction dim yarn_orig_ctx (int): YaRN original context size defrag_thold (float): defragment the KV cache if holes/size > thold, <= 0 disabled (default) cb_eval (ggml_backend_sched_eval_callback): callback for scheduling eval cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval type_k (int): data type for K cache type_v (int): data type for V cache abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU flash_attn (bool): whether to use flash attention no_perf (bool): whether to measure performance timings op_offload (bool): offload host tensor operations to device swa_full (bool): use full-size SWA cache kv_unified (bool): use a unified buffer across the input sequences when computing the attention r^n_ctxn_batchn_ubatch n_seq_max n_threadsn_threads_batchrope_scaling_type pooling_typeattention_typerJrope_freq_baserope_freq_scaleyarn_ext_factoryarn_attn_factoryarn_beta_fastyarn_beta_slow yarn_orig_ctx defrag_tholdz'Callable[[ctypes.c_void_p, bool], bool]cb_evalrcb_eval_user_datatype_ktype_vz!Callable[[ctypes.c_void_p], bool]abort_callbackabort_callback_datara embeddings offload_kqv flash_attnno_perf op_offloadswa_full kv_unifiedrMN)rOrPrQrRrSrrTrUc_uint32rrrrV ggml_backend_sched_eval_callbackrggml_abort_callbackrfrWrXrMrYrZrr s#!F   88**  99,,  &//" FOO$ V__% foo& fnn% FNN+ fll+ & 6<<( 6>>* FNN+ FNN+ V^^, 6>>* 6>>* &//* ( 45 foo. 6<< 6<< ./ 0 v}}%  & v}}% FMM" v}}% V]]# v}}%=HrYrc\rSrSr%Sr\(aAS\S'S\S'S\S'S\S'S\S 'S\S 'S\S 'S\S 'S\S 'S\S'S\S'S\S'S\S'S\R4S\R4S\R4S\R4S \R4S \R4S \R4S \R4S \R4S\R4S\R4S\R4S\R4/ r Sr g)llama_model_quantize_paramsiaParameters for llama_model_quantize Attributes: nthread (int): number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() ftype (int): quantize to this llama_ftype output_tensor_type (int): output tensor type token_embedding_type (int): token embeddings tensor type allow_requantize (bool): allow quantizing non-f32/f16 tensors quantize_output_tensor (bool): quantize output.weight only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored pure (bool): quantize all tensors to the default type keep_split (bool): quantize to the same number of shards imatrix (ctypes.c_void_p): pointer to importance matrix data kv_overrides (ctypes.c_void_p): pointer to vector containing overrides tensor_types (ctypes.c_void_p): pointer to vector containing tensor types prune_layers (ctypes.c_void_p): pointer to vector containing layer indices to prune r^nthreadftypeoutput_tensor_typetoken_embedding_typeraallow_requantizequantize_output_tensor only_copypure keep_splitrimatrixr tensor_types prune_layersrMN)rOrPrQrRrSrrTrUrrrrfrrWrXrMrYrZrrs$  !! $$   %%%%%% FNN# &,, v||, . V]]+ !6==1 fmm$  v}}% FOO$ ) ) )HrYrc`\rSrSr%Sr\(a S\S'S\S'S\4S\R4/r Sr g) llama_logit_biasiz^Used to store logit bias Attributes: token (llama_token): token id bias (float): biasrHrkrJbiasrMNrNrMrYrZrrs4   +  HrYrcP\rSrSr%Sr\(aS\S'S\R4/r Sr g)llama_sampler_chain_paramsizjParameters for llama_sampler_chain Attributes: no_perf (bool): whether to measure performance timingsrarrMN) rOrPrQrRrSrrTrUrfrWrXrMrYrZrrs'B   FMM"HrYrcL\rSrSrS\R 4S\R 4/rSrg)llama_chat_messageirolecontentrMN)rOrPrQrRrUc_char_prWrXrMrYrZrrs! ! FOO$HrYrllama_model_default_paramscg)z&Get default parameters for llama_modelNrMrMrYrZrrrYllama_context_default_paramscg)z(Get default parameters for llama_contextNrMrMrYrZrr!rrY"llama_sampler_chain_default_paramscg)z.Get default parameters for llama_sampler_chainNrMrMrYrZrr,rrY#llama_model_quantize_default_paramscg)z/Get default parameters for llama_model_quantizeNrMrMrYrZrr7rrYllama_backend_initcg)zIInitialize the llama + ggml backend Call once at the start of the programNrMrMrYrZrrErYllama_backend_freecg)zACall once at the end of the program - currently only used for MPINrMrMrYrZrrcrrYllama_numa_initcgNrM)numas rZrro rYllama_load_model_from_filecgrrM path_modelparamss rZrrrrYllama_model_load_from_filecg)zLoad the model from a file If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf If the split file name does not follow this pattern, use llama_model_load_from_splitsNrMrs rZrrrYllama_model_load_from_splitscg)zjLoad the model from multiple splits (support custom naming scheme) The paths must be in the correct orderNrM)pathsn_pathsrs rZrrrYllama_model_save_to_filecg)zSave the model to a fileNrM)modelrs rZrrrrYllama_free_modelcgrrMrs rZrrrrYllama_model_freecgrrMrs rZrrrrYllama_init_from_modelcgrrMrrs rZrrrrYllama_new_context_with_modelcgrrMrs rZrrrrY llama_freecg)zFrees all allocated memoryNrMctxs rZrrrrY llama_time_uscgrrMrMrYrZrr rrYllama_max_devicescgrrMrMrYrZrrrYllama_max_parallel_sequencescgrrMrMrYrZr r r rYllama_supports_mmapcgrrMrMrYrZr r r rYllama_supports_mlockcgrrMrMrYrZrr%r rYllama_supports_gpu_offloadcgrrMrMrYrZrr+r rYllama_supports_rpccgrrMrMrYrZrr1r rY llama_n_ctxcgrrMrs rZrr7r rY llama_n_batchcgrrMrs rZrr=r rYllama_n_ubatchcgrrMrs rZrrCr rYllama_n_seq_maxcgrrMrs rZrrIr rYllama_n_ctx_traincgrrMrs rZrrOr rY llama_n_embdcgrrMrs rZrrUr rY llama_n_layercgrrMrs rZr r [r rY llama_n_headcgrrMrs rZr"r"ar rY llama_n_vocabcgrrMrs rZr$r$gr rYllama_get_modelcgrrMrs rZr&r&mr rYllama_get_memorycg)zGet the memory for the contextNrMrs rZr(r(srYllama_pooling_typecgrrMrs rZr+r+zr rYllama_get_kv_selfcg)z0Get the KV cache for self-attention (DEPRECATED)NrMrs rZr-r-rrYllama_model_get_vocabcgrrMrs rZr/r/r rYllama_model_rope_typecgrrMrs rZr1r1r rYllama_model_n_ctx_traincgrrMrs rZr3r3r rYllama_model_n_embdcgrrMrs rZr5r5r rYllama_model_n_layercgrrMrs rZr7r7r rYllama_model_n_headcgrrMrs rZr9r9r rYllama_model_n_head_kvcgrrMrs rZr;r;r rYllama_model_n_swacgrrMrs rZr=r=r rY!llama_model_rope_freq_scale_traincgrrMrs rZr?r?r rYllama_model_n_cls_outcg)zKReturns the number of classifier outputs (only valid for classifier models)NrMrs rZrArAr*rYllama_model_cls_labelcg)zNReturns label of classifier output by index. Returns None if no label providedNrM)ris rZrCrCr*rYllama_vocab_typecgrrMvocabs rZrFrFr rYllama_vocab_n_tokenscgrrMrHs rZrJrJr rYllama_model_meta_val_strcg)z*Get metadata value as a string by key nameNrM)rrbufbuf_sizes rZrLrL$rYllama_model_meta_countcg)z*Get the number of metadata key/value pairsNrMrs rZrQrQr*rYllama_model_meta_key_by_indexcg)zGet metadata key name by indexNrMrrErNrOs rZrSrSrPrY!llama_model_meta_val_str_by_indexcg)z'Get metadata value as a string by indexNrMrUs rZrVrVrPrYllama_model_desccg)z&Get a string describing the model typeNrM)rrNrOs rZrXrX4rYllama_model_sizecg)z?Returns the total size of all the tensors in the model in bytesNrMrs rZr[r[Er*rYllama_model_chat_templatenamecg)zoGet the default chat template. Returns None if not available If name is None, returns the default chat templateNrM)rr^s rZr]r]NrYllama_model_n_paramscg)z3Returns the total number of parameters in the modelNrMrs rZraraWr*rYllama_model_has_encodercg)zOReturns true if the model contains an encoder that requires llama_encode() callNrMrs rZrcrc_r*rYllama_model_has_decodercg)zNReturns true if the model contains a decoder that requires llama_decode() callNrMrs rZreregr*rYllama_model_decoder_start_tokencg)zFor encoder-decoder models, this function returns id of the token that must be provided to the decoder to start generating output sequence. For other models, it returns -1. NrMrs rZrgrgprrYllama_model_is_recurrentcg)z?Returns true if the model is recurrent (like Mamba, RWKV, etc.)NrMrs rZriri|r*rYllama_model_is_diffusioncg)zFReturns true if the model is diffusion-based (like LLaDA, Dream, etc.)NrMrs rZrkrkr*rYllama_model_quantizecg)zReturns 0 on successNrM) fname_inp fname_outrs rZrmrm rYllama_adapter_lora_initcgrrM)r path_loras rZrrrrrrYllama_adapter_lora_freecgrrM)adapters rZrururrYllama_set_adapter_loracg)zNAdd a loaded LoRA adapter to given context This will not modify model's weightNrM)rrwscales rZrxrxrYllama_rm_adapter_loracg)zhRemove a specific LoRA adapter from given context Return -1 if the adapter is not present in the contextNrM)rrws rZr|r|r{rYllama_clear_adapter_loracg)z+Remove all LoRA adapters from given contextNrMrs rZr~r~rrYllama_apply_adapter_cveccg)aApply a loaded control vector to a llama_context, or if data is NULL, clear the currently loaded vector. n_embd should be the size of a single layer's control, and data should point to an n_embd x n_layers buffer starting from layer 1. il_start and il_end are the layer range the vector should apply to (both inclusive) See llama_control_vector_load in common to load a control vector.NrM)rr]lenn_embdil_startil_ends rZrrs6rYllama_memory_clearcg)zkClear the memory contents If data == true, the data buffers will also be cleared together with the metadataNrM)memr]s rZrr)rrYllama_memory_seq_rmcg)aRemoves all tokens that belong to the specified sequence and have positions in [p0, p1) Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails seq_id < 0 : match any sequence p0 < 0 : [0, p1] p1 < 0 : [p0, inf)NrM)rrpp0p1s rZrr>s0rYllama_memory_seq_cpcg)znCopy all tokens that belong to the specified sequence to another sequence p0 < 0 : [0, p1] p1 < 0 : [p0, inf)NrM)r seq_id_src seq_id_dstrrs rZrrb,rYllama_memory_seq_keepcg)z?Removes all tokens that do not belong to the specified sequenceNrMrrps rZrr rYllama_memory_seq_addcg)zAdds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) p0 < 0 : [0, p1] p1 < 0 : [p0, inf)NrM)rrprrdeltas rZrrrrYllama_memory_seq_divcg)z[Integer division of the positions by factor of `d > 1` p0 < 0 : [0, p1] p1 < 0 : [p0, inf)NrM)rrprrds rZrrrrYllama_memory_seq_pos_mincg)zReturns the smallest position present in the memory for the specified sequence This is typically non-zero only for SWA caches Return -1 if the sequence is emptyNrMrs rZrrrYllama_memory_seq_pos_maxcg)zpReturns the largest position present in the memory for the specified sequence Return -1 if the sequence is emptyNrMrs rZrrrrYllama_memory_can_shiftcg)z%Check if the memory supports shiftingNrM)rs rZrrr*rYllama_kv_self_n_tokenscg)zTReturns the number of tokens in the KV cache (slow, use only for debug) (DEPRECATED)NrMrs rZrrrrYllama_kv_self_used_cellscg)z0Returns the number of used KV cells (DEPRECATED)NrMrs rZrr rrYllama_kv_self_clearcg)zClear the KV cache (DEPRECATED)NrMrs rZrrrrYllama_kv_self_seq_rmcg)z(Remove tokens from KV cache (DEPRECATED)NrM)rrprrs rZrr)rPrYllama_kv_self_seq_cpcg)z$Copy tokens in KV cache (DEPRECATED)NrM)rrrrrs rZrrI(rYllama_kv_self_seq_keepcg)z5Keep only specified sequence in KV cache (DEPRECATED)NrMrrps rZrrerrYllama_kv_self_seq_addcg)z8Add delta to sequence positions in KV cache (DEPRECATED)NrM)rrprrrs rZrryrrYllama_kv_self_seq_divcg)z2Divide sequence positions in KV cache (DEPRECATED)NrM)rrprrrs rZrrrrYllama_kv_self_seq_pos_mincg)zCReturns the smallest position in KV cache for sequence (DEPRECATED)NrMrs rZrrrrYllama_kv_self_seq_pos_maxcg)zBReturns the largest position in KV cache for sequence (DEPRECATED)NrMrs rZrrrrYllama_kv_self_defragcg)z$Defragment the KV cache (DEPRECATED)NrMrs rZrrr*rYllama_kv_self_can_shiftcg)z rrYllama_set_state_datacg)z=Set the state reading from the specified address (DEPRECATED)NrM)rrs rZrrR rrYllama_state_load_filecgrrMr path_session tokens_outn_token_capacityn_token_count_outs rZrre &rYllama_load_session_filecgrrMrs rZrr rrYllama_state_save_filecgrrMrrtokens n_token_counts rZrr "rYllama_save_session_filecgrrMrs rZrr rrYllama_state_seq_get_sizecg)z@Get the exact size needed to copy the state of a single sequenceNrMrs rZrr rrYllama_state_seq_get_datacg)z=Copy the state of a single sequence into the specified bufferNrM)rrr_rps rZrr rPrYllama_state_seq_set_datacg)z2Copy the sequence data into the specified sequenceNrM)rrr_ dest_seq_ids rZrr rPrYllama_state_seq_save_filecgrrM)rfilepathrprrs rZrr rrYllama_state_seq_load_filecgrrM)rrrrrrs rZrr5 s*rYllama_batch_get_onecg)zReturn batch for single sequence of tokens NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it NrM)rris rZrrZ rrYllama_batch_initcg)aAllocates a batch of tokens on the heap that can hold a maximum of n_tokens Each token can be assigned up to n_seq_max sequence ids The batch has to be freed with llama_batch_free() If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) Otherwise, llama_batch.token will be allocated to store n_tokens llama_token The rest of the llama_batch members are allocated with size n_tokens All members are left uninitializedNrM)rirmrs rZrry rqrYllama_batch_freecg)z9Frees a batch of tokens allocated with llama_batch_init()NrM)batchs rZrr r*rY llama_encodecg)zDProcess a batch of tokens using the encoder. 0 - success < 0 - errorNrMrrs rZrr rrY llama_decodecg)a8Process a batch of tokens. 0 - success 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) 2 - aborted (processed ubatches will remain in the context's memory) -1 - invalid input batch < -1 - fatal error (processed ubatches will remain in the context's memory)NrMrs rZrr rrYllama_set_n_threadscg)zSet the number of threads used for decoding n_threads is the number of threads used for generation (single token) n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) NrM)rrrs rZrr rrYllama_n_threadscg)z?Get the number of threads used for generation of a single tokenNrMrs rZrr r*rYllama_n_threads_batchcg)zOGet the number of threads used for prompt and batch processing (multiple token)NrMrs rZrr r*rYllama_set_embeddingscg)z1Set whether the context outputs embeddings or notNrM)rrs rZrr r*rYllama_set_causal_attncg)zhSet whether to use causal attention or not If set to true, the model will only attend to the past tokensNrM)r causal_attns rZrr r`rYllama_set_warmupcg)zSet whether the model is in warmup mode or not If true, all model tensors are activated during llama_decode() to load and cache their weights.NrM)rwarmups rZrr r`rYllama_set_abort_callbackcg)zSet abort callbackNrM)rrrs rZrr rZrYllama_synchronizecg)zWait until all computations are finished This is automatically done when using one of the functions below to obtain the computation results and is not necessary to call it explicitly in most casesNrMrs rZrr rrYllama_get_logitscg)aBToken logits obtained from the last call to llama_decode() The logits for which llama_batch.logits[i] != 0 are stored contiguously in the order they have appeared in the batch. Rows: number of tokens for which llama_batch.logits[i] != 0 Cols: n_vocab Returns: Pointer to the logits buffer of shape (n_tokens, n_vocab)NrMrs rZr r * rZrYllama_get_logits_ithcg)zJLogits for the ith token. Equivalent to: llama_get_logits(ctx) + i*n_vocabNrMrrEs rZr r > r{rYllama_get_embeddingscg)z@Get the embeddings for the input shape: [n_embd] (1-dimensional)NrMrs rZrrS rrYllama_get_embeddings_ithcg)zLGet the embeddings for the ith sequence llama_get_embeddings(ctx) + i*n_embdNrMr s rZrrb r{rYllama_get_embeddings_seqcg)z|Get the embeddings for a sequence id Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE shape: [n_embd] (1-dimensional)NrMrs rZrrt rrYllama_vocab_get_textcgrrMrIrks rZrr rrYllama_vocab_get_scorecgrrMrs rZrr rrYllama_vocab_get_attrcgrrMrs rZrr rrYllama_vocab_is_eogcg)zXCheck if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)NrMrs rZrr rrYllama_vocab_is_controlcg)z>Identify if Token Id is a control token or a render-able tokenNrMrs rZrr rrYllama_vocab_boscg)zbeginning-of-sentenceNrMrHs rZrr r*rYllama_vocab_eoscg)zend-of-sentenceNrMrHs rZr!r! r*rYllama_vocab_eotcg)z end-of-turnNrMrHs rZr#r# r*rYllama_vocab_sepcg)zsentence separatorNrMrHs rZr%r% r*rYllama_vocab_nlcg)z next-lineNrMrHs rZr'r' r*rYllama_vocab_padcg)paddingNrMrHs rZr)r) r*rYllama_vocab_maskcg)maskNrMrHs rZr,r, r*rYllama_vocab_get_add_boscgrrMrHs rZr/r/ rrYllama_vocab_get_add_eoscgrrMrHs rZr1r1 rrYllama_vocab_get_add_sepcgrrMrHs rZr3r3 rrYllama_vocab_fim_precgrrMrHs rZr5r5 rrYllama_vocab_fim_sufcgrrMrHs rZr7r7 rrYllama_vocab_fim_midcgrrMrHs rZr9r9 rrYllama_vocab_fim_padcgrrMrHs rZr;r;) rrYllama_vocab_fim_repcgrrMrHs rZr=r=3 rrYllama_vocab_fim_sepcgrrMrHs rZr?r?= rrYllama_token_get_textcgrrMrs rZrArAH rrYllama_token_get_scorecgrrMrs rZrCrCT rrYllama_token_get_attrcgrrMrs rZrErE_ rrYllama_token_is_eogcgrrMrs rZrGrGj rrYllama_token_is_controlcgrrMrs rZrIrIu rrYllama_token_boscgrrMrHs rZrKrK rrYllama_token_eoscgrrMrHs rZrMrM rrYllama_token_eotcgrrMrHs rZrOrO rrYllama_token_clscgrrMrHs rZrQrQ rrYllama_token_sepcgrrMrHs rZrSrS rrYllama_token_nlcgrrMrHs rZrUrU rrYllama_token_padcgrrMrHs rZrWrW rrYllama_add_bos_tokencgrrMrHs rZrYrY rrYllama_add_eos_tokencgrrMrHs rZr[r[ rrYllama_token_fim_precgrrMrHs rZr]r] rrYllama_token_fim_sufcgrrMrHs rZr_r_ rrYllama_token_fim_midcgrrMrHs rZrara rrYllama_token_fim_padcgrrMrHs rZrcrc rrYllama_token_fim_repcgrrMrHs rZrere rrYllama_token_fim_sepcgrrMrHs rZrgrg rrYllama_vocab_clscgrrMrHs rZriri rrYllama_tokenizecg)aeConvert the provided text into tokens. Args: vocab: The vocabulary to use for tokenization. text: The text to tokenize. text_len: The length of the text. tokens: The tokens pointer must be large enough to hold the resulting tokens. n_max_tokens: The maximum number of tokens to return. add_special: Allow adding special tokens if the model is configured to do so. parse_special: Allow parsing special tokens. Returns: Returns the number of tokens on success, no more than n_tokens_max Returns a negative number on failure - the number of tokens that would have been returned NrM)rItexttext_lenr n_tokens_max add_special parse_specials rZrkrk, sLrYllama_token_to_piececg)aToken Id -> Piece. Uses the vocabulary in the provided context. Does not write null terminator to the buffer. User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens. Args: vocab: The vocabulary to use for tokenization. token: The token to convert. buf: The buffer to write the token to. length: The length of the buffer. lstrip: The number of leading spaces to skip. special: If true, special tokens are rendered in the output.NrM)rIrkrNlengthlstripspecials rZrrrra BrYllama_detokenizecg)aConvert the provided tokens into text (inverse of llama_tokenize()). Args: vocab: The vocabulary to use for tokenization. tokens: The tokens to convert. n_tokens: The number of tokens. text: The buffer to write the text to. text_len_max: The length of the buffer. remove_special: Allow to remove BOS and EOS tokens if model is configured to do so. unparse_special: If true, special tokens are rendered in the output.NrM)rIrrirm text_len_maxremove_specialunparse_specials rZrxrx rwrYllama_chat_apply_templatecg)aBApply chat template. Args: tmpl: Template to use. If None, uses model's default chat: Array of chat messages n_msg: Number of messages add_ass: Whether to end prompt with assistant token buf: Output buffer length: Buffer length Returns: Number of bytes written, or needed if buffer too small NrM)tmplchatn_msgadd_assrNrts rZr}r} sDrYllama_chat_builtin_templatescg)zGet list of built-in chat templates. Args: output: Output buffer to store template names. len: Length of the output buffer. Returns: Number of templates available. Returns a negative number on error. NrM)outputrs rZrr .rYc\rSrSrSrg)llama_sampler_ii!rMN)rOrPrQrRrXrMrYrZrr!srYrcD\rSrSrS\R "\54S\4/rSr g) llama_sampleri)ifacerrMN) rOrPrQrRrUrsrllama_sampler_context_trWrXrMrYrZrr)s# &..12 '(HrYracceptapplyresetclonefreellama_sampler_initcgrrM)rrs rZrrJrrYllama_sampler_namecgrrMsmpls rZrrVrrYllama_sampler_acceptcgrrM)rrks rZrr`rrYllama_sampler_applycgrrM)rcur_ps rZrrjrrYllama_sampler_resetcgrrMrs rZrrvrrYllama_sampler_clonecgrrMrs rZrrrrYllama_sampler_freecgrrMrs rZrrrrYllama_sampler_chain_initcgrrM)rs rZrrrrYllama_sampler_chain_addcgrrM)chainrs rZrrrrYllama_sampler_chain_getcgrrMrrEs rZrrrrYllama_sampler_chain_ncgrrMrs rZrrrrYllama_sampler_chain_removecgrrMrs rZrrrrYllama_sampler_init_greedycgrrMrMrYrZrrr rYllama_sampler_init_distcgrrM)seeds rZrrr rYllama_sampler_init_softmaxcgrrMrMrYrZrrr rYllama_sampler_init_top_kcgrrM)ks rZrrr rYllama_sampler_init_top_pcgrrMrLmin_keeps rZrrrrYllama_sampler_init_min_pcgrrMrs rZrrrrYllama_sampler_init_typicalcgrrMrs rZrrrrYllama_sampler_init_tempcgrrM)ts rZrrr rYllama_sampler_init_temp_extcgrrM)rrexponents rZrrrrYllama_sampler_init_xtccgrrM)rLrrrs rZrr%rrYllama_sampler_init_top_n_sigmacgrrM)ns rZrr2rrYllama_sampler_init_mirostatcgrrM)n_vocabrtauetams rZrrBrrYllama_sampler_init_mirostat_v2cgrrM)rrrs rZrrRrrYllama_sampler_init_grammarcgrrM)rI grammar_str grammar_roots rZrrbrrYllama_sampler_init_grammar_lazycgrrM)rIrr trigger_wordsnum_trigger_wordstrigger_tokensnum_trigger_tokenss rZrrvrrY(llama_sampler_init_grammar_lazy_patternscgrrM)rIrrtrigger_patternsnum_trigger_patternsrrs rZrrrrYllama_sampler_init_penaltiescgrrM)penalty_last_npenalty_repeat penalty_freqpenalty_presents rZrrrZrYllama_sampler_init_drycgrrM)rI n_ctx_traindry_multiplierdry_basedry_allowed_lengthdry_penalty_last_n seq_breakers num_breakerss rZrrs2rYllama_sampler_init_logit_biascgrrM)r n_logit_bias logit_biass rZrrrrYllama_sampler_init_infillcgrrMrHs rZrrrrYllama_sampler_get_seedcgrrMrs rZrr rrYllama_sampler_samplecgrrM)rridxs rZrrrrYllama_split_pathcg)z-Build a split GGUF final path for this chunk.NrM) split_pathmaxlen path_prefixsplit_no split_counts rZrr&rYllama_split_prefixcg)z^Extract the path prefix from the split_path if and only if the split_no and split_count match.NrM) split_prefixrrrrs rZrr9rrYllama_print_system_infocgrrMrMrYrZrrLr rY llama_log_setcg)zxSet callback for all future logging events. If this is not called, or NULL is supplied, everything is output on stderr.NrM) log_callback user_datas rZr r TrrYc\rSrSrS\R 4S\R 4S\R 4S\R 4S\R 4S\R 4S\R 4/rS rg ) llama_perf_context_datair t_start_ms t_load_ms t_p_eval_ms t_eval_msn_p_evaln_evaln_reusedrMN rOrPrQrRrUr~rrrWrXrMrYrZrrrs] v' foo& ( foo& V^^$ 6>>" V^^$HrYrcL\rSrSrS\R 4S\R 4/rSrg)llama_perf_sampler_datai t_sample_msn_samplerMNrrMrYrZrrs! ( V^^$HrYrllama_perf_contextcgrrMrs rZrrrrYllama_perf_context_printcgrrMrs rZrrrrYllama_perf_context_resetcgrrMrs rZr r rrYllama_perf_samplercgrrMrs rZr"r"rrYllama_perf_sampler_printcgrrMrs rZr$r$rrYllama_perf_sampler_resetcgrrMrs rZr&r&rrYllama_opt_param_filter_allcgrrM)tensoruserdatas rZr(r(rrYc\rSrSrS\R 4S\4S\R4S\R4S\R4/rSr g) llama_opt_paramsir param_filterparam_filter_ud get_opt_parsget_opt_pars_udrMN) rOrPrQrRrUrllama_opt_param_filterrrWrXrMrYrZr-r-sB ( /0 FOO, ) FOO, HrYr-llama_opt_initcgrrM)lctxr lopt_paramss rZr3r3rrYllama_opt_epochcgrrM)r5dataset result_train result_eval idata_splitcallback_train callback_evals rZr7r7rrY)returnr)r?r)r?r)r?r)rr^)rr}rrr?Optional[llama_model_p])rz List[bytes]rr^rrr?r@)rr4rr})rr4)rr4rrr?zOptional[llama_context_p])rr5)r?r^)r?ra)rr5r?r^)rr4r?r^)rr3r?r^)rr5r?r@)rr5r?zOptional[llama_memory_t])rr5r?zOptional[llama_kv_cache_p])rr4r?zOptional[llama_vocab_p])rr4r?rJ)rr4rEr^r?Optional[bytes])rIr3r?r^) rr4rzUnion[ctypes.c_char_p, bytes]rNr}rOr^r?r^) rr4rEUnion[ctypes.c_int, int]rN(Union[bytes, CtypesArray[ctypes.c_char]]rOr^r?r^)rr4rNrCrOUnion[ctypes.c_size_t, int]r?r^)rr4r^rAr?rA)rr4r?ra)ror}rpr}rz/CtypesPointerOrRef[llama_model_quantize_params]r?r^)rr4rtr}r?zOptional[llama_adapter_lora_p])rwllama_adapter_lora_p)rr5rwrErzrJr?r^)rr5rwrEr?r^)rr5r]z"CtypesPointerOrRef[ctypes.c_float]rr^rr^rr^rr^r?r^)rr6r]ra) rr6rpUnion[llama_seq_id, int]rUnion[llama_pos, int]rrGr?ra) rr6rrFrrFrrGrrG)rr6rprF) rr6rprFrrGrrGrrG) rr6rprFrrGrrGrrB)rr6rprFr?r^)rr6r?ra) rr5rprFrrGrrGr?ra) rr5rrFrrFrrGrrG)rr5rprF) rr5rprFrrGrrGrrG) rr5rprFrrGrrGrrB)rr5rprFr?r^)rr5r?ra)rr5rCtypesArray[ctypes.c_uint8]r_rDr?r^)rr5rrHr?r^)rr5rrHr_rDr?r^)rr5rrHr?r^) rr5rr}rrjrrDr#CtypesPointerOrRef[ctypes.c_size_t]r?ra) rr5rr}rrjrrDr?ra)rr5rprur?r^) rr5rrHr_rDrprur?r^) rr5rrHr_rDrrur?r^) rr5rr}rprurrjrrDr?r^)rr5rr}rrurrjrrDrrIr?r^)rrjrirBr?rh)riUnion[ctypes.c_int32, int]rmrJrrJr?rh)rrh)rr5rrhr?r^)rr5rrJrrJ)rr5rra)rr5rra)rr5rra)rr5rz!Callable[[ctypes.c_void_p], None]rr)rr5r?rl)rr5rErJr?rl)rr5rprFr?rl)rIr3rkUnion[llama_token, int]r?r})rIr3rkrKr?rJ)rIr3rkrKr?r^)rIr3rkrKr?ra)rIr3r?rH)rIr3r?ra)rIr3rmr}rnrBrrjrorBrpUnion[ctypes.c_bool, bool]rqrLr?r^)rIr3rkrKrNz9Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]]rtrBrurBrvrLr?r^)rIr3rrjrirBrmr}rzrBr{rLr|rLr?r^)rr}rzCtypesArray[llama_chat_message]rr^rrarNr}rtr^r?r^)rCtypesArray[bytes]rrDr?r^)rzctypes.POINTER(llama_sampler_i)rrr?llama_sampler_p)rrNr?r})rrNrkrK)rrNrz#CtypesArray[llama_token_data_array])rrN)rrNr?rN)rrr?rN)rrNrrN)rrNrErJr?rN)rrNr?r^)r?rN)rr^r?rN)rr^r?rN)rLrJrr^r?rN)rrJr?rN)rrJrrJrrJr?rN) rLrJrrJrr^rr^r?rN)rrJr?rN) rr^rr^rrJrrJrr^r?rN)rr^rrJrrJr?rN)rIr3rr}rr}r?rN)rIr3rr}rr}rrMrr^rrjrr^r?rN)rIr3rr}rr}rrMrr^rrjrr^r?rN) rr^rrJrrJrrJr?rN)rIr3rr^rrJrrJrr^rr^rr^r?rN)rr^rr^rzCtypesArray[llama_logit_bias]r?rN)rIr3r?rN)rrNr?r^)rrNrr5rr^r?r^) rr}rrDrr}rrBrrBr?r^) rr}rrDrr}rrBrrBr?r^)r?r})r zOptional[CtypesFuncPointer]r r)rr5r?r)rrNr?r)rrN)r*rr+rr?ra)r5r5rr4r6r-)r5r5r9rr:rr;rr<r^r=rr>r( __future__rosrUpathlibtypingrrrrrllama_cpp._ctypes_extensionsr r r r r rrrrr_lib_base_nameenvironget_override_base_pathPathpathabspathdirname__file__ _base_path_libctypes_function GGML_TYPE_F32 GGML_TYPE_F16GGML_TYPE_Q4_0GGML_TYPE_Q4_1GGML_TYPE_Q5_0GGML_TYPE_Q5_1GGML_TYPE_Q8_0GGML_TYPE_Q8_1GGML_TYPE_Q2_KGGML_TYPE_Q3_KGGML_TYPE_Q4_KGGML_TYPE_Q5_KGGML_TYPE_Q6_KGGML_TYPE_Q8_KGGML_TYPE_IQ2_XXSGGML_TYPE_IQ2_XSGGML_TYPE_IQ3_XXSGGML_TYPE_IQ1_SGGML_TYPE_IQ4_NLGGML_TYPE_IQ3_SGGML_TYPE_IQ2_SGGML_TYPE_IQ4_XS GGML_TYPE_I8 GGML_TYPE_I16 GGML_TYPE_I32 GGML_TYPE_I64 GGML_TYPE_F64GGML_TYPE_IQ1_MGGML_TYPE_COUNT CFUNCTYPErfrrrrargtypesrdrestypeLLAMA_MAX_DEVICESLLAMA_DEFAULT_SEEDLLAMA_TOKEN_NULLLLAMA_FILE_MAGIC_GGLALLAMA_FILE_MAGIC_GGSNLLAMA_FILE_MAGIC_GGSQLLAMA_SESSION_MAGICLLAMA_SESSION_VERSIONLLAMA_STATE_SEQ_MAGICLLAMA_STATE_SEQ_VERSIONr^r3llama_vocab_p_ctypesr4llama_model_p_ctypesr5llama_context_p_ctypesr6llama_memory_t_ctypesr7llama_kv_cache_p_ctypesrrrtrHrs llama_token_pruLLAMA_VOCAB_TYPE_NONELLAMA_VOCAB_TYPE_SPMLLAMA_VOCAB_TYPE_BPELLAMA_VOCAB_TYPE_WPMLLAMA_VOCAB_TYPE_UGMLLAMA_VOCAB_TYPE_RWKVLLAMA_VOCAB_TYPE_PLAMO2LLAMA_VOCAB_PRE_TYPE_DEFAULTLLAMA_VOCAB_PRE_TYPE_LLAMA3!LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM#LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODERLLAMA_VOCAB_PRE_TYPE_FALCONLLAMA_VOCAB_PRE_TYPE_MPTLLAMA_VOCAB_PRE_TYPE_STARCODERLLAMA_VOCAB_PRE_TYPE_GPT2LLAMA_VOCAB_PRE_TYPE_REFACTLLAMA_VOCAB_PRE_TYPE_COMMAND_RLLAMA_VOCAB_PRE_TYPE_STABLELM2LLAMA_VOCAB_PRE_TYPE_QWEN2LLAMA_VOCAB_PRE_TYPE_OLMOLLAMA_VOCAB_PRE_TYPE_DBRXLLAMA_VOCAB_PRE_TYPE_SMAUGLLAMA_VOCAB_PRE_TYPE_POROLLAMA_VOCAB_PRE_TYPE_CHATGLM3LLAMA_VOCAB_PRE_TYPE_CHATGLM4LLAMA_VOCAB_PRE_TYPE_VIKINGLLAMA_VOCAB_PRE_TYPE_JAISLLAMA_VOCAB_PRE_TYPE_TEKKENLLAMA_VOCAB_PRE_TYPE_SMOLLMLLAMA_VOCAB_PRE_TYPE_CODESHELLLLAMA_VOCAB_PRE_TYPE_BLOOM!LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISHLLAMA_VOCAB_PRE_TYPE_EXAONELLAMA_VOCAB_PRE_TYPE_CHAMELEONLLAMA_VOCAB_PRE_TYPE_MINERVA"LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLMLLAMA_VOCAB_PRE_TYPE_GPT4OLLAMA_VOCAB_PRE_TYPE_SUPERBPELLAMA_VOCAB_PRE_TYPE_TRILLIONLLAMA_VOCAB_PRE_TYPE_BAILINGMOELLAMA_VOCAB_PRE_TYPE_LLAMA4LLAMA_VOCAB_PRE_TYPE_PIXTRALLLAMA_VOCAB_PRE_TYPE_SEED_CODERLLAMA_ROPE_TYPE_NONELLAMA_ROPE_TYPE_NORMLLAMA_ROPE_TYPE_NEOXGGML_ROPE_TYPE_NEOXLLAMA_ROPE_TYPE_MROPEGGML_ROPE_TYPE_MROPELLAMA_ROPE_TYPE_VISIONGGML_ROPE_TYPE_VISIONLLAMA_TOKEN_TYPE_UNDEFINEDLLAMA_TOKEN_TYPE_NORMALLLAMA_TOKEN_TYPE_UNKNOWNLLAMA_TOKEN_TYPE_CONTROLLLAMA_TOKEN_TYPE_USER_DEFINEDLLAMA_TOKEN_TYPE_UNUSEDLLAMA_TOKEN_TYPE_BYTELLAMA_TOKEN_ATTR_UNDEFINEDLLAMA_TOKEN_ATTR_UNKNOWNLLAMA_TOKEN_ATTR_UNUSEDLLAMA_TOKEN_ATTR_NORMALLLAMA_TOKEN_ATTR_CONTROLLLAMA_TOKEN_ATTR_USER_DEFINEDLLAMA_TOKEN_ATTR_BYTELLAMA_TOKEN_ATTR_NORMALIZEDLLAMA_TOKEN_ATTR_LSTRIPLLAMA_TOKEN_ATTR_RSTRIPLLAMA_TOKEN_ATTR_SINGLE_WORDLLAMA_FTYPE_ALL_F32LLAMA_FTYPE_MOSTLY_F16LLAMA_FTYPE_MOSTLY_Q4_0LLAMA_FTYPE_MOSTLY_Q4_1LLAMA_FTYPE_MOSTLY_Q8_0LLAMA_FTYPE_MOSTLY_Q5_0LLAMA_FTYPE_MOSTLY_Q5_1LLAMA_FTYPE_MOSTLY_Q2_KLLAMA_FTYPE_MOSTLY_Q3_K_SLLAMA_FTYPE_MOSTLY_Q3_K_MLLAMA_FTYPE_MOSTLY_Q3_K_LLLAMA_FTYPE_MOSTLY_Q4_K_SLLAMA_FTYPE_MOSTLY_Q4_K_MLLAMA_FTYPE_MOSTLY_Q5_K_SLLAMA_FTYPE_MOSTLY_Q5_K_MLLAMA_FTYPE_MOSTLY_Q6_KLLAMA_FTYPE_MOSTLY_IQ2_XXSLLAMA_FTYPE_MOSTLY_IQ2_XSLLAMA_FTYPE_MOSTLY_Q2_K_SLLAMA_FTYPE_MOSTLY_IQ3_XSLLAMA_FTYPE_MOSTLY_IQ3_XXSLLAMA_FTYPE_MOSTLY_IQ1_SLLAMA_FTYPE_MOSTLY_IQ4_NLLLAMA_FTYPE_MOSTLY_IQ3_SLLAMA_FTYPE_MOSTLY_IQ3_MLLAMA_FTYPE_MOSTLY_IQ2_SLLAMA_FTYPE_MOSTLY_IQ2_MLLAMA_FTYPE_MOSTLY_IQ4_XSLLAMA_FTYPE_MOSTLY_IQ1_MLLAMA_FTYPE_MOSTLY_BF16LLAMA_FTYPE_MOSTLY_TQ1_0LLAMA_FTYPE_MOSTLY_TQ2_0LLAMA_FTYPE_MOSTLY_MXFP4_MOELLAMA_FTYPE_GUESSED#LLAMA_ROPE_SCALING_TYPE_UNSPECIFIEDLLAMA_ROPE_SCALING_TYPE_NONELLAMA_ROPE_SCALING_TYPE_LINEARLLAMA_ROPE_SCALING_TYPE_YARN LLAMA_ROPE_SCALING_TYPE_LONGROPE!LLAMA_ROPE_SCALING_TYPE_MAX_VALUELLAMA_POOLING_TYPE_UNSPECIFIEDLLAMA_POOLING_TYPE_NONELLAMA_POOLING_TYPE_MEANLLAMA_POOLING_TYPE_CLSLLAMA_POOLING_TYPE_LASTLLAMA_POOLING_TYPE_RANK LLAMA_ATTENTION_TYPE_UNSPECIFIEDLLAMA_ATTENTION_TYPE_CAUSALLLAMA_ATTENTION_TYPE_NON_CAUSALLLAMA_SPLIT_MODE_NONELLAMA_SPLIT_MODE_LAYERLLAMA_SPLIT_MODE_ROW StructurerGrcr\llama_token_data_array_prVrrhLLAMA_KV_OVERRIDE_TYPE_INTLLAMA_KV_OVERRIDE_TYPE_FLOATLLAMA_KV_OVERRIDE_TYPE_BOOLLLAMA_KV_OVERRIDE_TYPE_STRrxrrrrrllama_log_callbackrrllama_logit_bias_prrrEllama_adapter_lora_p_ctypesrrrrrGGML_NUMA_STRATEGY_DISABLEDGGML_NUMA_STRATEGY_DISTRIBUTEGGML_NUMA_STRATEGY_ISOLATEGGML_NUMA_STRATEGY_NUMACTLGGML_NUMA_STRATEGY_MIRRORGGML_NUMA_STRATEGY_COUNTrrrrrrrrrrrrerr r rrrrrrrrrrr r"r$r&r(r+r-r/r1r3r5r7r9r;r=r?rArCrFrJrLrQrSrVrXc_uint64r[r]rarcrergrirkrmrrrurxr|r~rrrrrrrrrrrrrrrrrrrrrrrrrc_uint8rrrrrrrrrrrrrrrrrrrrrrrrrrr r rrrrrrrrrr!r#r%r'r)r,r/r1r3r5r7r9r;r=r?rArCrErGrIrKrMrOrQrSrUrWrYr[r]r_rarcrergrirkrrrxr}rrrrrNllama_sampler_p_ctypesllama_sampler_i_namellama_sampler_i_acceptllama_sampler_i_applyllama_sampler_i_resetllama_sampler_i_clonellama_sampler_i_freerWrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rrrrr r"r$r&r2r(r-r3r7rMrYrZr!s;"  jjnn%9:QdQlW\\"''//"''//(*CD E Mryr~r~@SsT >:64T:L       $*#3#3 MM6??FMM6??$ &&v}}fooF#%!'**, ###,.- - +S1)3/-s3 // NN nn {+ ~~ @-'#1FV !$%!&'#!"!"!# " "   !#$&! !#!%'" " ""$ !"$-..*/00,133. ! !  ! &$  %^!')# !" #$ $D!"$$& "#v''(^^$45V--.">>*@A!** MM6>>6??>!&""!T  FLL  .f.. .j2))2Lc6++cX%%&,,J.0&"2"20nv''"^^$45 !1!1 &))$nnV__=      "    (    )        !     \\N   (  __() 1    __() 1  "" ^^FOO$foo7IJ  !$.@  6??+          /0  "6  "/0  "6       NN    $b&//:; /V__EF &FMM:; 'V]];< -r6==AB %r6==9: !7 8&//JK #9":FOOLM !$:#;V__MN "%;$RST #&<%=?TUV %(>'?NO     (+?*@BVWX (+?*@&,,OP *-A,BFNNST %(<'=v~~NO &)=(>OP %(<'=v~~NO (+?*@&..QR $';&)?PQ   NN   &     ),@+A6>>RS#   NN   2    '   NN   2    6??FOO< NN   1*    #&:%;V__MN,/CV__.UW]WfWfgh'*>)?QR*-A,BFMMRS*-A,BFMMRS%(<'=v~~+.B-CV]]ST+.B-CV]]ST23  OO <  $6??+  %*#   !   8&..I NN  #7@E  89 NN  #7     $v~~&  NN   ,         6FMM*       MM   $      4      ( )       $3\BD       $      !   .       $         *!6 Ey !9!6 Ey !9),A+BFMMRS56!7 8&..23T    MM   $      ,      ( )       "5|Dd"       $      !   0       $         ("8,!G ":"8,!G ":'*@)A4HI*-C,DfmmTU'*@)A4HI'*@)A6??ST'*@)A6??STv~~&  OO    $  &     v~~& OO  :V^^FNN;V__M OO   $ &    V^^FNN;< OO  :  v'  MM  )2  ;   $v'  MM  )2  ;       MM   %/        MM   %/    \* OO   v~~&   OO   $ &     (v~~&   OO   $ &    "  OO    %  /   $v'  OO      )  2  ;     2   $ &   .H+ (  $ *    $#k]D9:"8+!FWX,"8+!FWX     ) 0  "%;$EF/0&..2P  "V^^, NN6>>"  7    34fnnV^^6TV^^, NN6>>"  7   \* NN6>>"  ":   1;? !8 2K@&.. !8 1;? !8/=v}}3[A6== !8 "%9$:KHI "%9$:KHI "%9$:KHI "%9$:KHI !$8#9;GH "%9$:KHI #&:%;[IJ  MM     MM     MM                            ;' OO  !8   ;' NN  !8    ;' LL  !8   ;' MM  !8    ;' MM  !8                             MM    MM                          4   NN   ' %  +  , .  P  NN   " C %  %  (   L{#   NN   $'   +  / 0  X)*   NN   )        6"v' NN   $ .!//f&&F$$#M2O 6''9OP))$0F T(( ":((/EF(()?AWX''.DE !" %& #$ #$ #$ !"  ^^O$&=>  *1H   OO    [)    56  "E                  34    V^^,  9   LL     V^^,  9  ,b2HIJ *V__,=?UVW-r3IJK+fnn-=?UVW  ^^V__%     ^^V__%      ^^V__%    *V^^,<>TUV ! ^^V^^V^^4  &+   ^^V^^V__fooF  "%-0  $ ^^   ! ^^V__fnnfnnfnnU  "'.38;  $ __fnnfnn5   %   6??FOO<  ',