From 0ef586d1312430721b53d98264718811b78f29cf Mon Sep 17 00:00:00 2001 From: zefie Date: Wed, 4 Nov 2026 16:09:09 -0500 Subject: [PATCH] add scripts/manpages for llama.cpp converters --- docker/convert_hf_to_gguf | 2 + docker/convert_hf_to_gguf_update | 2 + docker/convert_llama_ggml_to_gguf | 2 + docker/convert_lora_to_gguf | 2 + man/man1/convert_hf_to_gguf.1 | 266 +++++++++++++++++++++++ man/man1/convert_hf_to_gguf.py.1 | 1 + man/man1/convert_hf_to_gguf_update.1 | 106 +++++++++ man/man1/convert_hf_to_gguf_update.py.1 | 1 + man/man1/convert_llama_ggml_to_gguf.1 | 130 +++++++++++ man/man1/convert_llama_ggml_to_gguf.py.1 | 1 + man/man1/convert_lora_to_gguf.1 | 153 +++++++++++++ man/man1/convert_lora_to_gguf.py.1 | 1 + 12 files changed, 667 insertions(+) create mode 100755 docker/convert_hf_to_gguf create mode 100755 docker/convert_hf_to_gguf_update create mode 100755 docker/convert_llama_ggml_to_gguf create mode 100755 docker/convert_lora_to_gguf create mode 100644 man/man1/convert_hf_to_gguf.1 create mode 120000 man/man1/convert_hf_to_gguf.py.1 create mode 100644 man/man1/convert_hf_to_gguf_update.1 create mode 120000 man/man1/convert_hf_to_gguf_update.py.1 create mode 100644 man/man1/convert_llama_ggml_to_gguf.1 create mode 120000 man/man1/convert_llama_ggml_to_gguf.py.1 create mode 100644 man/man1/convert_lora_to_gguf.1 create mode 120000 man/man1/convert_lora_to_gguf.py.1 diff --git a/docker/convert_hf_to_gguf b/docker/convert_hf_to_gguf new file mode 100755 index 0000000..7c4075a --- /dev/null +++ b/docker/convert_hf_to_gguf @@ -0,0 +1,2 @@ +#!/bin/bash +python /opt/nvfp4/llama.cpp/convert_hf_to_gguf.py "${@}" diff --git a/docker/convert_hf_to_gguf_update b/docker/convert_hf_to_gguf_update new file mode 100755 index 0000000..08feb6d --- /dev/null +++ b/docker/convert_hf_to_gguf_update @@ -0,0 +1,2 @@ +#!/bin/bash +python /opt/nvfp4/llama.cpp/convert_hf_to_gguf_update.py "${@}" diff --git a/docker/convert_llama_ggml_to_gguf b/docker/convert_llama_ggml_to_gguf new file mode 100755 index 0000000..3f03c5c --- /dev/null +++ b/docker/convert_llama_ggml_to_gguf @@ -0,0 +1,2 @@ +#!/bin/bash +python /opt/nvfp4/llama.cpp/convert_llama_ggml_to_gguf.py "${@}" diff --git a/docker/convert_lora_to_gguf b/docker/convert_lora_to_gguf new file mode 100755 index 0000000..039bf62 --- /dev/null +++ b/docker/convert_lora_to_gguf @@ -0,0 +1,2 @@ +#!/bin/bash +python /opt/nvfp4/llama.cpp/convert_lora_to_gguf.py "${@}" diff --git a/man/man1/convert_hf_to_gguf.1 b/man/man1/convert_hf_to_gguf.1 new file mode 100644 index 0000000..3f647b0 --- /dev/null +++ b/man/man1/convert_hf_to_gguf.1 @@ -0,0 +1,266 @@ +.TH CONVERT_HF_TO_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +convert_hf_to_gguf \- convert Hugging Face transformer models to GGUF format +.SH SYNOPSIS +.B convert_hf_to_gguf +[\fB\-\-vocab\-only\fR] +[\fB\-\-outfile\fR \fIOUTFILE\fR] +[\fB\-\-outtype\fR {\fBf32\fR|\fBf16\fR|\fBbf16\fR|\fBq8_0\fR|\fBtq1_0\fR|\fBtq2_0\fR|\fBauto\fR}] +[\fB\-\-bigendian\fR] +[\fB\-\-use\-temp\-file\fR] +[\fB\-\-no\-lazy\fR] +[\fB\-\-model\-name\fR \fINAME\fR] +[\fB\-\-split\-max\-tensors\fR \fIN\fR] +[\fB\-\-split\-max\-size\fR \fISIZE\fR] +[\fB\-\-dry\-run\fR] +[\fB\-\-no\-tensor\-first\-split\fR] +[\fB\-\-metadata\fR \fIPATH\fR] +[\fB\-\-remote\fR] +[\fB\-\-mmproj\fR] +[\fB\-\-mtp\fR|\fB\-\-no\-nextn\fR|\fB\-\-dspark\fR] +[\fB\-\-mistral\-format\fR] +[\fB\-\-disable\-mistral\-community\-chat\-template\fR] +[\fB\-\-sentence\-transformers\-dense\-modules\fR] +[\fB\-\-fuse\-gate\-up\-exps\fR] +[\fB\-\-fp8\-as\-q8\fR] +[\fB\-\-fuse\-qkv\fR] +[\fB\-\-target\-model\-dir\fR \fIDIR\fR] +[\fB\-\-verbose\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fImodel\fR +.PP +.B convert_hf_to_gguf +\fB\-\-print\-supported\-models\fR +.SH DESCRIPTION +.B convert_hf_to_gguf +converts Hugging Face format model checkpoints (safetensors, PyTorch bin, config, +and tokenizer files) into standard GGUF files compatible with \fBllama.cpp\fR and +downstream tools such as Ollama. +.PP +The script parses architecture hyperparameters, maps model-specific tensor names +to canonical GGML tensor names, handles tokenizer extraction and conversion (SPM, +BPE, WPM, and UGM), encodes chat templates, and quantizes or casts weights into +the requested precision format. +.PP +It also supports advanced operations including sharded split outputs, vision +projector (mmproj) extraction, speculative decoding draft head extraction (MTP and +DSpark), remote streaming from Hugging Face Hub without full checkpoint downloads, +and Mixture-of-Experts (MoE) tensor fusion. +.SH POSITIONAL ARGUMENTS +.TP +.I model +Path to a local directory containing model configuration and tensor files, or +a Hugging Face repository ID (e.g. \fIHuggingFaceTB/SmolLM2-1.7B-Instruct\fR) when +used in combination with \fB\-\-remote\fR. Optional only when +\fB\-\-print\-supported\-models\fR is specified. +.SH OPTIONS +.SS Output & Quantization +.TP +.BI \-\-outfile " " OUTFILE +Destination path for the converted GGUF file. Defaults to an automatically generated +path based on the model directory name and output type. If the pattern \fI{ftype}\fR +is present in the filename, it is dynamically replaced with the chosen output type string. +.TP +.BI \-\-outtype " " TYPE +Output tensor precision and format. Choices are: +.RS +.TP +.B auto +(Default) Preserves natively supported FP8 weights when present, and otherwise +selects the highest-fidelity 16-bit float representation (BF16 or F16) matching the +source model. +.TP +.B f32 +Convert all weight tensors to full 32-bit floating point. +.TP +.B f16 +Convert floating-point tensors to 16-bit IEEE half precision. +.TP +.B bf16 +Convert floating-point tensors to Brain Floating Point 16-bit. +.TP +.B q8_0 +Quantize non-1D weight tensors to 8-bit integer quantization (Q8_0). +.TP +.BR tq1_0 ", " tq2_0 +Convert weights to ternary quantized format. +.RE +.TP +.B \-\-fp8\-as\-q8 +Store tensors dequantized from FP8 input weights as \fBQ8_0\fR instead of +dequantizing all the way to BF16 or F16, saving memory and disk space. +.SS Model & Vocabulary Selection +.TP +.B \-\-vocab\-only +Extract and serialize only the model vocabulary, tokenizer merges, and metadata, +omitting all neural network weight tensors. +.TP +.BI \-\-model\-name " " NAME +Explicitly specify or override the model name stored in the GGUF general metadata. +.TP +.BI \-\-metadata " " PATH +Path to a JSON file containing authorship and licensing metadata overrides to inject +into the GGUF header. +.TP +.B \-\-print\-supported\-models +Print all registered Hugging Face model architectures supported by the conversion +framework and exit immediately. +.SS Remote & Vision Modes +.TP +.B \-\-remote +(Experimental) Stream safetensors index and tensor byte ranges directly from Hugging +Face Hub over HTTP using partial range requests without downloading entire multi-gigabyte +checkpoints to local disk. Only configuration and tokenizer files are downloaded. The +\fImodel\fR argument must be a valid Hugging Face repository ID. For private or gated +repositories, the \fBHF_TOKEN\fR environment variable must be set. +.TP +.B \-\-mmproj +Extract and export only the multimodal projector (\fImmproj\fR) weights for vision-language +models (e.g. LLaVA, Pixtral). The output filename is automatically prefixed with +\fImmproj-\fR. +.SS Speculative Decoding Draft Heads +.TP +.B \-\-mtp +Export only the multi-token prediction (MTP) head as an independent GGUF model, +suitable for use as a speculative decoding draft model. The output filename is +prefixed with \fImtp-\fR. Mutually exclusive with \fB\-\-no\-nextn\fR and \fB\-\-dspark\fR. +.TP +.BR \-\-no\-nextn ", " \-\-no\-mtp +Exclude NextN speculative draft tensors from the converted GGUF model. Typically paired +with a secondary run using \fB\-\-mtp\fR or \fB\-\-dspark\fR to distribute target and +draft heads in separate files. +.TP +.B \-\-dspark +Export only DeepSeek-V4 or Mistral Large 4 DSpark speculative draft tensors as a +separate GGUF file. Mutually exclusive with \fB\-\-mtp\fR and \fB\-\-no\-nextn\fR. +.TP +.BI \-\-target\-model\-dir " " DIR +Path to the base target model directory. Required when converting standalone draft +models (such as EAGLE3 or DFlash) that require tokenizer, hidden dimension, and layer +count metadata from the primary model to construct a complete GGUF container. +.SS Architecture & Tensor Adjustments +.TP +.B \-\-mistral\-format +Indicate that the source model directory follows native Mistral format rather than +standard Hugging Face Transformers conventions. Requires the \fBmistral\-common\fR +Python package. +.TP +.B \-\-disable\-mistral\-community\-chat\-template +Disable the fallback Hugging Face community chat template for Mistral models and use +the official \fBmistral\-common\fR tokenizer and template engine. +.TP +.B \-\-sentence\-transformers\-dense\-modules +Include dense projection modules from sentence-transformers embeddings models +(e.g. \fIgoogle/embeddinggemma-300m\fR). By default, these modules are omitted. +.TP +.B \-\-fuse\-gate\-up\-exps +Fuse separate MoE expert tensors \fIgate_exps\fR and \fIup_exps\fR into a unified +\fIgate_up_exps\fR tensor for improved memory access and inference performance. +.TP +.B \-\-fuse\-qkv +Fuse separate query, key, and value projection weight matrices into a single contiguous +QKV tensor where supported by the architecture. +.SS Sharding & Splitting +.TP +.BI \-\-split\-max\-tensors " " N +Maximum number of tensors per output GGUF shard. Defaults to 0 (splitting disabled). +.TP +.BI \-\-split\-max\-size " " SIZE +Maximum byte size per output GGUF shard. Can be specified with unit suffixes \fBK\fR, +\fBM\fR, or \fBG\fR (e.g. \fB4G\fR or \fB2000M\fR). Defaults to 0 (no size-based splitting). +.TP +.B \-\-no\-tensor\-first\-split +Do not store any model weight tensors in the first shard (shard 00001). Shard 1 will +contain only metadata, key-values, and vocabulary. +.TP +.B \-\-dry\-run +Simulate the conversion and splitting plan, logging projected shard sizes and tensor +distributions without writing any output files to disk. +.SS System & Execution +.TP +.B \-\-use\-temp\-file +Write intermediate tensors to temporary scratch files on disk rather than holding them +in RAM. Recommended on systems with constrained system memory to prevent out-of-memory +process termination. Cannot be used together with sharding. +.TP +.B \-\-no\-lazy +Disable lazy tensor evaluation, forcing all tensors to be computed and staged in RAM +prior to writing. Useful if lazy evaluation encounters issues. +.TP +.B \-\-bigendian +Target big-endian architectures when serializing tensor headers and raw data. +.TP +.B \-\-verbose +Enable verbose debug logging output. +.TP +.BR \-h ", " \-\-help +Display command-line syntax and option descriptions, then exit. +.SH ENVIRONMENT +.TP +.B HF_TOKEN +Hugging Face user authentication token. Required when accessing gated or private +models in conjunction with \fB\-\-remote\fR. +.TP +.B NO_LOCAL_GGUF +When set in the environment, prevents the script from automatically prepending the +local repository's \fIgguf\-py\fR subdirectory to Python's module search path. +.SH EXAMPLES +.TP +.B Convert a local Hugging Face model to default 16-bit GGUF: +.RS +.nf +convert_hf_to_gguf ./Meta-Llama-3-8B-Instruct +.fi +.RE +.TP +.B Convert directly to 8-bit integer quantization with explicit output name: +.RS +.nf +convert_hf_to_gguf --outtype q8_0 --outfile ./llama3-8b-q8.gguf ./Meta-Llama-3-8B +.fi +.RE +.TP +.B Convert a remote Hugging Face repository without downloading weights to disk: +.RS +.nf +export HF_TOKEN="hf_..." +convert_hf_to_gguf --remote HuggingFaceTB/SmolLM2-1.7B-Instruct +.fi +.RE +.TP +.B Split a large 70B model into 4-gigabyte GGUF shards: +.RS +.nf +convert_hf_to_gguf --split-max-size 4G --outtype f16 ./Meta-Llama-3-70B +.fi +.RE +.TP +.B Extract only the multimodal vision projector for a vision-language model: +.RS +.nf +convert_hf_to_gguf --mmproj ./Qwen2-VL-7B-Instruct +.fi +.RE +.TP +.B Extract only the vocabulary and tokenizer metadata: +.RS +.nf +convert_hf_to_gguf --vocab-only --outfile vocab.gguf ./Mistral-7B-v0.3 +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Successful completion. +.TP +.B 1 +Conversion failed due to invalid arguments, missing dependencies, unsupported architecture, +or I/O error. +.SH SEE ALSO +.BR convert_hf_to_gguf_update (1), +.BR convert_lora_to_gguf (1), +.BR convert_llama_ggml_to_gguf (1), +.BR hf_to_nvfp4_gguf (1), +.BR gguf_to_nvfp4_gguf (1), +.BR gguf_info (1) + diff --git a/man/man1/convert_hf_to_gguf.py.1 b/man/man1/convert_hf_to_gguf.py.1 new file mode 120000 index 0000000..0eba12e --- /dev/null +++ b/man/man1/convert_hf_to_gguf.py.1 @@ -0,0 +1 @@ +convert_hf_to_gguf.1 \ No newline at end of file diff --git a/man/man1/convert_hf_to_gguf_update.1 b/man/man1/convert_hf_to_gguf_update.1 new file mode 100644 index 0000000..7bfbad8 --- /dev/null +++ b/man/man1/convert_hf_to_gguf_update.1 @@ -0,0 +1,106 @@ +.TH CONVERT_HF_TO_GGUF_UPDATE 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +convert_hf_to_gguf_update \- update pre-tokenizer hashes and vocab tests for convert_hf_to_gguf +.SH SYNOPSIS +.B convert_hf_to_gguf_update +[\fB\-\-full\fR] +[\fB\-\-check\-missing\fR] +[\fB\-h\fR|\fB\-\-help\fR] +[\fIhf_token\fR] +.SH DESCRIPTION +.B convert_hf_to_gguf_update +is a developer and contributor utility for \fBllama.cpp\fR that analyzes Hugging Face +model tokenizers and updates the pre-tokenizer recognition logic in +\fIconversion/base.py\fR (specifically the \fBget_vocab_base_pre()\fR function). +.PP +Proper pre-tokenization is critical for inference correctness in \fBllama.cpp\fR. Different +model families preprocess whitespace, digits, punctuation, and Unicode characters +differently before applying byte-pair or subword merges. +.B convert_hf_to_gguf_update +tokenizes a standardized, multilingual benchmark string (\fBCHK_TXT\fR) across a curated +registry of models, computes SHA-256 hashes of the resulting token sequences, and maps +those hashes to recognized pre-tokenizer types: +.IP \(bu 2 +\fBSPM\fR: SentencePiece +.IP \(bu 2 +\fBBPE\fR: Byte-Pair Encoding (e.g. GPT-2, LLaMA 3, Qwen2, DeepSeek, Falcon) +.IP \(bu 2 +\fBWPM\fR: WordPiece (e.g. BERT, BGE) +.IP \(bu 2 +\fBUGM\fR: Unigram (e.g. T5) +.PP +In addition to updating source definitions, the script generates tokenizer test vectors +(\fI.inp\fR input text and \fI.out\fR expected token ID files) under \fImodels/\fR for validation +against \fBllama.cpp\fR's native test harness (\fItest-tokenizer-0\fR). +.SH POSITIONAL ARGUMENTS +.TP +.I hf_token +Optional Hugging Face user access token. If omitted, the script attempts to read the +cached token from \fI~/.cache/huggingface/token\fR. +.SH OPTIONS +.TP +.B \-\-full +Download the complete list of registered models in the script's registry. +Ensure that the provided Hugging Face token has access permissions to all gated models +(e.g. LLaMA, Gemma). +.TP +.B \-\-check\-missing +Only check for models in the registry that currently lack pre-tokenizer hash entries in +\fIconversion/base.py\fR without redownloading the full dataset. Ignored if \fB\-\-full\fR +is also passed. +.TP +.BR \-h ", " \-\-help +Display command-line help summary and exit. +.SH FILES +.TP +.I conversion/base.py +The target source file updated with new tokenizer hash mappings in +\fBget_vocab_base_pre()\fR. +.TP +.I ~/.cache/huggingface/token +Default fallback path used to authenticate against Hugging Face Hub. +.TP +.I models/tokenizers// +Local cache directory where downloaded tokenizer files are stored. +.TP +.I models/ggml-vocab-.gguf.inp +Input test strings written for llama.cpp tokenization testing. +.TP +.I models/ggml-vocab-.gguf.out +Reference token IDs produced by Hugging Face \fBAutoTokenizer\fR for regression tests. +.SH EXAMPLES +.TP +.B Check for newly added models missing pre-tokenizer signatures: +.RS +.nf +convert_hf_to_gguf_update --check-missing +.fi +.RE +.TP +.B Run full update with explicit Hugging Face access token: +.RS +.nf +convert_hf_to_gguf_update --full hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +.fi +.RE +.TP +.B Generate vocabulary testing files for a newly registered tokenizer: +.RS +.nf +python3 convert_hf_to_gguf.py models/tokenizers/llama-bpe/ \\ + --outfile models/ggml-vocab-llama-bpe.gguf --vocab-only +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +An error occurred (e.g. network failure, invalid token, or missing target files). +.SH SEE ALSO +.BR convert_hf_to_gguf (1), +.BR convert_lora_to_gguf (1), +.BR convert_llama_ggml_to_gguf (1), +.BR gguf_info (1) + diff --git a/man/man1/convert_hf_to_gguf_update.py.1 b/man/man1/convert_hf_to_gguf_update.py.1 new file mode 120000 index 0000000..75d59e8 --- /dev/null +++ b/man/man1/convert_hf_to_gguf_update.py.1 @@ -0,0 +1 @@ +convert_hf_to_gguf_update.1 \ No newline at end of file diff --git a/man/man1/convert_llama_ggml_to_gguf.1 b/man/man1/convert_llama_ggml_to_gguf.1 new file mode 100644 index 0000000..4dd2de1 --- /dev/null +++ b/man/man1/convert_llama_ggml_to_gguf.1 @@ -0,0 +1,130 @@ +.TH CONVERT_LLAMA_GGML_TO_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +convert_llama_ggml_to_gguf \- convert legacy GGMLv1/v2/v3 model files to GGUF format +.SH SYNOPSIS +.B convert_llama_ggml_to_gguf +\fB\-i\fR \fIINPUT\fR +\fB\-o\fR \fIOUTPUT\fR +[\fB\-\-name\fR \fINAME\fR] +[\fB\-\-desc\fR \fIDESC\fR] +[\fB\-\-gqa\fR \fIN\fR] +[\fB\-\-eps\fR \fIEPS\fR] +[\fB\-c\fR|\fB\-\-context\-length\fR \fILEN\fR] +[\fB\-m\fR|\fB\-\-model\-metadata\-dir\fR \fIDIR\fR] +[\fB\-\-vocab\-dir\fR \fIDIR\fR] +[\fB\-\-vocabtype\fR \fITYPE\fR] +[\fB\-\-verbose\fR] +[\fB\-h\fR|\fB\-\-help\fR] +.SH DESCRIPTION +.B convert_llama_ggml_to_gguf +converts legacy binary models in GGML formats (\fBGGML\fR, \fBGGMF\fR, and \fBGGJT\fR / +\fBGGMLv3\fR) into the modern \fBGGUF\fR format used by current versions of +\fBllama.cpp\fR. +.PP +The tool memory-maps the input file, parses model hyperparameters (embedding dimensions, +layer counts, head counts, rotary frequencies, and quantization type), reverses +legacy tensor dimension transpositions where needed, and writes standard GGUF metadata +and tensor tables. +.PP +.B Note: +This conversion script is provided as a best-effort migration tool for archiving or +recovering older checkpoints. Whenever feasible, it is strongly recommended to convert +directly from original Hugging Face source weights using \fBconvert_hf_to_gguf\fR(1). +.SH REQUIRED ARGUMENTS +.TP +.BI \-i " " INPUT ", " \-\-input " " INPUT +Path to the input legacy GGML binary file (e.g. \fIggml-model-q4_0.bin\fR). +.TP +.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT +Destination path for the converted GGUF model file (e.g. \fImodel-q4_0.gguf\fR). +.SH OPTIONS +.TP +.BI \-\-name " " NAME +Set the model name metadata attribute in the GGUF header. +.TP +.BI \-\-desc " " DESC +Set the model description text metadata in the GGUF header. +.TP +.BI \-\-gqa " " N +Grouped-query attention (GQA) factor. Defaults to \fB1\fR (standard multi-head attention). +For LLaMA 2 70B models, this must be set to \fB8\fR unless overridden via +\fB\-\-model\-metadata\-dir\fR. +.TP +.BI \-\-eps " " EPS +Root Mean Square (RMS) normalization epsilon value. Defaults to \fB5.0e-06\fR. +Typically \fB1e-6\fR for LLaMA 1 and OpenLLaMA, and \fB1e-5\fR for LLaMA 2. +.TP +.BI \-c " " LEN ", " \-\-context\-length " " LEN +Maximum context length in tokens. Defaults to \fB2048\fR (typical for LLaMA 1). For +LLaMA 2 models, set to \fB4096\fR. +.TP +.BI \-m " " DIR ", " \-\-model\-metadata\-dir " " DIR +Path to a directory containing original Hugging Face (\fIconfig.json\fR, \fItokenizer.model\fR) +or Meta PyTorch (\fIparams.json\fR) files. When specified, exact vocabulary items, +token scores, special tokens, GQA ratios, context limits, and RMS epsilon values are +loaded directly from the original configuration, overriding command-line heuristic flags. +.TP +.BI \-\-vocab\-dir " " DIR +Directory containing \fItokenizer.model\fR when stored separately from the model metadata +directory. Only meaningful when \fB\-\-model\-metadata\-dir\fR is also specified. +.TP +.BI \-\-vocabtype " " TYPE +Vocabulary loader types to attempt (comma-separated list, default: \fBspm,hfft\fR). Only +meaningful in conjunction with \fB\-\-model\-metadata\-dir\fR. +.TP +.B \-\-verbose +Increase logging verbosity, printing debug information about tensors and hyperparameter +offsets. +.TP +.BR \-h ", " \-\-help +Display command usage summary and exit. +.SH FORMAT COMPATIBILITY +.B convert_llama_ggml_to_gguf +recognizes the following historical format magic identifiers: +.TP +.B GGML (magic 'lmgg') +Version 1 format. Does not contain vocabulary scores; token scores default to 0.0 unless +\fB\-\-model\-metadata\-dir\fR is provided. +.TP +.B GGMF (magic 'fmgg') +Version 2 format. Adds vocabulary scores to token strings. +.TP +.B GGJT / GGMLv3 (magic 'tjgg') +Version 3 format with 32-byte tensor data alignment. Supports all GGML quantization +types up to Q6_K. +.SH EXAMPLES +.TP +.B Basic conversion of a legacy LLaMA 1 Q4_0 GGML file: +.RS +.nf +convert_llama_ggml_to_gguf -i llama-7b-q4_0.bin -o llama-7b-q4_0.gguf +.fi +.RE +.TP +.B Convert a LLaMA 2 70B GGML file with appropriate GQA and context flags: +.RS +.nf +convert_llama_ggml_to_gguf -i llama-2-70b-q4_0.bin -o llama-2-70b-q4_0.gguf \\ + --gqa 8 --eps 1e-5 -c 4096 +.fi +.RE +.TP +.B High-accuracy conversion using original Hugging Face metadata directory: +.RS +.nf +convert_llama_ggml_to_gguf -i model-q4_0.bin -o model-q4_0.gguf \\ + -m ./original-hf-model-dir/ +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +An error occurred (e.g. corrupt header, unsupported dimension, missing metadata). +.SH SEE ALSO +.BR convert_hf_to_gguf (1), +.BR convert_lora_to_gguf (1), +.BR gguf_info (1) + diff --git a/man/man1/convert_llama_ggml_to_gguf.py.1 b/man/man1/convert_llama_ggml_to_gguf.py.1 new file mode 120000 index 0000000..7f69baa --- /dev/null +++ b/man/man1/convert_llama_ggml_to_gguf.py.1 @@ -0,0 +1 @@ +convert_llama_ggml_to_gguf.1 \ No newline at end of file diff --git a/man/man1/convert_lora_to_gguf.1 b/man/man1/convert_lora_to_gguf.1 new file mode 100644 index 0000000..5616853 --- /dev/null +++ b/man/man1/convert_lora_to_gguf.1 @@ -0,0 +1,153 @@ +.TH CONVERT_LORA_TO_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +convert_lora_to_gguf \- convert Hugging Face PEFT LoRA adapters to GGUF format +.SH SYNOPSIS +.B convert_lora_to_gguf +[\fB\-\-outfile\fR \fIOUTFILE\fR] +[\fB\-\-outtype\fR {\fBf32\fR|\fBf16\fR|\fBbf16\fR|\fBq8_0\fR|\fBauto\fR}] +[\fB\-\-base\fR \fIBASE\fR] +[\fB\-\-base\-model\-id\fR \fIID\fR] +[\fB\-\-trust\-remote\-code\fR] +[\fB\-\-bigendian\fR] +[\fB\-\-no\-lazy\fR] +[\fB\-\-dry\-run\fR] +[\fB\-\-verbose\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fIlora_path\fR +.SH DESCRIPTION +.B convert_lora_to_gguf +converts fine-tuned Hugging Face PEFT LoRA (Low-Rank Adaptation) adapter checkpoints +into GGUF adapter files compatible with \fBllama.cpp\fR runtime adapter injection +(\fB\-\-lora\fR) and offline merging (\fBllama\-export\-lora\fR). +.PP +LoRA checkpoints store low-rank weight matrices \fIA\fR and \fIB\fR rather than full model +weights. To convert these tensors correctly into GGUF format with proper tensor names, +dimensions, and architecture attributes, +.B convert_lora_to_gguf +requires access to the configuration of the base model on which the adapter was trained. +Base model metadata is automatically retrieved from Hugging Face Hub using the +\fIbase_model_name_or_path\fR key inside \fIadapter_config.json\fR, or supplied locally via +\fB\-\-base\fR. +.PP +The tool also features full support for: +.IP \(bu 2 +\fBMergekit LoRAs\fR: Adapters created with \fImergekit\-extract\-lora\fR, including adapted +token embedding weights and normalization layers. +.IP \(bu 2 +\fBaLoRA (Activation-aware LoRA)\fR: Automatic tokenization of \fIinvocation_string\fR into +GGUF \fIadapter.alora_invocation_tokens\fR metadata. +.IP \(bu 2 +Both SafeTensors (\fIadapter_model.safetensors\fR) and legacy PyTorch +(\fIadapter_model.bin\fR) checkpoint formats. +.SH POSITIONAL ARGUMENTS +.TP +.I lora_path +Directory containing the Hugging Face PEFT LoRA adapter. Must contain +\fIadapter_config.json\fR along with \fIadapter_model.safetensors\fR or +\fIadapter_model.bin\fR. +.SH OPTIONS +.TP +.BI \-\-outfile " " OUTFILE +Destination path for the output GGUF file. Defaults to an automatically named file +inside the adapter directory. If the substring \fI{ftype}\fR is present in the path, it is +dynamically replaced with the chosen output format name. +.TP +.BI \-\-outtype " " TYPE +Output tensor precision format. Choices are: +.RS +.TP +.B f32 +(Default) 32-bit single-precision floating point. Recommended for preserving adapter +fidelity. +.TP +.B f16 +16-bit half-precision IEEE float. +.TP +.B bf16 +16-bit Brain Floating Point. +.TP +.B q8_0 +Quantize adapter weights to 8-bit integer format. +.TP +.B auto +Select the highest-fidelity 16-bit floating point format based on the first loaded +tensor. +.RE +.TP +.BI \-\-base " " BASE +Path to a local directory containing configuration files (\fIconfig.json\fR, +\fItokenizer.json\fR) of the base model. Note that only configuration and tokenizer +files are required; full model weights are not loaded. +.TP +.BI \-\-base\-model\-id " " ID +Hugging Face model repository ID of the base model (e.g. +\fImeta-llama/Llama-3.2-1B-Instruct\fR). If provided, overrides \fB\-\-base\fR and +fetches configuration directly from Hugging Face Hub. +.TP +.B \-\-trust\-remote\-code +Permit execution of custom modeling and tokenizer code hosted in remote Hugging Face +repositories when downloading base model configuration. +.TP +.B \-\-bigendian +Target big-endian architecture when serializing tensor headers and raw data. +.TP +.B \-\-no\-lazy +Disable lazy tensor loading, computing and staging all modified tensors in memory +before writing. +.TP +.B \-\-dry\-run +Simulate the conversion process and log all planned tensor operations and metadata +without writing any output files to disk. +.TP +.B \-\-verbose +Enable verbose debug logging output. +.TP +.BR \-h ", " \-\-help +Display command usage summary and option descriptions, then exit. +.SH EXAMPLES +.TP +.B Convert a local LoRA adapter with automatic base model download: +.RS +.nf +convert_lora_to_gguf ./my-llama3-lora/ +.fi +.RE +.TP +.B Convert a LoRA adapter using a local base model directory: +.RS +.nf +convert_lora_to_gguf --base ./Meta-Llama-3-8B/ ./my-llama3-lora/ +.fi +.RE +.TP +.B Convert with explicit output path in 16-bit float: +.RS +.nf +convert_lora_to_gguf --outtype f16 --outfile ./llama3-adapter-f16.gguf \\ + ./my-llama3-lora/ +.fi +.RE +.TP +.B Use the converted adapter in llama.cpp: +.RS +.nf +llama-cli -m llama-3-8b-Q4_K_M.gguf --lora llama3-adapter-f16.gguf \\ + -p "Hello, world!" +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +Conversion failed (e.g. missing adapter files, base model configuration mismatch, or +unsupported architecture). +.SH SEE ALSO +.BR convert_hf_to_gguf (1), +.BR convert_hf_to_gguf_update (1), +.BR convert_llama_ggml_to_gguf (1), +.BR gguf_to_nvfp4_gguf (1), +.BR hf_to_nvfp4_gguf (1), +.BR gguf_info (1) + diff --git a/man/man1/convert_lora_to_gguf.py.1 b/man/man1/convert_lora_to_gguf.py.1 new file mode 120000 index 0000000..b8ef687 --- /dev/null +++ b/man/man1/convert_lora_to_gguf.py.1 @@ -0,0 +1 @@ +convert_lora_to_gguf.1 \ No newline at end of file