236 lines
8.2 KiB
Groff
236 lines
8.2 KiB
Groff
.TH HF_TO_NVFP4_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
|
|
.SH NAME
|
|
hf_to_nvfp4_gguf \- quantize Hugging Face models to NVFP4 GGUF for llama.cpp and Ollama
|
|
.SH SYNOPSIS
|
|
.B hf_to_nvfp4_gguf
|
|
[\fB\-o\fR \fIOUTPUT\fR]
|
|
[\fB\-\-modelfile\-only\fR]
|
|
[\fB\-\-overwrite\fR]
|
|
[\fB\-\-mmproj\fR \fIMMPROJ\fR]
|
|
[\fB\-\-context\-length\fR \fILEN\fR]
|
|
[\fB\-\-no\-quant\fR]
|
|
[\fB\-\-outtype\fR \fITYPE\fR]
|
|
[\fB\-\-llama\-cpp\fR \fIPATH\fR]
|
|
[\fB\-\-convert\-to\-quant\fR \fIPATH\fR]
|
|
[\fB\-\-model\-name\fR \fINAME\fR]
|
|
[\fB\-\-device\fR \fIDEVICE\fR]
|
|
[\fB\-\-simple\fR]
|
|
[\fB\-\-optimizer\fR \fIOPT\fR]
|
|
[\fB\-\-num\-iter\fR \fIN\fR]
|
|
[\fB\-\-lr\fR \fILR\fR]
|
|
[\fB\-\-lr\-patience\fR \fIN\fR]
|
|
[\fB\-\-lr\-min\fR \fIMIN\fR]
|
|
[\fB\-\-lr\-schedule\fR \fISCHED\fR]
|
|
[\fB\-\-scale\-optimization\fR \fIMODE\fR]
|
|
[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR]
|
|
[\fB\-\-full\-matrix\fR]
|
|
[\fB\-\-top\-p\fR \fIP\fR]
|
|
[\fB\-\-min\-k\fR \fIK\fR]
|
|
[\fB\-\-max\-k\fR \fIK\fR]
|
|
[\fB\-\-early\-stop\-loss\fR \fILOSS\fR]
|
|
[\fB\-\-early\-stop\-stall\fR \fISTALL\fR]
|
|
[\fB\-\-target\-loss\fR \fITARGET\fR]
|
|
[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR]
|
|
[\fB\-\-calib\-samples\fR \fISAMPLES\fR]
|
|
[\fB\-\-seed\fR \fISEED\fR]
|
|
[\fB\-\-no\-bias\-correction\fR]
|
|
[\fB\-\-quantize\-output\fR]
|
|
[\fB\-\-exclude\fR \fIREGEX\fR]
|
|
[\fB\-\-checkpoint\-dir\fR \fIDIR\fR]
|
|
[\fB\-\-fresh\fR]
|
|
[\fB\-\-keep\-checkpoints\fR]
|
|
[\fB\-\-eager\fR]
|
|
[\fB\-\-no\-temp\-file\fR]
|
|
[\fB\-\-verbose\fR]
|
|
[\fB\-\-self\-test\fR]
|
|
[\fB\-h\fR|\fB\-\-help\fR]
|
|
\fImodel\fR
|
|
.SH DESCRIPTION
|
|
.B hf_to_nvfp4_gguf
|
|
quantizes a Hugging Face model directory (\fIconfig.json\fR + safetensors) to
|
|
NVIDIA FP4 (NVFP4) format packaged into a GGUF file suitable for \fBllama.cpp\fR
|
|
and \fBOllama\fR.
|
|
.PP
|
|
The model is loaded through \fBllama.cpp\fR's architecture conversion logic so that
|
|
all vocabulary, tensor naming conventions, tokenization metadata, and architectural
|
|
parameters match native GGUF standards. Weight matrices are quantized using learned
|
|
rounding (via \fBconvert_to_quant\fR) or simple nearest-grid projection onto the E2M1
|
|
floating point grid with two-level scaling (per-block E4M3/E8M0 scales and per-tensor scales).
|
|
.PP
|
|
Along with the output GGUF, an Ollama \fBModelfile\fR is generated automatically,
|
|
preserving the model's chat template, stop sequences, parameter defaults, and vision projectors.
|
|
.SH POSITIONAL ARGUMENTS
|
|
.TP
|
|
.I model
|
|
Path to a Hugging Face model directory containing \fIconfig.json\fR and one or more \fI.safetensors\fR files.
|
|
.SH OPTIONS
|
|
.SS Output & File Control
|
|
.TP
|
|
.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT
|
|
Output destination directory (default: \fI./<model-name>-NVFP4\fR, or \fI./<model-name>\fR with
|
|
\fB\-\-no\-quant\fR and \fBauto\fR). If the path ends in \fI.gguf\fR, it specifies the destination
|
|
GGUF file itself.
|
|
.TP
|
|
.B \-\-overwrite
|
|
Overwrite an existing GGUF, Modelfile, or copied projector without prompting.
|
|
.TP
|
|
.B \-\-modelfile\-only
|
|
Generate only the Ollama \fBModelfile\fR and exit without quantizing or writing GGUF tensors.
|
|
.TP
|
|
.BI \-\-mmproj " " MMPROJ
|
|
Path to a vision projector GGUF. The file is copied into the output directory and added as an
|
|
additional \fBFROM\fR layer in the Modelfile. May be specified multiple times for multimodal models.
|
|
.TP
|
|
.BI \-\-context\-length " " LEN
|
|
Override the maximum context length recorded in the GGUF metadata. Replaces checkpoint value
|
|
or defines one if missing.
|
|
.TP
|
|
.BI \-\-model\-name " " NAME
|
|
Override the model name recorded in the GGUF metadata.
|
|
.SS Quantization Mode
|
|
.TP
|
|
.B \-\-no\-quant
|
|
Skip learned rounding. Writes a float GGUF, or repacks an already NVFP4 checkpoint.
|
|
.TP
|
|
.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4}
|
|
Data type for non-quantized weights when learned rounding is skipped. \fBnvfp4\fR repacks a
|
|
checkpoint that already has packed 4-bit weights. \fBauto\fR preserves the source float types (default: \fBauto\fR).
|
|
.TP
|
|
.B \-\-simple
|
|
Round weights directly onto the E2M1 grid without iterative learned rounding (fast, non-iterative).
|
|
.TP
|
|
.B \-\-quantize\-output
|
|
Also quantize the final output projection (\fIlm_head\fR) matrix. By default, output layers are
|
|
kept in BF16/float for generation quality.
|
|
.TP
|
|
.BI \-\-exclude " " REGEX
|
|
Regular expression matching GGUF tensor names that should remain unquantized in BF16.
|
|
.SS Optimization & Learned Rounding
|
|
.TP
|
|
.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original}
|
|
Learned-rounding optimization algorithm. \fBauto\fR selects Prodigy if installed, otherwise AdamW (default: \fBauto\fR).
|
|
.TP
|
|
.BI \-\-num\-iter " " N
|
|
Learned-rounding optimization iterations per matrix (default: 4000).
|
|
.TP
|
|
.BI \-\-lr " " LR
|
|
Initial learning rate (default: 1.0).
|
|
.TP
|
|
.BI \-\-lr\-patience " " N
|
|
Number of steps without improvement before decaying the learning rate (default: 5).
|
|
.TP
|
|
.BI \-\-lr\-min " " MIN
|
|
Minimum learning rate floor. The flat-loss termination check begins once this floor is reached (default: 1e-8).
|
|
.TP
|
|
.BI \-\-lr\-schedule " " {plateau,exponential,adaptive}
|
|
Learning rate decay schedule (default: \fBplateau\fR).
|
|
.TP
|
|
.BI \-\-scale\-optimization " " {fixed,iterative,joint}
|
|
Block scale tuning strategy. \fBfixed\fR keeps initial scales; \fBiterative\fR refreshes scales periodically
|
|
using the original optimizer; \fBjoint\fR trains block scales concurrently with the codes (default: \fBjoint\fR).
|
|
.TP
|
|
.BI \-\-scale\-refinement\-rounds " " N
|
|
Number of scale refreshes when \fB\-\-scale\-optimization\fR is set to \fBiterative\fR (default: 2).
|
|
.TP
|
|
.B \-\-full\-matrix
|
|
Use full singular value decomposition (SVD) rather than a low-rank approximation for the loss objective.
|
|
.TP
|
|
.BI \-\-top\-p " " P
|
|
Cumulative energy cutoff for low-rank SVD (default: 0.2).
|
|
.TP
|
|
.BI \-\-min\-k " " K ", " \-\-max\-k " " K
|
|
Minimum and maximum rank limits for low-rank SVD (default: 128 and 1280).
|
|
.TP
|
|
.BI \-\-target\-loss " " TARGET
|
|
Stop optimization when best reconstruction loss reaches this threshold (default: 1e-5; 0 disables).
|
|
.TP
|
|
.BI \-\-early\-stop\-stall " " STALL
|
|
Stop optimization after this many steps without reaching a new best loss (default: 2000).
|
|
.TP
|
|
.BI \-\-early\-stop\-floor\-stall " " FLOOR
|
|
Stop optimization after this many steps without improvement once the learning rate has reached \fB\-\-lr\-min\fR (default: 64; 0 disables).
|
|
.TP
|
|
.BI \-\-calib\-samples " " SAMPLES
|
|
Number of random Gaussian calibration inputs used to calculate sibling bias correction (default: 3072).
|
|
.TP
|
|
.BI \-\-seed " " SEED
|
|
Random seed for calibration vectors (default: 42).
|
|
.TP
|
|
.B \-\-no\-bias\-correction
|
|
Disable sibling bias correction on downstream biases.
|
|
.SS System & Execution
|
|
.TP
|
|
.BI \-\-device " " DEVICE
|
|
Device to perform optimization on: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR).
|
|
.TP
|
|
.BI \-\-llama\-cpp " " PATH
|
|
Path to local \fBllama.cpp\fR repository checkout (default: \fI~/llama.cpp\fR).
|
|
.TP
|
|
.BI \-\-convert\-to\-quant " " PATH
|
|
Path to local \fBconvert_to_quant\fR repository checkout, if not installed as a Python package.
|
|
.TP
|
|
.BI \-\-checkpoint\-dir " " DIR
|
|
Directory used to store and resume per-tensor quantization checkpoints.
|
|
.TP
|
|
.B \-\-fresh
|
|
Remove existing checkpoint directory before starting.
|
|
.TP
|
|
.B \-\-keep\-checkpoints
|
|
Preserve per-tensor checkpoints after successful conversion.
|
|
.TP
|
|
.B \-\-eager
|
|
Load all model tensors into memory up front rather than streaming (requires more system RAM).
|
|
.TP
|
|
.B \-\-no\-temp\-file
|
|
Keep finished GGUF tensors in memory rather than buffering through a temporary scratch file.
|
|
.TP
|
|
.B \-\-verbose
|
|
Enable detailed verbose logging.
|
|
.TP
|
|
.B \-\-self\-test
|
|
Run internal verification of NVFP4 packing against llama.cpp's reference dequantizer and exit immediately.
|
|
.SH EXAMPLES
|
|
.TP
|
|
.B Full learned-rounding conversion to GGUF:
|
|
.RS
|
|
.nf
|
|
hf_to_nvfp4_gguf -o ./llama3-8b-NVFP4 ./meta-llama-3-8b
|
|
.fi
|
|
.RE
|
|
.TP
|
|
.B Fast quantization with simple grid rounding:
|
|
.RS
|
|
.nf
|
|
hf_to_nvfp4_gguf --simple -o ./qwen-nvfp4 ./Qwen2.5-7B-Instruct
|
|
.fi
|
|
.RE
|
|
.TP
|
|
.B Multi-modal model with vision projector:
|
|
.RS
|
|
.nf
|
|
hf_to_nvfp4_gguf --mmproj mmproj-model-f16.gguf -o ./vision-nvfp4 ./Qwen2-VL-7B
|
|
.fi
|
|
.RE
|
|
.TP
|
|
.B Specify CUDA GPU and custom llama.cpp directory:
|
|
.RS
|
|
.nf
|
|
hf_to_nvfp4_gguf --llama-cpp /opt/llama.cpp --device cuda:0 -o ./output ./model
|
|
.fi
|
|
.RE
|
|
.SH EXIT STATUS
|
|
.TP
|
|
.B 0
|
|
Success.
|
|
.TP
|
|
.B 1
|
|
An error occurred during conversion or validation.
|
|
.SH SEE ALSO
|
|
.BR gguf_to_nvfp4_gguf (1),
|
|
.BR safetensors_to_nvfp4_gguf (1),
|
|
.BR hf_to_nvfp4_safetensors (1),
|
|
.BR hf_info (1),
|
|
.BR gguf_info (1),
|
|
.BR ollama_nvfp4 (1)
|
|
|