220 lines
6.8 KiB
Groff
220 lines
6.8 KiB
Groff
.TH SAFETENSORS_TO_NVFP4_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
|
|
.SH NAME
|
|
safetensors_to_nvfp4_gguf \- quantize a safetensors file to an NVFP4 GGUF weight container
|
|
.SH SYNOPSIS
|
|
.B safetensors_to_nvfp4_gguf
|
|
[\fB\-o\fR \fIOUTPUT\fR]
|
|
[\fB\-\-all\-weights\fR]
|
|
[\fB\-\-no\-quant\fR]
|
|
[\fB\-\-outtype\fR \fITYPE\fR]
|
|
[\fB\-\-llama\-cpp\fR \fIPATH\fR]
|
|
[\fB\-\-convert\-to\-quant\fR \fIPATH\fR]
|
|
[\fB\-\-overwrite\fR|\fB\-\-force\fR]
|
|
[\fB\-\-mmproj\fR \fIMMPROJ\fR]
|
|
[\fB\-\-context\-length\fR \fILEN\fR]
|
|
[\fB\-\-checkpoint\-dir\fR \fIDIR\fR]
|
|
[\fB\-\-fresh\fR]
|
|
[\fB\-\-keep\-checkpoints\fR]
|
|
[\fB\-\-no\-temp\-file\fR]
|
|
[\fB\-\-verbose\fR]
|
|
[\fB\-\-self\-test\fR]
|
|
[\fB\-\-device\fR \fIDEVICE\fR]
|
|
[\fB\-\-simple\fR]
|
|
[\fB\-\-optimizer\fR \fIOPT\fR]
|
|
[\fB\-\-num\-iter\fR \fIN\fR]
|
|
[\fB\-\-lr\fR \fILR\fR]
|
|
[\fB\-\-lr\-schedule\fR \fISCHED\fR]
|
|
[\fB\-\-scale\-optimization\fR \fIMODE\fR]
|
|
[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR]
|
|
[\fB\-\-full\-matrix\fR]
|
|
[\fB\-\-top\-p\fR \fIP\fR]
|
|
[\fB\-\-min\-k\fR \fIK\fR]
|
|
[\fB\-\-max\-k\fR \fIK\fR]
|
|
[\fB\-\-early\-stop\-loss\fR \fILOSS\fR]
|
|
[\fB\-\-early\-stop\-lr\fR \fILR\fR]
|
|
[\fB\-\-early\-stop\-stall\fR \fISTALL\fR]
|
|
[\fB\-\-target\-loss\fR \fITARGET\fR]
|
|
[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR]
|
|
[\fB\-\-lr\-min\fR \fIMIN\fR]
|
|
[\fB\-\-lr\-patience\fR \fIN\fR]
|
|
[\fB\-\-use\-speed\fR]
|
|
[\fB\-\-calib\-samples\fR \fISAMPLES\fR]
|
|
[\fB\-\-seed\fR \fISEED\fR]
|
|
[\fB\-\-no\-bias\-correction\fR]
|
|
[\fB\-\-quantize\-output\fR]
|
|
[\fB\-\-exclude\fR \fIREGEX\fR]
|
|
[\fB\-h\fR|\fB\-\-help\fR]
|
|
\fIsafetensors\fR
|
|
.SH DESCRIPTION
|
|
.B safetensors_to_nvfp4_gguf
|
|
quantizes the tensor weights stored in a single \fI.safetensors\fR file directly into
|
|
an NVFP4 GGUF weight container, without requiring a Hugging Face model directory or
|
|
tokenizer configuration.
|
|
.PP
|
|
This tool is particularly useful for standalone diffusion models (e.g. FLUX, SDXL),
|
|
isolated LoRA weights, or intermediate checkpoints. All original tensor names are
|
|
preserved.
|
|
.PP
|
|
By default, layers on \fBconvert_to_quant\fR's standard exclusion list (normalization
|
|
layers, biases, embeddings, and output heads) remain unquantized in their original precision.
|
|
Passing \fB\-\-all\-weights\fR overrides this heuristic and quantizes every eligible 2D
|
|
matrix whose trailing dimension is a multiple of 64.
|
|
.PP
|
|
The tool also supports repacking ComfyUI-style NVFP4 \fI.safetensors\fR files (which contain
|
|
\fIweight_scale\fR and \fIweight_scale_2\fR) into GGUF format without re-optimization using
|
|
\fB\-\-no\-quant \-\-outtype nvfp4\fR.
|
|
.SH POSITIONAL ARGUMENTS
|
|
.TP
|
|
.I safetensors
|
|
Path to the input \fI.safetensors\fR file.
|
|
.SH OPTIONS
|
|
.SS Output & Packaging
|
|
.TP
|
|
.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT
|
|
Output GGUF file path (default: \fI<stem>-NVFP4.gguf\fR, or \fI<stem>.gguf\fR with \fB\-\-no\-quant\fR and \fBauto\fR).
|
|
.TP
|
|
.BR \-\-overwrite ", " \-\-force
|
|
Replace an existing GGUF or auxiliary output file without prompting.
|
|
.TP
|
|
.B \-\-all\-weights
|
|
Quantize every 2D weight matrix eligible for 4-bit block packing, ignoring convert_to_quant's
|
|
default exclusion list.
|
|
.TP
|
|
.B \-\-no\-quant
|
|
Skip learned rounding. Writes a float GGUF or repacks ComfyUI NVFP4 weights.
|
|
.TP
|
|
.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4}
|
|
Data type when learned rounding is skipped: \fBnvfp4\fR repacks ComfyUI NVFP4 weights; \fBauto\fR preserves original dtypes (default: \fBauto\fR).
|
|
.TP
|
|
.BI \-\-mmproj " " MMPROJ
|
|
Path to a vision projector GGUF to copy next to output. May be specified multiple times.
|
|
.TP
|
|
.BI \-\-context\-length " " LEN
|
|
Maximum context length to store in the GGUF metadata.
|
|
.SS Quantization & Optimization
|
|
.TP
|
|
.B \-\-simple
|
|
Round each value directly onto the E2M1 grid without learned rounding.
|
|
.TP
|
|
.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original}
|
|
Learned-rounding optimizer (default: \fBauto\fR).
|
|
.TP
|
|
.BI \-\-num\-iter " " N
|
|
Optimization iterations per matrix (default: 4000).
|
|
.TP
|
|
.BI \-\-lr " " LR
|
|
Initial learning rate (default: 1.0).
|
|
.TP
|
|
.BI \-\-lr\-schedule " " {plateau,exponential,adaptive}
|
|
Learning rate decay schedule (default: \fBplateau\fR).
|
|
.TP
|
|
.BI \-\-lr\-min " " MIN
|
|
Lowest learning rate floor (default: 1e-8).
|
|
.TP
|
|
.BI \-\-lr\-patience " " N
|
|
Patience steps before decaying the learning rate (default: 5).
|
|
.TP
|
|
.B \-\-use\-speed
|
|
Enable Prodigy optimizer's \fIuse_speed\fR mode.
|
|
.TP
|
|
.BI \-\-scale\-optimization " " {fixed,iterative,joint}
|
|
Block scale tuning method (default: \fBjoint\fR).
|
|
.TP
|
|
.BI \-\-scale\-refinement\-rounds " " N
|
|
Scale refresh iterations for iterative mode (default: 2).
|
|
.TP
|
|
.B \-\-full\-matrix
|
|
Use full SVD instead of low-rank SVD.
|
|
.TP
|
|
.BI \-\-top\-p " " P ", " \-\-min\-k " " K ", " \-\-max\-k " " K
|
|
Singular value decomposition rank selection bounds.
|
|
.TP
|
|
.BI \-\-target\-loss " " TARGET
|
|
Target reconstruction loss threshold (default: 1e-5; 0 disables).
|
|
.TP
|
|
.BI \-\-early\-stop\-stall " " STALL
|
|
Early stop stall steps without reaching a new best loss (default: 2000).
|
|
.TP
|
|
.BI \-\-early\-stop\-floor\-stall " " FLOOR
|
|
Stall steps at minimum learning rate before stopping (default: 64; 0 disables).
|
|
.TP
|
|
.BI \-\-early\-stop\-lr " " LR
|
|
Learning rate threshold for early stop check.
|
|
.TP
|
|
.BI \-\-calib\-samples " " SAMPLES
|
|
Random Gaussian calibration inputs for sibling bias correction (default: 3072).
|
|
.TP
|
|
.BI \-\-seed " " SEED
|
|
Random seed (default: 42).
|
|
.TP
|
|
.B \-\-no\-bias\-correction
|
|
Disable sibling bias correction.
|
|
.TP
|
|
.B \-\-quantize\-output
|
|
Also quantize \fIoutput.weight\fR.
|
|
.TP
|
|
.BI \-\-exclude " " REGEX
|
|
Regex matching tensor names to leave unquantized.
|
|
.SS System & Execution
|
|
.TP
|
|
.BI \-\-device " " DEVICE
|
|
Device for quantization computations: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR).
|
|
.TP
|
|
.BI \-\-llama\-cpp " " PATH
|
|
Path to llama.cpp repository checkout (default: \fI~/llama.cpp\fR).
|
|
.TP
|
|
.BI \-\-convert\-to\-quant " " PATH
|
|
Path to convert_to_quant checkout.
|
|
.TP
|
|
.BI \-\-checkpoint\-dir " " DIR
|
|
Directory for per-tensor resume checkpoints.
|
|
.TP
|
|
.B \-\-fresh
|
|
Delete checkpoint directory before starting.
|
|
.TP
|
|
.B \-\-keep\-checkpoints
|
|
Keep per-tensor checkpoints after completion.
|
|
.TP
|
|
.B \-\-no\-temp\-file
|
|
Keep finished GGUF tensors in memory rather than a temporary file.
|
|
.TP
|
|
.B \-\-verbose
|
|
Enable verbose logging.
|
|
.TP
|
|
.B \-\-self\-test
|
|
Verify NVFP4 packing against llama.cpp dequantizer and exit.
|
|
.SH EXAMPLES
|
|
.TP
|
|
.B Quantize a standalone diffusion model:
|
|
.RS
|
|
.nf
|
|
safetensors_to_nvfp4_gguf -o flux1-dev-nvfp4.gguf flux1-dev.safetensors
|
|
.fi
|
|
.RE
|
|
.TP
|
|
.B Repack ComfyUI NVFP4 safetensors to GGUF:
|
|
.RS
|
|
.nf
|
|
safetensors_to_nvfp4_gguf --no-quant --outtype nvfp4 -o repacked.gguf comfy-nvfp4.safetensors
|
|
.fi
|
|
.RE
|
|
.TP
|
|
.B Quantize every eligible weight matrix with simple grid rounding:
|
|
.RS
|
|
.nf
|
|
safetensors_to_nvfp4_gguf --simple --all-weights -o fast-nvfp4.gguf model.safetensors
|
|
.fi
|
|
.RE
|
|
.SH EXIT STATUS
|
|
.TP
|
|
.B 0
|
|
Success.
|
|
.TP
|
|
.B 1
|
|
An error occurred during conversion or packaging.
|
|
.SH SEE ALSO
|
|
.BR hf_to_nvfp4_safetensors (1),
|
|
.BR hf_to_nvfp4_gguf (1),
|
|
.BR gguf_to_nvfp4_gguf (1)
|
|
|