manfiles and new scripts
This commit is contained in:
zefie committed
1 parent
83566ee414
commit
b6037b3ea5
20 files changed
+1564
-10
No files matched your search
+39
-9
@@ -1,11 +1,37 @@
|
||||
FROM runpod/pytorch:1.5.0-cu1320-torch2130-ubuntu2404
|
||||
WORKDIR /root
|
||||
FROM nvidia/cuda:13.4.3-cudnn-devel-ubuntu26.04
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN rm -f /etc/dpkg/dpkg.cfg.d/excludes && \
|
||||
rm -f /etc/apt/apt.conf.d/docker-clean && \
|
||||
rm -f /etc/apt/apt.conf.d/01-nodoc && \
|
||||
rm -f /usr/bin/man && \
|
||||
dpkg-divert --quiet --remove /usr/bin/man
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt update && apt-get dist-upgrade --autoremove -y && apt-get install -y \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
python3-venv \
|
||||
build-essential \
|
||||
man \
|
||||
man-db \
|
||||
wget \
|
||||
bzip2 \
|
||||
curl \
|
||||
git \
|
||||
cmake
|
||||
|
||||
RUN python3 -m venv /root/venv
|
||||
ENV PATH="/root/venv/bin:${PATH}"
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install torch torchvision torchaudio --upgrade --index-url \
|
||||
https://download.pytorch.org/whl/cu134/
|
||||
|
||||
WORKDIR /root
|
||||
RUN mkdir llama.cpp
|
||||
COPY --exclude=build/ llama.cpp ./llama.cpp/
|
||||
WORKDIR /root/llama.cpp
|
||||
ENV LD_LIBRARY_PATH=/usr/local/cuda-13.2/compat:${LD_LIBRARY_PATH}
|
||||
ENV LIBRARY_PATH=/usr/local/cuda-13.2/compat:${LIBRARY_PATH}
|
||||
ENV LD_LIBRARY_PATH=/usr/local/cuda-13.4/compat:${LD_LIBRARY_PATH}
|
||||
ENV LIBRARY_PATH=/usr/local/cuda-13.4/compat:${LIBRARY_PATH}
|
||||
|
||||
RUN cmake -B build -S . \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
@@ -18,18 +44,22 @@ RUN cmake -B build -S . \
|
||||
&& rm -rf build
|
||||
|
||||
WORKDIR /root
|
||||
RUN git clone https://github.com/silveroxides/convert_to_quant && cd convert_to_quant && pip install -r requirements.txt && pip install .
|
||||
RUN git clone https://github.com/silveroxides/convert_to_quant && cd convert_to_quant && pip3 install -r requirements.txt && pip3 install .
|
||||
|
||||
RUN mkdir nvfp4_gguf_quant
|
||||
WORKDIR /root/nvfp4_gguf_quant
|
||||
COPY --exclude=__pycache__ nvfp4gguf ./nvfp4gguf
|
||||
COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py requirements.txt safetensors_to_nvfp4_gguf.py .
|
||||
COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py requirements.txt safetensors_to_nvfp4_gguf.py hf_info.py gguf_info.py .
|
||||
|
||||
RUN pip install -r requirements.txt
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install -r requirements.txt
|
||||
COPY --chmod=0644 docker/README.txt /workspace/README.txt
|
||||
COPY --chmod=0755 docker/hf_to_nvfp4_gguf docker/hf_to_nvfp4_safetensors docker/gguf_to_nvfp4_gguf docker/safetensors_to_nvfp4_gguf /usr/local/bin/
|
||||
COPY --chmod=0755 docker/hf_info docker/gguf_info docker/hf_to_nvfp4_gguf docker/hf_to_nvfp4_safetensors docker/gguf_to_nvfp4_gguf docker/safetensors_to_nvfp4_gguf /usr/local/bin/
|
||||
|
||||
RUN apt remove -y python3-cryptography && pip install heretic-llm
|
||||
RUN mkdir -p /usr/local/share/man/man1
|
||||
COPY --chmod=0644 man/man1/* /usr/local/share/man/man1/
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm
|
||||
RUN rm -f /opt/nvidia/entrypoint.d/*banner*.sh /opt/nvidia/entrypoint.d/*license*.sh 2>/dev/null
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/bin/bash
|
||||
python /root/nvfp4_gguf_quant/gguf_info.py "${@}"
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/bin/bash
|
||||
python /root/nvfp4_gguf_quant/hf_info.py "${@}"
|
||||
@@ -309,7 +309,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser.add_argument("--early-stop-loss", type=float, default=5e-9)
|
||||
parser.add_argument(
|
||||
"--early-stop-stall", type=int, default=DEFAULT_EARLY_STOP_STALL,
|
||||
help="Stop after this many steps without progress. At the learning-rate floor, progress means a 1% loss drop (default: 2000)",
|
||||
help="Stop after this many steps without progress. At the learning-rate floor, progress means a 1%% loss drop (default: 2000)",
|
||||
)
|
||||
parser.add_argument("--calib-samples", type=int, default=3072)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
.TH CTQ 1 "November 2026" "convert_to_quant" "User Commands"
|
||||
.SH NAME
|
||||
ctq \- convert safetensors model weights to scaled FP8, NVFP4, INT8, and MXFP8 formats
|
||||
.SH SYNOPSIS
|
||||
.B ctq
|
||||
\fB\-i\fR \fIINPUT\fR
|
||||
[\fB\-o\fR \fIOUTPUT\fR]
|
||||
[\fB\-\-comfy_quant\fR]
|
||||
[\fB\-\-simple\fR]
|
||||
[\fB\-\-device\fR \fIDEVICE\fR]
|
||||
[\fB\-\-low\-memory\fR]
|
||||
[\fB\-\-nvfp4\fR|\fB\-\-int8\fR|\fB\-\-mxfp8\fR]
|
||||
[\fIoptions...\fR]
|
||||
.SH DESCRIPTION
|
||||
.B ctq
|
||||
(\fBconvert_to_quant\fR) quantizes neural network weights stored in \fI.safetensors\fR
|
||||
format into scaled lower-precision representations, primarily targeting
|
||||
\fBComfyUI\fR and hardware-accelerated inference runtimes.
|
||||
.PP
|
||||
By default, \fBctq\fR performs Scaled FP8 quantization using SVD-guided learned rounding
|
||||
(Prodigy optimizer) and per-tensor scaling. It supports multiple quantization formats:
|
||||
.IP \(bu 2
|
||||
\fBFP8\fR (Float8 E4M3 / E5M2) with tensor-wise, row-wise, or block-wise scaling.
|
||||
.IP \(bu 2
|
||||
\fBNVFP4\fR (NVIDIA 4-bit floating point, E2M1 with dual-level block scaling).
|
||||
.IP \(bu 2
|
||||
\fBINT8\fR block-wise quantization with optional group-wise Hadamard rotation (\fIConvRot\fR).
|
||||
.IP \(bu 2
|
||||
\fBMXFP8\fR (Microscaling FP8) and Hybrid MXFP8 formats.
|
||||
.PP
|
||||
Quantization error can optionally be extracted into a companion Error Correction LoRA
|
||||
adapter (\fB\-\-extract\-lora\fR) to restore high-precision fidelity.
|
||||
.SH OPTIONS
|
||||
.SS Standard Options
|
||||
.TP
|
||||
.BI \-i " " INPUT ", " \-\-input " " INPUT
|
||||
Path to the input \fI.safetensors\fR weight file (required).
|
||||
.TP
|
||||
.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT
|
||||
Path to the output \fI.safetensors\fR destination file. Auto-generated if omitted.
|
||||
.TP
|
||||
.BR \-\-comfy_quant ", " \-\-comfy\-quant
|
||||
Write ComfyUI-compatible quantization tensors and metadata (\fI.comfy_quant\fR).
|
||||
.TP
|
||||
.B \-\-simple
|
||||
Skip SVD learned rounding and apply fast, direct grid quantization.
|
||||
.TP
|
||||
.BR \-fpmm ", " \-\-full_precision_matrix_mult
|
||||
Add \fIfull_precision_matrix_mult=True\fR to \fI.comfy_quant\fR metadata.
|
||||
.TP
|
||||
.B \-\-heur
|
||||
Skip layers with poor quantization characteristics (aspect ratio, size).
|
||||
.TP
|
||||
.BI \-\-device " " DEVICE
|
||||
Execution device (e.g. \fBcuda\fR, \fBcuda:0\fR, \fBcpu\fR). Overrides auto-detection.
|
||||
.TP
|
||||
.BR \-lm ", " \-\-low\-memory
|
||||
Stream tensor loading to minimize RAM consumption (recommended for models exceeding 50% available memory).
|
||||
.TP
|
||||
.BI \-\-verbose " " {DEBUG,VERBOSE,NORMAL,MINIMAL}
|
||||
Logging verbosity level (default: \fBNORMAL\fR).
|
||||
.TP
|
||||
.BI \-\-manual_seed " " SEED
|
||||
Random seed for reproducibility. Set to \-1 for non-deterministic execution.
|
||||
.TP
|
||||
.B \-\-save\-quant\-metadata
|
||||
Write quantization metadata directly into safetensors file header (\fI_quantization_metadata\fR).
|
||||
.SS Quantization Formats & Modes
|
||||
.TP
|
||||
.B \-\-nvfp4
|
||||
Quantize to NVIDIA FP4 (E2M1) with dual-level block scaling (Blackwell architecture).
|
||||
.TP
|
||||
.B \-\-int8
|
||||
Quantize to INT8 block-wise format instead of FP8.
|
||||
.TP
|
||||
.B \-\-convrot
|
||||
Enable group-wise Hadamard rotation (ConvRot) for INT8 row-wise quantization to mitigate activation outliers.
|
||||
.TP
|
||||
.BI \-\-convrot\-group\-size " " SIZE
|
||||
Group size for ConvRot (must be a power of 4, e.g. 64, 256, 1024; default: 256).
|
||||
.TP
|
||||
.B \-\-mxfp8
|
||||
Quantize to MXFP8 (Microscaling FP8) format.
|
||||
.TP
|
||||
.BI \-\-scaling_mode " " {tensor,row,block}
|
||||
FP8 scaling granularity: \fBtensor\fR (single scale), \fBrow\fR (per-row), or \fBblock\fR (2D tile). Default: \fBtensor\fR.
|
||||
.TP
|
||||
.BI \-\-block_size " " SIZE
|
||||
Block size for block-wise scaling (default: 128).
|
||||
.SS Learned Rounding & Optimization
|
||||
.TP
|
||||
.BI \-\-optimizer " " {prodigy,adamw,radam,original}
|
||||
Optimization algorithm for learned rounding (default: \fBprodigy\fR).
|
||||
.TP
|
||||
.BI \-\-num_iter " " N
|
||||
Optimization iterations per tensor (default: 4000).
|
||||
.TP
|
||||
.BI \-\-lr " " LR
|
||||
Initial learning rate for AdamW/RAdam/Original (default: 1.0).
|
||||
.TP
|
||||
.BI \-\-lr_schedule " " {plateau,exponential,adaptive}
|
||||
Learning rate decay schedule (default: \fBplateau\fR).
|
||||
.TP
|
||||
.B \-\-use_speed
|
||||
Enable Prodigy's \fIuse_speed\fR acceleration parameter.
|
||||
.TP
|
||||
.BI \-\-top_p " " P
|
||||
Cumulative energy proportion of principal components for SVD (default: 0.2).
|
||||
.TP
|
||||
.BI \-\-min_k " " K ", " \-\-max_k " " K
|
||||
Minimum and maximum SVD rank bounds (default: 256 and 1280).
|
||||
.TP
|
||||
.B \-\-full_matrix
|
||||
Compute full SVD instead of low-rank SVD.
|
||||
.TP
|
||||
.BI \-\-scale\-optimization " " {fixed,iterative,joint,dualround}
|
||||
Block scale tuning strategy (default: \fBfixed\fR).
|
||||
.SS Error Correction LoRA
|
||||
.TP
|
||||
.B \-\-extract\-lora
|
||||
Extract quantization residual error into a separate LoRA adapter file.
|
||||
.TP
|
||||
.BI \-\-lora\-rank " " RANK
|
||||
Rank for extracted LoRA layers (default: 32).
|
||||
.TP
|
||||
.BI \-\-lora\-target " " REGEX
|
||||
Regex matching layers to target for LoRA extraction.
|
||||
.TP
|
||||
.BI \-\-lora\-output " " PATH
|
||||
Output file path for the extracted LoRA adapter (\fI.safetensors\fR).
|
||||
.SS Model Filters
|
||||
.B ctq
|
||||
includes pre-tuned layer exclusion filters for common architecture families:
|
||||
.TP
|
||||
.BR \-\-gemma4 ", " \-\-qwen_vlm ", " \-\-t5xxl ", " \-\-wan ", " \-\-hunyuan ", " \-\-ltxv2 ", " \-\-qwen ", " \-\-zimage
|
||||
Keep architecture-critical layers (embeddings, layernorms, vision towers, cross-attention) in native precision.
|
||||
.SH EXAMPLES
|
||||
.TP
|
||||
.B Standard FP8 quantization for ComfyUI:
|
||||
.RS
|
||||
.nf
|
||||
ctq -i model.safetensors -o model-fp8.safetensors --comfy_quant
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B NVFP4 quantization using learned rounding:
|
||||
.RS
|
||||
.nf
|
||||
ctq -i model.safetensors -o model-nvfp4.safetensors --nvfp4 --scale-optimization joint
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Fast INT8 conversion with ConvRot outlier mitigation:
|
||||
.RS
|
||||
.nf
|
||||
ctq -i model.safetensors --int8 --convrot --simple -o model-int8.safetensors
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Quantize with Error Correction LoRA extraction:
|
||||
.RS
|
||||
.nf
|
||||
ctq -i model.safetensors --nvfp4 --extract-lora --lora-rank 32 --lora-output model-ec-lora.safetensors
|
||||
.fi
|
||||
.RE
|
||||
.SH SEE ALSO
|
||||
.BR hf_to_nvfp4_safetensors (1),
|
||||
.BR safetensors_to_nvfp4_gguf (1),
|
||||
.BR heretic (1)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
.TH GGUF_INFO 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
|
||||
.SH NAME
|
||||
gguf_info \- print architecture, quantization, and tensor metadata from a GGUF header
|
||||
.SH SYNOPSIS
|
||||
.B gguf_info
|
||||
[\fB\-\-metadata\fR]
|
||||
[\fB\-\-tensors\fR]
|
||||
[\fB\-h\fR|\fB\-\-help\fR]
|
||||
\fIgguf\fR [\fIgguf\fR ...]
|
||||
.SH DESCRIPTION
|
||||
.B gguf_info
|
||||
inspects one or more GGUF files (versions 2 and 3) by reading only their header
|
||||
and tensor directories. The raw weight bytes are never loaded into memory,
|
||||
making the inspection fast even for multi-gigabyte models.
|
||||
.PP
|
||||
By default,
|
||||
.B gguf_info
|
||||
displays a concise summary of the model, including:
|
||||
.IP \(bu 2
|
||||
GGUF version, file size, and total tensor and key-value counts.
|
||||
.IP \(bu 2
|
||||
Model architecture identifier (e.g., \fIllama\fR, \fIqwen2\fR, \fIgemma2\fR).
|
||||
.IP \(bu 2
|
||||
General metadata: model name, author, license, context length, embedding length,
|
||||
attention head counts, layer counts, feed-forward dimension, and vocabulary size.
|
||||
.IP \(bu 2
|
||||
Quantization summary: file type classification (e.g., \fIMOSTLY_NVFP4\fR,
|
||||
\fIMOSTLY_Q4_K_M\fR, \fIMOSTLY_BF16\fR) and an exact count and breakdown of tensor
|
||||
quantization types found across the model.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BR \-h ", " \-\-help
|
||||
Show command-line usage help and exit.
|
||||
.TP
|
||||
.B \-\-metadata
|
||||
Print every key-value metadata pair found in the GGUF header. Long string
|
||||
values, large numeric arrays, and vocabulary/merges arrays are truncated or
|
||||
summarized for readability.
|
||||
.TP
|
||||
.B \-\-tensors
|
||||
Print every tensor present in the file with its full name, GGML quantization
|
||||
type (e.g., \fINVFP4\fR, \fIBF16\fR, \fIF16\fR, \fIQ4_K\fR), shape dimensions,
|
||||
and byte offset within the file.
|
||||
.SH SUPPORTED QUANTIZATION TYPES
|
||||
.B gguf_info
|
||||
recognizes all standard GGML types including:
|
||||
.PP
|
||||
.BR F32 ", " F16 ", " BF16 ", " F64 ", "
|
||||
.BR Q4_0 ", " Q4_1 ", " Q5_0 ", " Q5_1 ", " Q8_0 ", " Q8_1 ", "
|
||||
.BR Q2_K ", " Q3_K ", " Q4_K ", " Q5_K ", " Q6_K ", " Q8_K ", "
|
||||
.BR IQ1_S ", " IQ1_M ", " IQ2_XXS ", " IQ2_XS ", " IQ2_S ", "
|
||||
.BR IQ3_XXS ", " IQ3_S ", " IQ4_XS ", " IQ4_NL ", "
|
||||
.BR TQ1_0 ", " TQ2_0 ", " MXFP4 ", " NVFP4 ", " Q1_0 ", " Q2_0 .
|
||||
.SH EXAMPLES
|
||||
.TP
|
||||
.B Inspect a single GGUF model:
|
||||
.RS
|
||||
.nf
|
||||
gguf_info model-NVFP4.gguf
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Dump all metadata keys and parameters:
|
||||
.RS
|
||||
.nf
|
||||
gguf_info \-\-metadata model-NVFP4.gguf
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B List all tensors and their formats:
|
||||
.RS
|
||||
.nf
|
||||
gguf_info \-\-tensors model-NVFP4.gguf
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Inspect multiple models in batch:
|
||||
.RS
|
||||
.nf
|
||||
gguf_info model-f16.gguf model-NVFP4.gguf
|
||||
.fi
|
||||
.RE
|
||||
.SH EXIT STATUS
|
||||
.TP
|
||||
.B 0
|
||||
Success.
|
||||
.TP
|
||||
.B 1
|
||||
An error occurred (e.g. file not found, invalid magic bytes, or unsupported GGUF version).
|
||||
.SH SEE ALSO
|
||||
.BR hf_info (1),
|
||||
.BR gguf_to_nvfp4_gguf (1),
|
||||
.BR hf_to_nvfp4_gguf (1),
|
||||
.BR ollama_nvfp4 (1)
|
||||
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
gguf_info.1
|
||||
@@ -0,0 +1,216 @@
|
||||
.TH GGUF_TO_NVFP4_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
|
||||
.SH NAME
|
||||
gguf_to_nvfp4_gguf \- quantize or repack an existing GGUF model to NVFP4 GGUF
|
||||
.SH SYNOPSIS
|
||||
.B gguf_to_nvfp4_gguf
|
||||
[\fB\-o\fR \fIOUTPUT\fR]
|
||||
[\fB\-\-modelfile\-only\fR]
|
||||
[\fB\-\-overwrite\fR]
|
||||
[\fB\-\-mmproj\fR \fIMMPROJ\fR]
|
||||
[\fB\-\-context\-length\fR \fILEN\fR]
|
||||
[\fB\-\-no\-quant\fR]
|
||||
[\fB\-\-outtype\fR \fITYPE\fR]
|
||||
[\fB\-\-llama\-cpp\fR \fIPATH\fR]
|
||||
[\fB\-\-convert\-to\-quant\fR \fIPATH\fR]
|
||||
[\fB\-\-model\-name\fR \fINAME\fR]
|
||||
[\fB\-\-device\fR \fIDEVICE\fR]
|
||||
[\fB\-\-simple\fR]
|
||||
[\fB\-\-optimizer\fR \fIOPT\fR]
|
||||
[\fB\-\-num\-iter\fR \fIN\fR]
|
||||
[\fB\-\-lr\fR \fILR\fR]
|
||||
[\fB\-\-lr\-patience\fR \fIN\fR]
|
||||
[\fB\-\-lr\-min\fR \fIMIN\fR]
|
||||
[\fB\-\-lr\-schedule\fR \fISCHED\fR]
|
||||
[\fB\-\-scale\-optimization\fR \fIMODE\fR]
|
||||
[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR]
|
||||
[\fB\-\-full\-matrix\fR]
|
||||
[\fB\-\-top\-p\fR \fIP\fR]
|
||||
[\fB\-\-min\-k\fR \fIK\fR]
|
||||
[\fB\-\-max\-k\fR \fIK\fR]
|
||||
[\fB\-\-early\-stop\-loss\fR \fILOSS\fR]
|
||||
[\fB\-\-early\-stop\-stall\fR \fISTALL\fR]
|
||||
[\fB\-\-target\-loss\fR \fITARGET\fR]
|
||||
[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR]
|
||||
[\fB\-\-calib\-samples\fR \fISAMPLES\fR]
|
||||
[\fB\-\-seed\fR \fISEED\fR]
|
||||
[\fB\-\-no\-bias\-correction\fR]
|
||||
[\fB\-\-quantize\-output\fR]
|
||||
[\fB\-\-exclude\fR \fIREGEX\fR]
|
||||
[\fB\-\-checkpoint\-dir\fR \fIDIR\fR]
|
||||
[\fB\-\-fresh\fR]
|
||||
[\fB\-\-keep\-checkpoints\fR]
|
||||
[\fB\-\-eager\fR]
|
||||
[\fB\-\-no\-temp\-file\fR]
|
||||
[\fB\-\-verbose\fR]
|
||||
[\fB\-\-self\-test\fR]
|
||||
[\fB\-h\fR|\fB\-\-help\fR]
|
||||
\fIgguf\fR
|
||||
.SH DESCRIPTION
|
||||
.B gguf_to_nvfp4_gguf
|
||||
reads an existing GGUF model (such as an F16, BF16, or existing quantized checkpoint)
|
||||
and converts it to NVIDIA FP4 (NVFP4) format, or repacks an existing NVFP4 GGUF file.
|
||||
.PP
|
||||
Vocabulary, tokenizer settings, architecture metadata, and tensor names are copied directly
|
||||
from the source GGUF header. When quantizing unquantized (float) tensors, learned rounding
|
||||
or simple grid rounding is applied using the same optimization flags as \fBhf_to_nvfp4_gguf\fR.
|
||||
.PP
|
||||
An accompanying Ollama \fBModelfile\fR is written alongside the output GGUF, configured with
|
||||
the source model's parameters and templates.
|
||||
.SH POSITIONAL ARGUMENTS
|
||||
.TP
|
||||
.I gguf
|
||||
Input GGUF file path.
|
||||
.SH OPTIONS
|
||||
.SS Output & File Control
|
||||
.TP
|
||||
.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT
|
||||
Output directory (default: \fI./<name>-NVFP4\fR, or \fI./<name>\fR with \fB\-\-no\-quant\fR and \fBauto\fR).
|
||||
If ending in \fI.gguf\fR, specifies the destination file directly.
|
||||
.TP
|
||||
.B \-\-overwrite
|
||||
Overwrite existing output GGUF, Modelfile, or copied projector without prompting.
|
||||
.TP
|
||||
.B \-\-modelfile\-only
|
||||
Write the Ollama Modelfile and skip quantization and GGUF generation.
|
||||
.TP
|
||||
.BI \-\-mmproj " " MMPROJ
|
||||
Path to a vision projector GGUF to bundle into the output directory and add as another \fBFROM\fR layer.
|
||||
May be repeated for multi-modal models.
|
||||
.TP
|
||||
.BI \-\-context\-length " " LEN
|
||||
Override maximum context length stored in the GGUF metadata.
|
||||
.TP
|
||||
.BI \-\-model\-name " " NAME
|
||||
Override the model name stored in the GGUF metadata.
|
||||
.SS Quantization Mode
|
||||
.TP
|
||||
.B \-\-no\-quant
|
||||
Skip learned rounding. Writes a float GGUF, or copies/repacks an existing NVFP4 GGUF when combined with \fB\-\-outtype nvfp4\fR.
|
||||
.TP
|
||||
.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4}
|
||||
Target data type when learned rounding is skipped. \fBnvfp4\fR copies an already NVFP4 GGUF without re-optimizing;
|
||||
\fBauto\fR leaves tensors as stored in the source file (default: \fBauto\fR).
|
||||
.TP
|
||||
.B \-\-simple
|
||||
Round each value directly onto the E2M1 grid without learned rounding (fast, non-iterative).
|
||||
.TP
|
||||
.B \-\-quantize\-output
|
||||
Also quantize the output (\fIlm_head\fR) matrix. By default, output projection layers are left unquantized.
|
||||
.TP
|
||||
.BI \-\-exclude " " REGEX
|
||||
Regular expression matching GGUF tensor names that should remain in their source precision.
|
||||
.SS Optimization & Learned Rounding
|
||||
.TP
|
||||
.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original}
|
||||
Learned-rounding optimization algorithm (default: \fBauto\fR; selects Prodigy if available, else AdamW).
|
||||
.TP
|
||||
.BI \-\-num\-iter " " N
|
||||
Learned-rounding optimization iterations per matrix (default: 4000).
|
||||
.TP
|
||||
.BI \-\-lr " " LR
|
||||
Initial learning rate (default: 1.0).
|
||||
.TP
|
||||
.BI \-\-lr\-patience " " N
|
||||
Patience steps before decaying the learning rate (default: 5).
|
||||
.TP
|
||||
.BI \-\-lr\-min " " MIN
|
||||
Lowest learning rate floor (default: 1e-8).
|
||||
.TP
|
||||
.BI \-\-lr\-schedule " " {plateau,exponential,adaptive}
|
||||
Learning rate decay schedule (default: \fBplateau\fR).
|
||||
.TP
|
||||
.BI \-\-scale\-optimization " " {fixed,iterative,joint}
|
||||
Block scale tuning strategy: \fBfixed\fR, \fBiterative\fR, or \fBjoint\fR (default: \fBjoint\fR).
|
||||
.TP
|
||||
.BI \-\-scale\-refinement\-rounds " " N
|
||||
Scale refresh iterations when using \fBiterative\fR scale optimization (default: 2).
|
||||
.TP
|
||||
.B \-\-full\-matrix
|
||||
Use full SVD instead of low-rank SVD for reconstruction objective.
|
||||
.TP
|
||||
.BI \-\-top\-p " " P ", " \-\-min\-k " " K ", " \-\-max\-k " " K
|
||||
Singular value decomposition rank selection bounds.
|
||||
.TP
|
||||
.BI \-\-target\-loss " " TARGET
|
||||
Target reconstruction loss threshold for early exit (default: 1e-5; 0 disables).
|
||||
.TP
|
||||
.BI \-\-early\-stop\-stall " " STALL
|
||||
Early stop threshold in steps without loss reduction (default: 2000).
|
||||
.TP
|
||||
.BI \-\-early\-stop\-floor\-stall " " FLOOR
|
||||
Steps without improvement after reaching \fB\-\-lr\-min\fR before terminating matrix search (default: 64; 0 disables).
|
||||
.TP
|
||||
.BI \-\-calib\-samples " " SAMPLES
|
||||
Random calibration vectors used for sibling bias correction (default: 3072).
|
||||
.TP
|
||||
.BI \-\-seed " " SEED
|
||||
Random seed (default: 42).
|
||||
.TP
|
||||
.B \-\-no\-bias\-correction
|
||||
Disable sibling bias correction.
|
||||
.SS System & Execution
|
||||
.TP
|
||||
.BI \-\-device " " DEVICE
|
||||
Compute device: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR).
|
||||
.TP
|
||||
.BI \-\-llama\-cpp " " PATH
|
||||
Path to local llama.cpp repository checkout (default: \fI~/llama.cpp\fR).
|
||||
.TP
|
||||
.BI \-\-convert\-to\-quant " " PATH
|
||||
Path to local convert_to_quant repository checkout.
|
||||
.TP
|
||||
.BI \-\-checkpoint\-dir " " DIR
|
||||
Directory for per-tensor resume checkpoints.
|
||||
.TP
|
||||
.B \-\-fresh
|
||||
Delete checkpoint directory before starting.
|
||||
.TP
|
||||
.B \-\-keep\-checkpoints
|
||||
Keep per-tensor checkpoints on disk after completion.
|
||||
.TP
|
||||
.B \-\-eager
|
||||
Map and load tensors up front.
|
||||
.TP
|
||||
.B \-\-no\-temp\-file
|
||||
Keep finished GGUF tensors in RAM rather than a temporary file.
|
||||
.TP
|
||||
.B \-\-verbose
|
||||
Enable verbose logging.
|
||||
.TP
|
||||
.B \-\-self\-test
|
||||
Verify NVFP4 packing against llama.cpp dequantizer and exit.
|
||||
.SH EXAMPLES
|
||||
.TP
|
||||
.B Quantize an F16 GGUF to NVFP4:
|
||||
.RS
|
||||
.nf
|
||||
gguf_to_nvfp4_gguf -o ./llama3-nvfp4 ./llama-3-8b-f16.gguf
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Fast conversion with simple grid rounding:
|
||||
.RS
|
||||
.nf
|
||||
gguf_to_nvfp4_gguf --simple -o ./qwen-nvfp4 ./qwen2.5-7b-bf16.gguf
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Repack existing NVFP4 GGUF into a new directory with Modelfile:
|
||||
.RS
|
||||
.nf
|
||||
gguf_to_nvfp4_gguf --no-quant --outtype nvfp4 -o ./repacked ./model-nvfp4.gguf
|
||||
.fi
|
||||
.RE
|
||||
.SH EXIT STATUS
|
||||
.TP
|
||||
.B 0
|
||||
Success.
|
||||
.TP
|
||||
.B 1
|
||||
An error occurred during quantization or repacking.
|
||||
.SH SEE ALSO
|
||||
.BR hf_to_nvfp4_gguf (1),
|
||||
.BR safetensors_to_nvfp4_gguf (1),
|
||||
.BR gguf_info (1),
|
||||
.BR ollama_nvfp4 (1)
|
||||
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
gguf_to_nvfp4_gguf.1
|
||||
@@ -0,0 +1,142 @@
|
||||
.TH HERETIC 1 "November 2026" "heretic-llm" "User Commands"
|
||||
.SH NAME
|
||||
heretic \- automated directional ablation and refusal suppression for large language models
|
||||
.SH SYNOPSIS
|
||||
.B heretic
|
||||
\fB\-\-model\fR \fIMODEL\fR
|
||||
[\fB\-\-quantization\fR \fIMETHOD\fR]
|
||||
[\fB\-\-n\-trials\fR \fIN\fR]
|
||||
[\fB\-\-export\-strategy\fR \fISTRATEGY\fR]
|
||||
[\fB\-\-save\-directory\fR \fIDIR\fR]
|
||||
[\fB\-\-upload\-repo\-id\fR \fIREPO\fR]
|
||||
[\fB\-\-reproduce\fR \fIFILE\fR]
|
||||
[\fIoptions...\fR]
|
||||
.SH DESCRIPTION
|
||||
.B heretic
|
||||
(\fBheretic-llm\fR) is an automated optimization and directional ablation tool for
|
||||
modifying and un-aligning Large Language Models (LLMs). It isolates refusal vectors
|
||||
and behavioral directions across internal residual streams and attention/MLP layers,
|
||||
optimizing modifications via genetic multi-objective search (Optuna Pareto front analysis).
|
||||
.PP
|
||||
The tool balances refusal suppression ("abliteration" / "decensoring") against model capability
|
||||
preservation by evaluating candidate modifications against built-in language benchmarks
|
||||
(MMLU, GSM8K, IFEval, AGIEval, BBH, etc.).
|
||||
.PP
|
||||
Modified models can be saved to disk as merged safetensors checkpoints or adapter weights,
|
||||
or uploaded directly to Hugging Face with complete reproducibility logs (\fIreproduce.json\fR).
|
||||
.SH OPTIONS
|
||||
.SS Model & Input
|
||||
.TP
|
||||
.BI \-\-model " " MODEL
|
||||
Hugging Face model repository ID (e.g. \fIgoogle/gemma-2-9b-it\fR) or local filesystem path (required).
|
||||
.TP
|
||||
.BI \-\-model\-commit " " HASH
|
||||
Specific Git commit hash of the Hugging Face model repository.
|
||||
.TP
|
||||
.BI \-\-quantization " " {NONE,BNB_4BIT}
|
||||
Quantization mode used when loading model weights into VRAM: \fBNONE\fR (native precision) or \fBBNB_4BIT\fR (4-bit bitsandbytes). Default: \fBNONE\fR.
|
||||
.TP
|
||||
.BI \-\-dtypes " " LIST
|
||||
List of PyTorch data types to attempt when loading tensors (default: \fI['auto', 'float16', 'bfloat16', 'float32']\fR).
|
||||
.TP
|
||||
.BI \-\-system\-prompt " " STR
|
||||
System prompt passed to the model during evaluation (default: \fI"You are a helpful assistant."\fR).
|
||||
.SS Execution & Memory
|
||||
.TP
|
||||
.BI \-\-device\-map " " MAP
|
||||
Device allocation strategy passed to Hugging Face Accelerate (default: \fBauto\fR).
|
||||
.TP
|
||||
.BI \-\-max\-memory " " DICT
|
||||
Maximum VRAM/RAM limits per device (e.g. \fI'{"0": "22GB", "cpu": "64GB"}'\fR).
|
||||
.TP
|
||||
.BR \-\-offload\-outputs\-to\-cpu ", " \-\-no\-offload\-outputs\-to\-cpu
|
||||
Offload intermediate residual tensors and logits to CPU RAM immediately to reduce peak VRAM usage (default: enabled).
|
||||
.TP
|
||||
.BI \-\-batch\-size " " N
|
||||
Batch size for evaluation sequences. Set to 0 for automatic detection (default: 0).
|
||||
.TP
|
||||
.BI \-\-max\-batch\-size " " N
|
||||
Upper bound during automatic batch size discovery (default: 128).
|
||||
.TP
|
||||
.BI \-\-max\-response\-length " " N
|
||||
Maximum token generation length during prompt scoring.
|
||||
.SS Optimization & Genetic Search
|
||||
.TP
|
||||
.BI \-\-n\-trials " " N
|
||||
Total number of optimization trials to explore across the Pareto front.
|
||||
.TP
|
||||
.BI \-\-n\-startup\-trials " " N
|
||||
Number of initial random trials before heuristic exploration begins.
|
||||
.TP
|
||||
.BI \-\-seed " " SEED
|
||||
Random seed for reproducibility.
|
||||
.TP
|
||||
.BI \-\-study\-checkpoint\-dir " " DIR
|
||||
Directory where Optuna study progression and intermediate trials are saved.
|
||||
.TP
|
||||
.BI \-\-scorers " " JSON
|
||||
Custom or overriding scoring configurations (loss metrics, refusal detectors, perplexity monitors).
|
||||
.TP
|
||||
.BI \-\-modifiers " " JSON
|
||||
Custom modification operators (directional projection, orthogonalization, rank cuts).
|
||||
.SS Benchmarks & Evaluation
|
||||
.TP
|
||||
.BI \-\-benchmarks " " LIST
|
||||
List of evaluation benchmarks to verify model integrity. Supported benchmarks include:
|
||||
\fIagieval\fR, \fIbbh\fR, \fIcommonsense_qa\fR, \fIeq_bench\fR, \fIgsm8k\fR, \fIhellaswag\fR, \fIifeval\fR, \fImmlu\fR, \fImmlu_pro\fR, \fIpiqa\fR, \fIwinogrande\fR.
|
||||
.TP
|
||||
.BI \-\-evaluate\-model " " MODEL
|
||||
Evaluate an existing model relative to the base model instead of running search trials.
|
||||
.SS Export & Publishing
|
||||
.TP
|
||||
.BI \-\-export\-strategy " " {MERGE,ADAPTER}
|
||||
Model export strategy: \fBMERGE\fR (bake modifications directly into output safetensors weights)
|
||||
or \fBADAPTER\fR (export as differential weight delta).
|
||||
.TP
|
||||
.BI \-\-save\-directory " " DIR
|
||||
Local directory path where modified safetensors weights and tokenizer files are written.
|
||||
.TP
|
||||
.BI \-\-max\-shard\-size " " SIZE
|
||||
Maximum file size for exported safetensors shards (default: \fI5GB\fR).
|
||||
.TP
|
||||
.BI \-\-upload\-repo\-id " " REPO
|
||||
Hugging Face repository ID to upload the final modified model to.
|
||||
.TP
|
||||
.B \-\-upload\-repo\-private
|
||||
Make the destination Hugging Face repository private.
|
||||
.TP
|
||||
.BI \-\-upload\-tags " " LIST
|
||||
Tags added to the Hugging Face model card (default: \fI['uncensored', 'decensored', 'abliterated']\fR).
|
||||
.SS Reproduction
|
||||
.TP
|
||||
.BI \-\-reproduce " " PATH_OR_URL
|
||||
Load parameters from an existing \fIreproduce.json\fR file or URL and reproduce the exact model modification.
|
||||
.TP
|
||||
.BI \-\-collect\-reproducibles " " DIR
|
||||
Download all \fIreproduce.json\fR files from public Heretic model repositories on Hugging Face into \fIDIR\fR for archival.
|
||||
.SH EXAMPLES
|
||||
.TP
|
||||
.B Run automated abliteration search on a local model:
|
||||
.RS
|
||||
.nf
|
||||
heretic --model ./meta-llama-3-8b-instruct --n-trials 50 --save-directory ./llama3-abliterated
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Optimize with 4-bit base weights to reduce VRAM:
|
||||
.RS
|
||||
.nf
|
||||
heretic --model mistralai/Mistral-7B-Instruct-v0.3 --quantization BNB_4BIT --save-directory ./mistral-heretic
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Reproduce an existing published Heretic model:
|
||||
.RS
|
||||
.nf
|
||||
heretic --model google/gemma-2-9b-it --reproduce https://huggingface.co/owner/repo/raw/main/reproduce.json --save-directory ./gemma2-reproduced
|
||||
.fi
|
||||
.RE
|
||||
.SH SEE ALSO
|
||||
.BR ctq (1),
|
||||
.BR hf_to_nvfp4_gguf (1)
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
.TH HF_INFO 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
|
||||
.SH NAME
|
||||
hf_info \- print architecture, quantization, and tensor layout from Hugging Face checkpoints
|
||||
.SH SYNOPSIS
|
||||
.B hf_info
|
||||
[\fB\-\-metadata\fR]
|
||||
[\fB\-\-tensors\fR]
|
||||
[\fB\-h\fR|\fB\-\-help\fR]
|
||||
\fImodel\fR [\fImodel\fR ...]
|
||||
.SH DESCRIPTION
|
||||
.B hf_info
|
||||
inspects one or more Hugging Face model directories or standalone \fI.safetensors\fR files.
|
||||
It reads configuration files (\fIconfig.json\fR, \fIgeneration_config.json\fR,
|
||||
\fItokenizer_config.json\fR) and safetensors file headers without loading weight bytes
|
||||
into RAM, allowing rapid inspection of multi-gigabyte or sharded checkpoints.
|
||||
.PP
|
||||
By default,
|
||||
.B hf_info
|
||||
displays a comprehensive summary, including:
|
||||
.IP \(bu 2
|
||||
Model architecture class, text/vision config hierarchy, and model type.
|
||||
.IP \(bu 2
|
||||
Model dimensions: context length, hidden size, intermediate size, layer counts, attention heads, and key-value heads.
|
||||
.IP \(bu 2
|
||||
Parameter counts: total parameters, active parameters, and size on disk.
|
||||
.IP \(bu 2
|
||||
Chat template configuration and capability detection (such as tool/function calling support).
|
||||
.IP \(bu 2
|
||||
Quantization format detection: Hugging Face quantization configs (bitsandbytes, AWQ, GPTQ, FP8),
|
||||
embedded NVFP4 scales, and ComfyUI NVFP4 metadata (\fI_quantization_metadata\fR, \fIweight_scale\fR, \fI.comfy_quant\fR).
|
||||
.IP \(bu 2
|
||||
Sharding structure: list of shard files, header sizes, and index verification against \fImodel.safetensors.index.json\fR.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BR \-h ", " \-\-help
|
||||
Show command-line usage help and exit.
|
||||
.TP
|
||||
.B \-\-metadata
|
||||
Print complete config, generation configuration, tokenizer configuration, and safetensors header metadata.
|
||||
Long strings and large arrays are shortened for readability.
|
||||
.TP
|
||||
.B \-\-tensors
|
||||
Print every tensor name, data type (e.g. \fIBF16\fR, \fIF16\fR, \fIU8\fR), shape, byte size,
|
||||
and the shard file in which it resides.
|
||||
.SH EXAMPLES
|
||||
.TP
|
||||
.B Inspect a Hugging Face model directory:
|
||||
.RS
|
||||
.nf
|
||||
hf_info ./gemma-4-12B-it
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Inspect a standalone safetensors file:
|
||||
.RS
|
||||
.nf
|
||||
hf_info ./model-00001-of-00004.safetensors
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Dump full config and chat template metadata:
|
||||
.RS
|
||||
.nf
|
||||
hf_info \-\-metadata ./meta-llama-3-8b-instruct
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B List all tensors across all checkpoint shards:
|
||||
.RS
|
||||
.nf
|
||||
hf_info \-\-tensors ./mistral-7b-v0.3
|
||||
.fi
|
||||
.RE
|
||||
.SH EXIT STATUS
|
||||
.TP
|
||||
.B 0
|
||||
Success.
|
||||
.TP
|
||||
.B 1
|
||||
An error occurred (e.g. model directory not found or missing configuration).
|
||||
.SH SEE ALSO
|
||||
.BR gguf_info (1),
|
||||
.BR hf_to_nvfp4_gguf (1),
|
||||
.BR hf_to_nvfp4_safetensors (1),
|
||||
.BR safetensors_to_nvfp4_gguf (1)
|
||||
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
hf_info.1
|
||||
@@ -0,0 +1,235 @@
|
||||
.TH HF_TO_NVFP4_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
|
||||
.SH NAME
|
||||
hf_to_nvfp4_gguf \- quantize Hugging Face models to NVFP4 GGUF for llama.cpp and Ollama
|
||||
.SH SYNOPSIS
|
||||
.B hf_to_nvfp4_gguf
|
||||
[\fB\-o\fR \fIOUTPUT\fR]
|
||||
[\fB\-\-modelfile\-only\fR]
|
||||
[\fB\-\-overwrite\fR]
|
||||
[\fB\-\-mmproj\fR \fIMMPROJ\fR]
|
||||
[\fB\-\-context\-length\fR \fILEN\fR]
|
||||
[\fB\-\-no\-quant\fR]
|
||||
[\fB\-\-outtype\fR \fITYPE\fR]
|
||||
[\fB\-\-llama\-cpp\fR \fIPATH\fR]
|
||||
[\fB\-\-convert\-to\-quant\fR \fIPATH\fR]
|
||||
[\fB\-\-model\-name\fR \fINAME\fR]
|
||||
[\fB\-\-device\fR \fIDEVICE\fR]
|
||||
[\fB\-\-simple\fR]
|
||||
[\fB\-\-optimizer\fR \fIOPT\fR]
|
||||
[\fB\-\-num\-iter\fR \fIN\fR]
|
||||
[\fB\-\-lr\fR \fILR\fR]
|
||||
[\fB\-\-lr\-patience\fR \fIN\fR]
|
||||
[\fB\-\-lr\-min\fR \fIMIN\fR]
|
||||
[\fB\-\-lr\-schedule\fR \fISCHED\fR]
|
||||
[\fB\-\-scale\-optimization\fR \fIMODE\fR]
|
||||
[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR]
|
||||
[\fB\-\-full\-matrix\fR]
|
||||
[\fB\-\-top\-p\fR \fIP\fR]
|
||||
[\fB\-\-min\-k\fR \fIK\fR]
|
||||
[\fB\-\-max\-k\fR \fIK\fR]
|
||||
[\fB\-\-early\-stop\-loss\fR \fILOSS\fR]
|
||||
[\fB\-\-early\-stop\-stall\fR \fISTALL\fR]
|
||||
[\fB\-\-target\-loss\fR \fITARGET\fR]
|
||||
[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR]
|
||||
[\fB\-\-calib\-samples\fR \fISAMPLES\fR]
|
||||
[\fB\-\-seed\fR \fISEED\fR]
|
||||
[\fB\-\-no\-bias\-correction\fR]
|
||||
[\fB\-\-quantize\-output\fR]
|
||||
[\fB\-\-exclude\fR \fIREGEX\fR]
|
||||
[\fB\-\-checkpoint\-dir\fR \fIDIR\fR]
|
||||
[\fB\-\-fresh\fR]
|
||||
[\fB\-\-keep\-checkpoints\fR]
|
||||
[\fB\-\-eager\fR]
|
||||
[\fB\-\-no\-temp\-file\fR]
|
||||
[\fB\-\-verbose\fR]
|
||||
[\fB\-\-self\-test\fR]
|
||||
[\fB\-h\fR|\fB\-\-help\fR]
|
||||
\fImodel\fR
|
||||
.SH DESCRIPTION
|
||||
.B hf_to_nvfp4_gguf
|
||||
quantizes a Hugging Face model directory (\fIconfig.json\fR + safetensors) to
|
||||
NVIDIA FP4 (NVFP4) format packaged into a GGUF file suitable for \fBllama.cpp\fR
|
||||
and \fBOllama\fR.
|
||||
.PP
|
||||
The model is loaded through \fBllama.cpp\fR's architecture conversion logic so that
|
||||
all vocabulary, tensor naming conventions, tokenization metadata, and architectural
|
||||
parameters match native GGUF standards. Weight matrices are quantized using learned
|
||||
rounding (via \fBconvert_to_quant\fR) or simple nearest-grid projection onto the E2M1
|
||||
floating point grid with two-level scaling (per-block E4M3/E8M0 scales and per-tensor scales).
|
||||
.PP
|
||||
Along with the output GGUF, an Ollama \fBModelfile\fR is generated automatically,
|
||||
preserving the model's chat template, stop sequences, parameter defaults, and vision projectors.
|
||||
.SH POSITIONAL ARGUMENTS
|
||||
.TP
|
||||
.I model
|
||||
Path to a Hugging Face model directory containing \fIconfig.json\fR and one or more \fI.safetensors\fR files.
|
||||
.SH OPTIONS
|
||||
.SS Output & File Control
|
||||
.TP
|
||||
.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT
|
||||
Output destination directory (default: \fI./<model-name>-NVFP4\fR, or \fI./<model-name>\fR with
|
||||
\fB\-\-no\-quant\fR and \fBauto\fR). If the path ends in \fI.gguf\fR, it specifies the destination
|
||||
GGUF file itself.
|
||||
.TP
|
||||
.B \-\-overwrite
|
||||
Overwrite an existing GGUF, Modelfile, or copied projector without prompting.
|
||||
.TP
|
||||
.B \-\-modelfile\-only
|
||||
Generate only the Ollama \fBModelfile\fR and exit without quantizing or writing GGUF tensors.
|
||||
.TP
|
||||
.BI \-\-mmproj " " MMPROJ
|
||||
Path to a vision projector GGUF. The file is copied into the output directory and added as an
|
||||
additional \fBFROM\fR layer in the Modelfile. May be specified multiple times for multimodal models.
|
||||
.TP
|
||||
.BI \-\-context\-length " " LEN
|
||||
Override the maximum context length recorded in the GGUF metadata. Replaces checkpoint value
|
||||
or defines one if missing.
|
||||
.TP
|
||||
.BI \-\-model\-name " " NAME
|
||||
Override the model name recorded in the GGUF metadata.
|
||||
.SS Quantization Mode
|
||||
.TP
|
||||
.B \-\-no\-quant
|
||||
Skip learned rounding. Writes a float GGUF, or repacks an already NVFP4 checkpoint.
|
||||
.TP
|
||||
.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4}
|
||||
Data type for non-quantized weights when learned rounding is skipped. \fBnvfp4\fR repacks a
|
||||
checkpoint that already has packed 4-bit weights. \fBauto\fR preserves the source float types (default: \fBauto\fR).
|
||||
.TP
|
||||
.B \-\-simple
|
||||
Round weights directly onto the E2M1 grid without iterative learned rounding (fast, non-iterative).
|
||||
.TP
|
||||
.B \-\-quantize\-output
|
||||
Also quantize the final output projection (\fIlm_head\fR) matrix. By default, output layers are
|
||||
kept in BF16/float for generation quality.
|
||||
.TP
|
||||
.BI \-\-exclude " " REGEX
|
||||
Regular expression matching GGUF tensor names that should remain unquantized in BF16.
|
||||
.SS Optimization & Learned Rounding
|
||||
.TP
|
||||
.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original}
|
||||
Learned-rounding optimization algorithm. \fBauto\fR selects Prodigy if installed, otherwise AdamW (default: \fBauto\fR).
|
||||
.TP
|
||||
.BI \-\-num\-iter " " N
|
||||
Learned-rounding optimization iterations per matrix (default: 4000).
|
||||
.TP
|
||||
.BI \-\-lr " " LR
|
||||
Initial learning rate (default: 1.0).
|
||||
.TP
|
||||
.BI \-\-lr\-patience " " N
|
||||
Number of steps without improvement before decaying the learning rate (default: 5).
|
||||
.TP
|
||||
.BI \-\-lr\-min " " MIN
|
||||
Minimum learning rate floor. The flat-loss termination check begins once this floor is reached (default: 1e-8).
|
||||
.TP
|
||||
.BI \-\-lr\-schedule " " {plateau,exponential,adaptive}
|
||||
Learning rate decay schedule (default: \fBplateau\fR).
|
||||
.TP
|
||||
.BI \-\-scale\-optimization " " {fixed,iterative,joint}
|
||||
Block scale tuning strategy. \fBfixed\fR keeps initial scales; \fBiterative\fR refreshes scales periodically
|
||||
using the original optimizer; \fBjoint\fR trains block scales concurrently with the codes (default: \fBjoint\fR).
|
||||
.TP
|
||||
.BI \-\-scale\-refinement\-rounds " " N
|
||||
Number of scale refreshes when \fB\-\-scale\-optimization\fR is set to \fBiterative\fR (default: 2).
|
||||
.TP
|
||||
.B \-\-full\-matrix
|
||||
Use full singular value decomposition (SVD) rather than a low-rank approximation for the loss objective.
|
||||
.TP
|
||||
.BI \-\-top\-p " " P
|
||||
Cumulative energy cutoff for low-rank SVD (default: 0.2).
|
||||
.TP
|
||||
.BI \-\-min\-k " " K ", " \-\-max\-k " " K
|
||||
Minimum and maximum rank limits for low-rank SVD (default: 128 and 1280).
|
||||
.TP
|
||||
.BI \-\-target\-loss " " TARGET
|
||||
Stop optimization when best reconstruction loss reaches this threshold (default: 1e-5; 0 disables).
|
||||
.TP
|
||||
.BI \-\-early\-stop\-stall " " STALL
|
||||
Stop optimization after this many steps without reaching a new best loss (default: 2000).
|
||||
.TP
|
||||
.BI \-\-early\-stop\-floor\-stall " " FLOOR
|
||||
Stop optimization after this many steps without improvement once the learning rate has reached \fB\-\-lr\-min\fR (default: 64; 0 disables).
|
||||
.TP
|
||||
.BI \-\-calib\-samples " " SAMPLES
|
||||
Number of random Gaussian calibration inputs used to calculate sibling bias correction (default: 3072).
|
||||
.TP
|
||||
.BI \-\-seed " " SEED
|
||||
Random seed for calibration vectors (default: 42).
|
||||
.TP
|
||||
.B \-\-no\-bias\-correction
|
||||
Disable sibling bias correction on downstream biases.
|
||||
.SS System & Execution
|
||||
.TP
|
||||
.BI \-\-device " " DEVICE
|
||||
Device to perform optimization on: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR).
|
||||
.TP
|
||||
.BI \-\-llama\-cpp " " PATH
|
||||
Path to local \fBllama.cpp\fR repository checkout (default: \fI~/llama.cpp\fR).
|
||||
.TP
|
||||
.BI \-\-convert\-to\-quant " " PATH
|
||||
Path to local \fBconvert_to_quant\fR repository checkout, if not installed as a Python package.
|
||||
.TP
|
||||
.BI \-\-checkpoint\-dir " " DIR
|
||||
Directory used to store and resume per-tensor quantization checkpoints.
|
||||
.TP
|
||||
.B \-\-fresh
|
||||
Remove existing checkpoint directory before starting.
|
||||
.TP
|
||||
.B \-\-keep\-checkpoints
|
||||
Preserve per-tensor checkpoints after successful conversion.
|
||||
.TP
|
||||
.B \-\-eager
|
||||
Load all model tensors into memory up front rather than streaming (requires more system RAM).
|
||||
.TP
|
||||
.B \-\-no\-temp\-file
|
||||
Keep finished GGUF tensors in memory rather than buffering through a temporary scratch file.
|
||||
.TP
|
||||
.B \-\-verbose
|
||||
Enable detailed verbose logging.
|
||||
.TP
|
||||
.B \-\-self\-test
|
||||
Run internal verification of NVFP4 packing against llama.cpp's reference dequantizer and exit immediately.
|
||||
.SH EXAMPLES
|
||||
.TP
|
||||
.B Full learned-rounding conversion to GGUF:
|
||||
.RS
|
||||
.nf
|
||||
hf_to_nvfp4_gguf -o ./llama3-8b-NVFP4 ./meta-llama-3-8b
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Fast quantization with simple grid rounding:
|
||||
.RS
|
||||
.nf
|
||||
hf_to_nvfp4_gguf --simple -o ./qwen-nvfp4 ./Qwen2.5-7B-Instruct
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Multi-modal model with vision projector:
|
||||
.RS
|
||||
.nf
|
||||
hf_to_nvfp4_gguf --mmproj mmproj-model-f16.gguf -o ./vision-nvfp4 ./Qwen2-VL-7B
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Specify CUDA GPU and custom llama.cpp directory:
|
||||
.RS
|
||||
.nf
|
||||
hf_to_nvfp4_gguf --llama-cpp /opt/llama.cpp --device cuda:0 -o ./output ./model
|
||||
.fi
|
||||
.RE
|
||||
.SH EXIT STATUS
|
||||
.TP
|
||||
.B 0
|
||||
Success.
|
||||
.TP
|
||||
.B 1
|
||||
An error occurred during conversion or validation.
|
||||
.SH SEE ALSO
|
||||
.BR gguf_to_nvfp4_gguf (1),
|
||||
.BR safetensors_to_nvfp4_gguf (1),
|
||||
.BR hf_to_nvfp4_safetensors (1),
|
||||
.BR hf_info (1),
|
||||
.BR gguf_info (1),
|
||||
.BR ollama_nvfp4 (1)
|
||||
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
hf_to_nvfp4_gguf.1
|
||||
@@ -0,0 +1,169 @@
|
||||
.TH HF_TO_NVFP4_SAFETENSORS 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
|
||||
.SH NAME
|
||||
hf_to_nvfp4_safetensors \- quantize a Hugging Face directory to a ComfyUI NVFP4 safetensors file
|
||||
.SH SYNOPSIS
|
||||
.B hf_to_nvfp4_safetensors
|
||||
[\fB\-o\fR \fIOUTPUT\fR]
|
||||
[\fB\-\-overwrite\fR]
|
||||
[\fB\-\-output\-dtype\fR \fIDTYPE\fR]
|
||||
[\fB\-\-low\-memory\fR]
|
||||
[\fB\-\-heur\fR]
|
||||
[\fB\-\-exclude\-layers\fR \fIREGEX\fR]
|
||||
[\fB\-\-convert\-to\-quant\fR \fIPATH\fR]
|
||||
[\fB\-\-simple\fR]
|
||||
[\fB\-\-optimizer\fR \fIOPT\fR]
|
||||
[\fB\-\-num\-iter\fR \fIN\fR]
|
||||
[\fB\-\-lr\fR \fILR\fR]
|
||||
[\fB\-\-lr\-min\fR \fIMIN\fR]
|
||||
[\fB\-\-lr\-patience\fR \fIN\fR]
|
||||
[\fB\-\-lr\-schedule\fR \fISCHED\fR]
|
||||
[\fB\-\-scale\-optimization\fR \fIMODE\fR]
|
||||
[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR]
|
||||
[\fB\-\-full\-matrix\fR]
|
||||
[\fB\-\-top\-p\fR \fIP\fR]
|
||||
[\fB\-\-min\-k\fR \fIK\fR]
|
||||
[\fB\-\-max\-k\fR \fIK\fR]
|
||||
[\fB\-\-early\-stop\-loss\fR \fILOSS\fR]
|
||||
[\fB\-\-early\-stop\-stall\fR \fISTALL\fR]
|
||||
[\fB\-\-calib\-samples\fR \fISAMPLES\fR]
|
||||
[\fB\-\-seed\fR \fISEED\fR]
|
||||
[\fB\-\-use\-speed\fR]
|
||||
[\fB\-\-verbose\fR]
|
||||
[\fB\-h\fR|\fB\-\-help\fR]
|
||||
\fImodel\fR
|
||||
.SH DESCRIPTION
|
||||
.B hf_to_nvfp4_safetensors
|
||||
quantizes a Hugging Face model directory (\fIconfig.json\fR and sharded \fI.safetensors\fR files)
|
||||
into a single consolidated \fI.safetensors\fR file formatted for ComfyUI's native NVFP4 loader.
|
||||
.PP
|
||||
For each quantized weight matrix, the script outputs packed NVFP4 bytes alongside
|
||||
companion scaling tensors:
|
||||
.IP \(bu 2
|
||||
\fI<name>.weight\fR (packed NVFP4 data, two 4-bit values per byte)
|
||||
.IP \(bu 2
|
||||
\fI<name>.weight_scale\fR (per-block E4M3 scale factors)
|
||||
.IP \(bu 2
|
||||
\fI<name>.weight_scale_2\fR (per-tensor scale factor)
|
||||
.IP \(bu 2
|
||||
\fI<name>.comfy_quant\fR (metadata tensor specifying quantization type)
|
||||
.PP
|
||||
A unified \fI_quantization_metadata\fR header is embedded in the output safetensors header.
|
||||
When the source model is split across multiple shards, each shard is quantized in a temporary
|
||||
workspace and then efficiently merged into the final single output file.
|
||||
.PP
|
||||
Normalization layers, embeddings, biases, and layers matching \fB\-\-exclude\-layers\fR
|
||||
remain in the target float format (controlled by \fB\-\-output\-dtype\fR).
|
||||
.SH POSITIONAL ARGUMENTS
|
||||
.TP
|
||||
.I model
|
||||
Hugging Face model directory containing \fIconfig.json\fR and \fI.safetensors\fR weights.
|
||||
.SH OPTIONS
|
||||
.SS Output & Memory Control
|
||||
.TP
|
||||
.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT
|
||||
Output \fI.safetensors\fR file path (default: \fI./<model-name>-NVFP4.safetensors\fR).
|
||||
.TP
|
||||
.B \-\-overwrite
|
||||
Overwrite an existing output file without prompting.
|
||||
.TP
|
||||
.BI \-\-output\-dtype " " {bfloat16,float16}
|
||||
Data type recorded for quantized layer scales and used for unquantized 2D weights (default: \fBbfloat16\fR).
|
||||
.TP
|
||||
.B \-\-low\-memory
|
||||
Stream each shard sequentially through quantization rather than loading the entire model shard into RAM at once.
|
||||
.TP
|
||||
.B \-\-heur
|
||||
Skip layers that convert_to_quant's built-in size heuristic rejects.
|
||||
.TP
|
||||
.BI \-\-exclude\-layers " " REGEX
|
||||
Regular expression matching layer or tensor names to leave unquantized in float format.
|
||||
.SS Quantization & Optimization
|
||||
.TP
|
||||
.B \-\-simple
|
||||
Round onto the E2M1 grid directly without iterative learned rounding.
|
||||
.TP
|
||||
.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original}
|
||||
Learned-rounding optimizer. \fBauto\fR uses Prodigy when installed, otherwise AdamW (default: \fBauto\fR).
|
||||
.TP
|
||||
.BI \-\-num\-iter " " N
|
||||
Learned-rounding iterations per matrix (default: 4000).
|
||||
.TP
|
||||
.BI \-\-lr " " LR
|
||||
Initial learning rate (default: 1.0).
|
||||
.TP
|
||||
.BI \-\-lr\-min " " MIN
|
||||
Lowest learning rate floor (default: 1e-8).
|
||||
.TP
|
||||
.BI \-\-lr\-patience " " N
|
||||
Patience steps before decaying learning rate (default: 5).
|
||||
.TP
|
||||
.BI \-\-lr\-schedule " " {plateau,exponential,adaptive}
|
||||
Learning rate decay schedule (default: \fBplateau\fR).
|
||||
.TP
|
||||
.BI \-\-scale\-optimization " " {fixed,iterative,joint}
|
||||
Scale tuning strategy: \fBfixed\fR, \fBiterative\fR, or \fBjoint\fR (default: \fBjoint\fR).
|
||||
.TP
|
||||
.BI \-\-scale\-refinement\-rounds " " N
|
||||
Scale refresh iterations when using iterative scale optimization (default: 2).
|
||||
.TP
|
||||
.B \-\-full\-matrix
|
||||
Use full SVD instead of low-rank SVD.
|
||||
.TP
|
||||
.BI \-\-top\-p " " P ", " \-\-min\-k " " K ", " \-\-max\-k " " K
|
||||
Singular value decomposition rank selection bounds.
|
||||
.TP
|
||||
.BI \-\-early\-stop\-loss " " LOSS
|
||||
Early stopping loss target (default: 5e-9).
|
||||
.TP
|
||||
.BI \-\-early\-stop\-stall " " STALL
|
||||
Stop after this many steps without loss improvement (default: 2000).
|
||||
.TP
|
||||
.BI \-\-calib\-samples " " SAMPLES
|
||||
Random Gaussian calibration inputs for bias correction (default: 3072).
|
||||
.TP
|
||||
.BI \-\-seed " " SEED
|
||||
Random seed (default: 42).
|
||||
.TP
|
||||
.B \-\-use\-speed
|
||||
Enable Prodigy optimizer's \fIuse_speed\fR mode.
|
||||
.SS System & Execution
|
||||
.TP
|
||||
.BI \-\-convert\-to\-quant " " PATH
|
||||
Path to local convert_to_quant repository checkout.
|
||||
.TP
|
||||
.B \-\-verbose
|
||||
Enable verbose output during shard quantization and merge.
|
||||
.SH EXAMPLES
|
||||
.TP
|
||||
.B Quantize a model directory into a ComfyUI NVFP4 safetensors file:
|
||||
.RS
|
||||
.nf
|
||||
hf_to_nvfp4_safetensors -o gemma-4-12B-it-NVFP4.safetensors ./gemma-4-12B-it
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Low-memory streaming mode with float16 weights:
|
||||
.RS
|
||||
.nf
|
||||
hf_to_nvfp4_safetensors --low-memory --output-dtype float16 -o ./model-nvfp4.safetensors ./model
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Exclude specific projection layers:
|
||||
.RS
|
||||
.nf
|
||||
hf_to_nvfp4_safetensors --exclude-layers ".*gate_proj.*" -o ./model-nvfp4.safetensors ./model
|
||||
.fi
|
||||
.RE
|
||||
.SH EXIT STATUS
|
||||
.TP
|
||||
.B 0
|
||||
Success.
|
||||
.TP
|
||||
.B 1
|
||||
An error occurred during quantization or shard merging.
|
||||
.SH SEE ALSO
|
||||
.BR safetensors_to_nvfp4_gguf (1),
|
||||
.BR hf_to_nvfp4_gguf (1),
|
||||
.BR hf_info (1)
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
hf_to_nvfp4_safetensors.1
|
||||
@@ -0,0 +1,181 @@
|
||||
.TH OLLAMA_NVFP4 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
|
||||
.SH NAME
|
||||
ollama_nvfp4 \- inspect Ollama sha256 model stores and quantize models to NVFP4 GGUF
|
||||
.SH SYNOPSIS
|
||||
.B ollama_nvfp4
|
||||
[\fB\-\-models\fR \fIDIR\fR]
|
||||
.I subcommand
|
||||
[\fIoptions...\fR]
|
||||
.br
|
||||
.B ollama_nvfp4
|
||||
[\fB\-\-models\fR \fIDIR\fR]
|
||||
.B \-\-info
|
||||
\fImodel\fR
|
||||
[\fIgguf_info_args...\fR]
|
||||
.SH DESCRIPTION
|
||||
.B ollama_nvfp4
|
||||
is a management and quantization utility for local \fBOllama\fR model installations.
|
||||
Ollama stores model artifacts as content-addressed blobs (\fIblobs/sha256-<digest>\fR)
|
||||
referenced by JSON manifests (\fImanifests/\fR).
|
||||
.PP
|
||||
.B ollama_nvfp4
|
||||
resolves these hashes into human-readable model names, displays model capabilities and
|
||||
parameters, exports native GGUF weights alongside reconstructed \fBModelfile\fRs,
|
||||
and directly quantizes models to NVIDIA FP4 (NVFP4) format. Quantized models can
|
||||
optionally be registered straight back into Ollama's local library.
|
||||
.PP
|
||||
The generated \fBModelfile\fR preserves the original prompt template, stop sequences,
|
||||
system message, renderer, and parser settings so that thinking/reasoning modes and
|
||||
generation behavior remain intact.
|
||||
.SH GLOBAL OPTIONS
|
||||
.TP
|
||||
.BI \-\-models " " DIR
|
||||
Path to the Ollama models store directory containing \fIblobs/\fR and \fImanifests/\fR.
|
||||
If not provided, the location is resolved via the \fBOLLAMA_MODELS\fR environment variable,
|
||||
\fI/models/ollama\fR, or \fI~/.ollama/models\fR.
|
||||
.TP
|
||||
.BR \-h ", " \-\-help
|
||||
Show command help and exit.
|
||||
.SH SUBCOMMANDS
|
||||
.SS list
|
||||
.B ollama_nvfp4
|
||||
[\fB\-\-models\fR \fIDIR\fR]
|
||||
.B list
|
||||
[\fB\-\-full\-scan\fR]
|
||||
[\fB\-\-ctx\-scan\fR]
|
||||
.PP
|
||||
Display every manifest in the store as a readable model entry with family, quantization type,
|
||||
and capability badges (chat, tool call, thinking, vision).
|
||||
.TP
|
||||
.B \-\-full\-scan
|
||||
Read the underlying GGUF header of each model blob. This is slower on large stores but reveals
|
||||
capabilities and vision encoders not declared in manifest metadata.
|
||||
.TP
|
||||
.B \-\-ctx\-scan
|
||||
Scan parameter blobs and report models whose Modelfile explicitly configures \fInum_ctx\fR.
|
||||
.SS show
|
||||
.B ollama_nvfp4
|
||||
[\fB\-\-models\fR \fIDIR\fR]
|
||||
.B show
|
||||
\fImodel\fR
|
||||
.PP
|
||||
Display detailed information for a specified model, including manifest layers, sha256 blob
|
||||
hashes, component media types, GGUF architecture metadata, context length, and system settings.
|
||||
The \fImodel\fR argument can be a full name (e.g. \fIllama3.2:latest\fR) or a unique substring.
|
||||
.SS export
|
||||
.B ollama_nvfp4
|
||||
[\fB\-\-models\fR \fIDIR\fR]
|
||||
.B export
|
||||
[\fB\-o\fR \fIDIR\fR]
|
||||
[\fB\-\-modelfile\-only\fR]
|
||||
[\fB\-\-overwrite\fR|\fB\-\-force\fR]
|
||||
\fImodel\fR
|
||||
.PP
|
||||
Export the model's underlying GGUF weight blob and generate a fully configured \fBModelfile\fR
|
||||
into the destination directory (default: \fI./<model>\fR).
|
||||
.TP
|
||||
.BI \-o " " DIR ", " \-\-output " " DIR
|
||||
Output directory path.
|
||||
.TP
|
||||
.B \-\-modelfile\-only
|
||||
Write only the Modelfile without copying the GGUF or vision projector blobs.
|
||||
.TP
|
||||
.BR \-\-overwrite ", " \-\-force
|
||||
Replace existing files in the output directory.
|
||||
.SS quant
|
||||
.B ollama_nvfp4
|
||||
[\fB\-\-models\fR \fIDIR\fR]
|
||||
.B quant
|
||||
[\fB\-o\fR \fIDIR\fR]
|
||||
[\fB\-\-name\fR \fINAME\fR]
|
||||
[\fB\-\-install\fR]
|
||||
[\fB\-\-float\-only\fR]
|
||||
[\fB\-\-context\-length\fR \fILEN\fR]
|
||||
[\fIquant_opts...\fR]
|
||||
\fImodel\fR
|
||||
.PP
|
||||
Quantize an Ollama model to NVFP4 GGUF format and generate an accompanying Modelfile.
|
||||
.TP
|
||||
.BI \-o " " DIR ", " \-\-output " " DIR
|
||||
Output directory (default: \fI./<model>-NVFP4\fR).
|
||||
.TP
|
||||
.BI \-\-name " " NAME
|
||||
Model tag used when registering with \fB\-\-install\fR (default: \fI<model>:nvfp4\fR).
|
||||
.TP
|
||||
.B \-\-install
|
||||
Automatically register the quantized model back into Ollama using \fIollama create\fR.
|
||||
.TP
|
||||
.B \-\-float\-only
|
||||
Refuse to requantize a model if its source blob is already quantized (e.g. \fIQ4_K_M\fR).
|
||||
Quantizing from float16/BF16 produces superior NVFP4 fidelity.
|
||||
.TP
|
||||
.BI \-\-context\-length " " LEN
|
||||
Override maximum context length stored in the GGUF.
|
||||
.TP
|
||||
.I quant_opts
|
||||
All standard learned-rounding and execution options supported by \fBgguf_to_nvfp4_gguf\fR
|
||||
are supported: \fB\-\-simple\fR, \fB\-\-optimizer\fR, \fB\-\-lr\fR, \fB\-\-num\-iter\fR,
|
||||
\fB\-\-scale\-optimization\fR, \fB\-\-device\fR, \fB\-\-checkpoint\-dir\fR, etc.
|
||||
.SS \-\-info
|
||||
.B ollama_nvfp4
|
||||
[\fB\-\-models\fR \fIDIR\fR]
|
||||
.B \-\-info
|
||||
\fImodel\fR
|
||||
[\fIgguf_info_args...\fR]
|
||||
.PP
|
||||
Resolve \fImodel\fR to its sha256 GGUF weight blob in the store and invoke
|
||||
.BR gguf_info (1)
|
||||
directly on the file. Any subsequent flags (such as \fB\-\-metadata\fR or \fB\-\-tensors\fR)
|
||||
are forwarded directly to \fBgguf_info\fR.
|
||||
.SH ENVIRONMENT VARIABLES
|
||||
.TP
|
||||
.B OLLAMA_MODELS
|
||||
Path to the default Ollama models directory. Overridden by \fB\-\-models\fR.
|
||||
.SH EXAMPLES
|
||||
.TP
|
||||
.B List installed models with capability summary:
|
||||
.RS
|
||||
.nf
|
||||
ollama_nvfp4 list
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Inspect detailed manifest and parameters of a model:
|
||||
.RS
|
||||
.nf
|
||||
ollama_nvfp4 show llama3.2:3b
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Run gguf_info on an Ollama model's weight blob:
|
||||
.RS
|
||||
.nf
|
||||
ollama_nvfp4 --info llama3.2:3b --metadata
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Export a model and its Modelfile:
|
||||
.RS
|
||||
.nf
|
||||
ollama_nvfp4 export llama3.2:3b -o ./llama3-export
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Quantize an Ollama FP16 model to NVFP4 and install it:
|
||||
.RS
|
||||
.nf
|
||||
ollama_nvfp4 quant --install llama3.2:3b-instruct-fp16
|
||||
.fi
|
||||
.RE
|
||||
.SH EXIT STATUS
|
||||
.TP
|
||||
.B 0
|
||||
Success.
|
||||
.TP
|
||||
.B 1
|
||||
An error occurred (e.g. model not found, invalid store directory, or quantization failure).
|
||||
.SH SEE ALSO
|
||||
.BR gguf_info (1),
|
||||
.BR gguf_to_nvfp4_gguf (1),
|
||||
.BR hf_to_nvfp4_gguf (1)
|
||||
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
ollama_nvfp4.1
|
||||
@@ -0,0 +1,219 @@
|
||||
.TH SAFETENSORS_TO_NVFP4_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
|
||||
.SH NAME
|
||||
safetensors_to_nvfp4_gguf \- quantize a safetensors file to an NVFP4 GGUF weight container
|
||||
.SH SYNOPSIS
|
||||
.B safetensors_to_nvfp4_gguf
|
||||
[\fB\-o\fR \fIOUTPUT\fR]
|
||||
[\fB\-\-all\-weights\fR]
|
||||
[\fB\-\-no\-quant\fR]
|
||||
[\fB\-\-outtype\fR \fITYPE\fR]
|
||||
[\fB\-\-llama\-cpp\fR \fIPATH\fR]
|
||||
[\fB\-\-convert\-to\-quant\fR \fIPATH\fR]
|
||||
[\fB\-\-overwrite\fR|\fB\-\-force\fR]
|
||||
[\fB\-\-mmproj\fR \fIMMPROJ\fR]
|
||||
[\fB\-\-context\-length\fR \fILEN\fR]
|
||||
[\fB\-\-checkpoint\-dir\fR \fIDIR\fR]
|
||||
[\fB\-\-fresh\fR]
|
||||
[\fB\-\-keep\-checkpoints\fR]
|
||||
[\fB\-\-no\-temp\-file\fR]
|
||||
[\fB\-\-verbose\fR]
|
||||
[\fB\-\-self\-test\fR]
|
||||
[\fB\-\-device\fR \fIDEVICE\fR]
|
||||
[\fB\-\-simple\fR]
|
||||
[\fB\-\-optimizer\fR \fIOPT\fR]
|
||||
[\fB\-\-num\-iter\fR \fIN\fR]
|
||||
[\fB\-\-lr\fR \fILR\fR]
|
||||
[\fB\-\-lr\-schedule\fR \fISCHED\fR]
|
||||
[\fB\-\-scale\-optimization\fR \fIMODE\fR]
|
||||
[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR]
|
||||
[\fB\-\-full\-matrix\fR]
|
||||
[\fB\-\-top\-p\fR \fIP\fR]
|
||||
[\fB\-\-min\-k\fR \fIK\fR]
|
||||
[\fB\-\-max\-k\fR \fIK\fR]
|
||||
[\fB\-\-early\-stop\-loss\fR \fILOSS\fR]
|
||||
[\fB\-\-early\-stop\-lr\fR \fILR\fR]
|
||||
[\fB\-\-early\-stop\-stall\fR \fISTALL\fR]
|
||||
[\fB\-\-target\-loss\fR \fITARGET\fR]
|
||||
[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR]
|
||||
[\fB\-\-lr\-min\fR \fIMIN\fR]
|
||||
[\fB\-\-lr\-patience\fR \fIN\fR]
|
||||
[\fB\-\-use\-speed\fR]
|
||||
[\fB\-\-calib\-samples\fR \fISAMPLES\fR]
|
||||
[\fB\-\-seed\fR \fISEED\fR]
|
||||
[\fB\-\-no\-bias\-correction\fR]
|
||||
[\fB\-\-quantize\-output\fR]
|
||||
[\fB\-\-exclude\fR \fIREGEX\fR]
|
||||
[\fB\-h\fR|\fB\-\-help\fR]
|
||||
\fIsafetensors\fR
|
||||
.SH DESCRIPTION
|
||||
.B safetensors_to_nvfp4_gguf
|
||||
quantizes the tensor weights stored in a single \fI.safetensors\fR file directly into
|
||||
an NVFP4 GGUF weight container, without requiring a Hugging Face model directory or
|
||||
tokenizer configuration.
|
||||
.PP
|
||||
This tool is particularly useful for standalone diffusion models (e.g. FLUX, SDXL),
|
||||
isolated LoRA weights, or intermediate checkpoints. All original tensor names are
|
||||
preserved.
|
||||
.PP
|
||||
By default, layers on \fBconvert_to_quant\fR's standard exclusion list (normalization
|
||||
layers, biases, embeddings, and output heads) remain unquantized in their original precision.
|
||||
Passing \fB\-\-all\-weights\fR overrides this heuristic and quantizes every eligible 2D
|
||||
matrix whose trailing dimension is a multiple of 64.
|
||||
.PP
|
||||
The tool also supports repacking ComfyUI-style NVFP4 \fI.safetensors\fR files (which contain
|
||||
\fIweight_scale\fR and \fIweight_scale_2\fR) into GGUF format without re-optimization using
|
||||
\fB\-\-no\-quant \-\-outtype nvfp4\fR.
|
||||
.SH POSITIONAL ARGUMENTS
|
||||
.TP
|
||||
.I safetensors
|
||||
Path to the input \fI.safetensors\fR file.
|
||||
.SH OPTIONS
|
||||
.SS Output & Packaging
|
||||
.TP
|
||||
.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT
|
||||
Output GGUF file path (default: \fI<stem>-NVFP4.gguf\fR, or \fI<stem>.gguf\fR with \fB\-\-no\-quant\fR and \fBauto\fR).
|
||||
.TP
|
||||
.BR \-\-overwrite ", " \-\-force
|
||||
Replace an existing GGUF or auxiliary output file without prompting.
|
||||
.TP
|
||||
.B \-\-all\-weights
|
||||
Quantize every 2D weight matrix eligible for 4-bit block packing, ignoring convert_to_quant's
|
||||
default exclusion list.
|
||||
.TP
|
||||
.B \-\-no\-quant
|
||||
Skip learned rounding. Writes a float GGUF or repacks ComfyUI NVFP4 weights.
|
||||
.TP
|
||||
.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4}
|
||||
Data type when learned rounding is skipped: \fBnvfp4\fR repacks ComfyUI NVFP4 weights; \fBauto\fR preserves original dtypes (default: \fBauto\fR).
|
||||
.TP
|
||||
.BI \-\-mmproj " " MMPROJ
|
||||
Path to a vision projector GGUF to copy next to output. May be specified multiple times.
|
||||
.TP
|
||||
.BI \-\-context\-length " " LEN
|
||||
Maximum context length to store in the GGUF metadata.
|
||||
.SS Quantization & Optimization
|
||||
.TP
|
||||
.B \-\-simple
|
||||
Round each value directly onto the E2M1 grid without learned rounding.
|
||||
.TP
|
||||
.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original}
|
||||
Learned-rounding optimizer (default: \fBauto\fR).
|
||||
.TP
|
||||
.BI \-\-num\-iter " " N
|
||||
Optimization iterations per matrix (default: 4000).
|
||||
.TP
|
||||
.BI \-\-lr " " LR
|
||||
Initial learning rate (default: 1.0).
|
||||
.TP
|
||||
.BI \-\-lr\-schedule " " {plateau,exponential,adaptive}
|
||||
Learning rate decay schedule (default: \fBplateau\fR).
|
||||
.TP
|
||||
.BI \-\-lr\-min " " MIN
|
||||
Lowest learning rate floor (default: 1e-8).
|
||||
.TP
|
||||
.BI \-\-lr\-patience " " N
|
||||
Patience steps before decaying the learning rate (default: 5).
|
||||
.TP
|
||||
.B \-\-use\-speed
|
||||
Enable Prodigy optimizer's \fIuse_speed\fR mode.
|
||||
.TP
|
||||
.BI \-\-scale\-optimization " " {fixed,iterative,joint}
|
||||
Block scale tuning method (default: \fBjoint\fR).
|
||||
.TP
|
||||
.BI \-\-scale\-refinement\-rounds " " N
|
||||
Scale refresh iterations for iterative mode (default: 2).
|
||||
.TP
|
||||
.B \-\-full\-matrix
|
||||
Use full SVD instead of low-rank SVD.
|
||||
.TP
|
||||
.BI \-\-top\-p " " P ", " \-\-min\-k " " K ", " \-\-max\-k " " K
|
||||
Singular value decomposition rank selection bounds.
|
||||
.TP
|
||||
.BI \-\-target\-loss " " TARGET
|
||||
Target reconstruction loss threshold (default: 1e-5; 0 disables).
|
||||
.TP
|
||||
.BI \-\-early\-stop\-stall " " STALL
|
||||
Early stop stall steps without reaching a new best loss (default: 2000).
|
||||
.TP
|
||||
.BI \-\-early\-stop\-floor\-stall " " FLOOR
|
||||
Stall steps at minimum learning rate before stopping (default: 64; 0 disables).
|
||||
.TP
|
||||
.BI \-\-early\-stop\-lr " " LR
|
||||
Learning rate threshold for early stop check.
|
||||
.TP
|
||||
.BI \-\-calib\-samples " " SAMPLES
|
||||
Random Gaussian calibration inputs for sibling bias correction (default: 3072).
|
||||
.TP
|
||||
.BI \-\-seed " " SEED
|
||||
Random seed (default: 42).
|
||||
.TP
|
||||
.B \-\-no\-bias\-correction
|
||||
Disable sibling bias correction.
|
||||
.TP
|
||||
.B \-\-quantize\-output
|
||||
Also quantize \fIoutput.weight\fR.
|
||||
.TP
|
||||
.BI \-\-exclude " " REGEX
|
||||
Regex matching tensor names to leave unquantized.
|
||||
.SS System & Execution
|
||||
.TP
|
||||
.BI \-\-device " " DEVICE
|
||||
Device for quantization computations: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR).
|
||||
.TP
|
||||
.BI \-\-llama\-cpp " " PATH
|
||||
Path to llama.cpp repository checkout (default: \fI~/llama.cpp\fR).
|
||||
.TP
|
||||
.BI \-\-convert\-to\-quant " " PATH
|
||||
Path to convert_to_quant checkout.
|
||||
.TP
|
||||
.BI \-\-checkpoint\-dir " " DIR
|
||||
Directory for per-tensor resume checkpoints.
|
||||
.TP
|
||||
.B \-\-fresh
|
||||
Delete checkpoint directory before starting.
|
||||
.TP
|
||||
.B \-\-keep\-checkpoints
|
||||
Keep per-tensor checkpoints after completion.
|
||||
.TP
|
||||
.B \-\-no\-temp\-file
|
||||
Keep finished GGUF tensors in memory rather than a temporary file.
|
||||
.TP
|
||||
.B \-\-verbose
|
||||
Enable verbose logging.
|
||||
.TP
|
||||
.B \-\-self\-test
|
||||
Verify NVFP4 packing against llama.cpp dequantizer and exit.
|
||||
.SH EXAMPLES
|
||||
.TP
|
||||
.B Quantize a standalone diffusion model:
|
||||
.RS
|
||||
.nf
|
||||
safetensors_to_nvfp4_gguf -o flux1-dev-nvfp4.gguf flux1-dev.safetensors
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Repack ComfyUI NVFP4 safetensors to GGUF:
|
||||
.RS
|
||||
.nf
|
||||
safetensors_to_nvfp4_gguf --no-quant --outtype nvfp4 -o repacked.gguf comfy-nvfp4.safetensors
|
||||
.fi
|
||||
.RE
|
||||
.TP
|
||||
.B Quantize every eligible weight matrix with simple grid rounding:
|
||||
.RS
|
||||
.nf
|
||||
safetensors_to_nvfp4_gguf --simple --all-weights -o fast-nvfp4.gguf model.safetensors
|
||||
.fi
|
||||
.RE
|
||||
.SH EXIT STATUS
|
||||
.TP
|
||||
.B 0
|
||||
Success.
|
||||
.TP
|
||||
.B 1
|
||||
An error occurred during conversion or packaging.
|
||||
.SH SEE ALSO
|
||||
.BR hf_to_nvfp4_safetensors (1),
|
||||
.BR hf_to_nvfp4_gguf (1),
|
||||
.BR gguf_to_nvfp4_gguf (1)
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
safetensors_to_nvfp4_gguf.1
|
||||
Reference in new issue
Block a user