From b6037b3ea5f76f1fba462ceb44a69ac745336561 Mon Sep 17 00:00:00 2001 From: zefie Date: Tue, 3 Nov 2026 15:23:48 -0500 Subject: [PATCH] manfiles and new scripts --- Dockerfile | 48 ++++- docker/gguf_info | 2 + docker/hf_info | 2 + hf_to_nvfp4_safetensors.py | 2 +- man/man1/ctq.1 | 170 +++++++++++++++++ man/man1/gguf_info.1 | 95 ++++++++++ man/man1/gguf_info.py.1 | 1 + man/man1/gguf_to_nvfp4_gguf.1 | 216 ++++++++++++++++++++++ man/man1/gguf_to_nvfp4_gguf.py.1 | 1 + man/man1/heretic.1 | 142 ++++++++++++++ man/man1/hf_info.1 | 86 +++++++++ man/man1/hf_info.py.1 | 1 + man/man1/hf_to_nvfp4_gguf.1 | 235 ++++++++++++++++++++++++ man/man1/hf_to_nvfp4_gguf.py.1 | 1 + man/man1/hf_to_nvfp4_safetensors.1 | 169 +++++++++++++++++ man/man1/hf_to_nvfp4_safetensors.py.1 | 1 + man/man1/ollama_nvfp4.1 | 181 ++++++++++++++++++ man/man1/ollama_nvfp4.py.1 | 1 + man/man1/safetensors_to_nvfp4_gguf.1 | 219 ++++++++++++++++++++++ man/man1/safetensors_to_nvfp4_gguf.py.1 | 1 + 20 files changed, 1564 insertions(+), 10 deletions(-) create mode 100755 docker/gguf_info create mode 100755 docker/hf_info create mode 100644 man/man1/ctq.1 create mode 100644 man/man1/gguf_info.1 create mode 120000 man/man1/gguf_info.py.1 create mode 100644 man/man1/gguf_to_nvfp4_gguf.1 create mode 120000 man/man1/gguf_to_nvfp4_gguf.py.1 create mode 100644 man/man1/heretic.1 create mode 100644 man/man1/hf_info.1 create mode 120000 man/man1/hf_info.py.1 create mode 100644 man/man1/hf_to_nvfp4_gguf.1 create mode 120000 man/man1/hf_to_nvfp4_gguf.py.1 create mode 100644 man/man1/hf_to_nvfp4_safetensors.1 create mode 120000 man/man1/hf_to_nvfp4_safetensors.py.1 create mode 100644 man/man1/ollama_nvfp4.1 create mode 120000 man/man1/ollama_nvfp4.py.1 create mode 100644 man/man1/safetensors_to_nvfp4_gguf.1 create mode 120000 man/man1/safetensors_to_nvfp4_gguf.py.1 diff --git a/Dockerfile b/Dockerfile index b58db7e..67687f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,37 @@ -FROM runpod/pytorch:1.5.0-cu1320-torch2130-ubuntu2404 -WORKDIR /root +FROM nvidia/cuda:13.4.3-cudnn-devel-ubuntu26.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN rm -f /etc/dpkg/dpkg.cfg.d/excludes && \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + rm -f /etc/apt/apt.conf.d/01-nodoc && \ + rm -f /usr/bin/man && \ + dpkg-divert --quiet --remove /usr/bin/man +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update && apt-get dist-upgrade --autoremove -y && apt-get install -y \ + python3-pip \ + python3-dev \ + python3-venv \ + build-essential \ + man \ + man-db \ + wget \ + bzip2 \ + curl \ + git \ + cmake + +RUN python3 -m venv /root/venv +ENV PATH="/root/venv/bin:${PATH}" +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install torch torchvision torchaudio --upgrade --index-url \ + https://download.pytorch.org/whl/cu134/ + +WORKDIR /root RUN mkdir llama.cpp COPY --exclude=build/ llama.cpp ./llama.cpp/ WORKDIR /root/llama.cpp -ENV LD_LIBRARY_PATH=/usr/local/cuda-13.2/compat:${LD_LIBRARY_PATH} -ENV LIBRARY_PATH=/usr/local/cuda-13.2/compat:${LIBRARY_PATH} +ENV LD_LIBRARY_PATH=/usr/local/cuda-13.4/compat:${LD_LIBRARY_PATH} +ENV LIBRARY_PATH=/usr/local/cuda-13.4/compat:${LIBRARY_PATH} RUN cmake -B build -S . \ -DCMAKE_BUILD_TYPE=Release \ @@ -18,18 +44,22 @@ RUN cmake -B build -S . \ && rm -rf build WORKDIR /root -RUN git clone https://github.com/silveroxides/convert_to_quant && cd convert_to_quant && pip install -r requirements.txt && pip install . +RUN git clone https://github.com/silveroxides/convert_to_quant && cd convert_to_quant && pip3 install -r requirements.txt && pip3 install . RUN mkdir nvfp4_gguf_quant WORKDIR /root/nvfp4_gguf_quant COPY --exclude=__pycache__ nvfp4gguf ./nvfp4gguf -COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py requirements.txt safetensors_to_nvfp4_gguf.py . +COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py requirements.txt safetensors_to_nvfp4_gguf.py hf_info.py gguf_info.py . -RUN pip install -r requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install -r requirements.txt COPY --chmod=0644 docker/README.txt /workspace/README.txt -COPY --chmod=0755 docker/hf_to_nvfp4_gguf docker/hf_to_nvfp4_safetensors docker/gguf_to_nvfp4_gguf docker/safetensors_to_nvfp4_gguf /usr/local/bin/ +COPY --chmod=0755 docker/hf_info docker/gguf_info docker/hf_to_nvfp4_gguf docker/hf_to_nvfp4_safetensors docker/gguf_to_nvfp4_gguf docker/safetensors_to_nvfp4_gguf /usr/local/bin/ -RUN apt remove -y python3-cryptography && pip install heretic-llm +RUN mkdir -p /usr/local/share/man/man1 +COPY --chmod=0644 man/man1/* /usr/local/share/man/man1/ + +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm +RUN rm -f /opt/nvidia/entrypoint.d/*banner*.sh /opt/nvidia/entrypoint.d/*license*.sh 2>/dev/null WORKDIR /workspace diff --git a/docker/gguf_info b/docker/gguf_info new file mode 100755 index 0000000..8e0b64b --- /dev/null +++ b/docker/gguf_info @@ -0,0 +1,2 @@ +#!/bin/bash +python /root/nvfp4_gguf_quant/gguf_info.py "${@}" diff --git a/docker/hf_info b/docker/hf_info new file mode 100755 index 0000000..0f030ab --- /dev/null +++ b/docker/hf_info @@ -0,0 +1,2 @@ +#!/bin/bash +python /root/nvfp4_gguf_quant/hf_info.py "${@}" diff --git a/hf_to_nvfp4_safetensors.py b/hf_to_nvfp4_safetensors.py index 5042fb7..baa434a 100644 --- a/hf_to_nvfp4_safetensors.py +++ b/hf_to_nvfp4_safetensors.py @@ -309,7 +309,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--early-stop-loss", type=float, default=5e-9) parser.add_argument( "--early-stop-stall", type=int, default=DEFAULT_EARLY_STOP_STALL, - help="Stop after this many steps without progress. At the learning-rate floor, progress means a 1% loss drop (default: 2000)", + help="Stop after this many steps without progress. At the learning-rate floor, progress means a 1%% loss drop (default: 2000)", ) parser.add_argument("--calib-samples", type=int, default=3072) parser.add_argument("--seed", type=int, default=42) diff --git a/man/man1/ctq.1 b/man/man1/ctq.1 new file mode 100644 index 0000000..ba2918e --- /dev/null +++ b/man/man1/ctq.1 @@ -0,0 +1,170 @@ +.TH CTQ 1 "November 2026" "convert_to_quant" "User Commands" +.SH NAME +ctq \- convert safetensors model weights to scaled FP8, NVFP4, INT8, and MXFP8 formats +.SH SYNOPSIS +.B ctq +\fB\-i\fR \fIINPUT\fR +[\fB\-o\fR \fIOUTPUT\fR] +[\fB\-\-comfy_quant\fR] +[\fB\-\-simple\fR] +[\fB\-\-device\fR \fIDEVICE\fR] +[\fB\-\-low\-memory\fR] +[\fB\-\-nvfp4\fR|\fB\-\-int8\fR|\fB\-\-mxfp8\fR] +[\fIoptions...\fR] +.SH DESCRIPTION +.B ctq +(\fBconvert_to_quant\fR) quantizes neural network weights stored in \fI.safetensors\fR +format into scaled lower-precision representations, primarily targeting +\fBComfyUI\fR and hardware-accelerated inference runtimes. +.PP +By default, \fBctq\fR performs Scaled FP8 quantization using SVD-guided learned rounding +(Prodigy optimizer) and per-tensor scaling. It supports multiple quantization formats: +.IP \(bu 2 +\fBFP8\fR (Float8 E4M3 / E5M2) with tensor-wise, row-wise, or block-wise scaling. +.IP \(bu 2 +\fBNVFP4\fR (NVIDIA 4-bit floating point, E2M1 with dual-level block scaling). +.IP \(bu 2 +\fBINT8\fR block-wise quantization with optional group-wise Hadamard rotation (\fIConvRot\fR). +.IP \(bu 2 +\fBMXFP8\fR (Microscaling FP8) and Hybrid MXFP8 formats. +.PP +Quantization error can optionally be extracted into a companion Error Correction LoRA +adapter (\fB\-\-extract\-lora\fR) to restore high-precision fidelity. +.SH OPTIONS +.SS Standard Options +.TP +.BI \-i " " INPUT ", " \-\-input " " INPUT +Path to the input \fI.safetensors\fR weight file (required). +.TP +.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT +Path to the output \fI.safetensors\fR destination file. Auto-generated if omitted. +.TP +.BR \-\-comfy_quant ", " \-\-comfy\-quant +Write ComfyUI-compatible quantization tensors and metadata (\fI.comfy_quant\fR). +.TP +.B \-\-simple +Skip SVD learned rounding and apply fast, direct grid quantization. +.TP +.BR \-fpmm ", " \-\-full_precision_matrix_mult +Add \fIfull_precision_matrix_mult=True\fR to \fI.comfy_quant\fR metadata. +.TP +.B \-\-heur +Skip layers with poor quantization characteristics (aspect ratio, size). +.TP +.BI \-\-device " " DEVICE +Execution device (e.g. \fBcuda\fR, \fBcuda:0\fR, \fBcpu\fR). Overrides auto-detection. +.TP +.BR \-lm ", " \-\-low\-memory +Stream tensor loading to minimize RAM consumption (recommended for models exceeding 50% available memory). +.TP +.BI \-\-verbose " " {DEBUG,VERBOSE,NORMAL,MINIMAL} +Logging verbosity level (default: \fBNORMAL\fR). +.TP +.BI \-\-manual_seed " " SEED +Random seed for reproducibility. Set to \-1 for non-deterministic execution. +.TP +.B \-\-save\-quant\-metadata +Write quantization metadata directly into safetensors file header (\fI_quantization_metadata\fR). +.SS Quantization Formats & Modes +.TP +.B \-\-nvfp4 +Quantize to NVIDIA FP4 (E2M1) with dual-level block scaling (Blackwell architecture). +.TP +.B \-\-int8 +Quantize to INT8 block-wise format instead of FP8. +.TP +.B \-\-convrot +Enable group-wise Hadamard rotation (ConvRot) for INT8 row-wise quantization to mitigate activation outliers. +.TP +.BI \-\-convrot\-group\-size " " SIZE +Group size for ConvRot (must be a power of 4, e.g. 64, 256, 1024; default: 256). +.TP +.B \-\-mxfp8 +Quantize to MXFP8 (Microscaling FP8) format. +.TP +.BI \-\-scaling_mode " " {tensor,row,block} +FP8 scaling granularity: \fBtensor\fR (single scale), \fBrow\fR (per-row), or \fBblock\fR (2D tile). Default: \fBtensor\fR. +.TP +.BI \-\-block_size " " SIZE +Block size for block-wise scaling (default: 128). +.SS Learned Rounding & Optimization +.TP +.BI \-\-optimizer " " {prodigy,adamw,radam,original} +Optimization algorithm for learned rounding (default: \fBprodigy\fR). +.TP +.BI \-\-num_iter " " N +Optimization iterations per tensor (default: 4000). +.TP +.BI \-\-lr " " LR +Initial learning rate for AdamW/RAdam/Original (default: 1.0). +.TP +.BI \-\-lr_schedule " " {plateau,exponential,adaptive} +Learning rate decay schedule (default: \fBplateau\fR). +.TP +.B \-\-use_speed +Enable Prodigy's \fIuse_speed\fR acceleration parameter. +.TP +.BI \-\-top_p " " P +Cumulative energy proportion of principal components for SVD (default: 0.2). +.TP +.BI \-\-min_k " " K ", " \-\-max_k " " K +Minimum and maximum SVD rank bounds (default: 256 and 1280). +.TP +.B \-\-full_matrix +Compute full SVD instead of low-rank SVD. +.TP +.BI \-\-scale\-optimization " " {fixed,iterative,joint,dualround} +Block scale tuning strategy (default: \fBfixed\fR). +.SS Error Correction LoRA +.TP +.B \-\-extract\-lora +Extract quantization residual error into a separate LoRA adapter file. +.TP +.BI \-\-lora\-rank " " RANK +Rank for extracted LoRA layers (default: 32). +.TP +.BI \-\-lora\-target " " REGEX +Regex matching layers to target for LoRA extraction. +.TP +.BI \-\-lora\-output " " PATH +Output file path for the extracted LoRA adapter (\fI.safetensors\fR). +.SS Model Filters +.B ctq +includes pre-tuned layer exclusion filters for common architecture families: +.TP +.BR \-\-gemma4 ", " \-\-qwen_vlm ", " \-\-t5xxl ", " \-\-wan ", " \-\-hunyuan ", " \-\-ltxv2 ", " \-\-qwen ", " \-\-zimage +Keep architecture-critical layers (embeddings, layernorms, vision towers, cross-attention) in native precision. +.SH EXAMPLES +.TP +.B Standard FP8 quantization for ComfyUI: +.RS +.nf +ctq -i model.safetensors -o model-fp8.safetensors --comfy_quant +.fi +.RE +.TP +.B NVFP4 quantization using learned rounding: +.RS +.nf +ctq -i model.safetensors -o model-nvfp4.safetensors --nvfp4 --scale-optimization joint +.fi +.RE +.TP +.B Fast INT8 conversion with ConvRot outlier mitigation: +.RS +.nf +ctq -i model.safetensors --int8 --convrot --simple -o model-int8.safetensors +.fi +.RE +.TP +.B Quantize with Error Correction LoRA extraction: +.RS +.nf +ctq -i model.safetensors --nvfp4 --extract-lora --lora-rank 32 --lora-output model-ec-lora.safetensors +.fi +.RE +.SH SEE ALSO +.BR hf_to_nvfp4_safetensors (1), +.BR safetensors_to_nvfp4_gguf (1), +.BR heretic (1) + diff --git a/man/man1/gguf_info.1 b/man/man1/gguf_info.1 new file mode 100644 index 0000000..acca8b5 --- /dev/null +++ b/man/man1/gguf_info.1 @@ -0,0 +1,95 @@ +.TH GGUF_INFO 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +gguf_info \- print architecture, quantization, and tensor metadata from a GGUF header +.SH SYNOPSIS +.B gguf_info +[\fB\-\-metadata\fR] +[\fB\-\-tensors\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fIgguf\fR [\fIgguf\fR ...] +.SH DESCRIPTION +.B gguf_info +inspects one or more GGUF files (versions 2 and 3) by reading only their header +and tensor directories. The raw weight bytes are never loaded into memory, +making the inspection fast even for multi-gigabyte models. +.PP +By default, +.B gguf_info +displays a concise summary of the model, including: +.IP \(bu 2 +GGUF version, file size, and total tensor and key-value counts. +.IP \(bu 2 +Model architecture identifier (e.g., \fIllama\fR, \fIqwen2\fR, \fIgemma2\fR). +.IP \(bu 2 +General metadata: model name, author, license, context length, embedding length, +attention head counts, layer counts, feed-forward dimension, and vocabulary size. +.IP \(bu 2 +Quantization summary: file type classification (e.g., \fIMOSTLY_NVFP4\fR, +\fIMOSTLY_Q4_K_M\fR, \fIMOSTLY_BF16\fR) and an exact count and breakdown of tensor +quantization types found across the model. +.SH OPTIONS +.TP +.BR \-h ", " \-\-help +Show command-line usage help and exit. +.TP +.B \-\-metadata +Print every key-value metadata pair found in the GGUF header. Long string +values, large numeric arrays, and vocabulary/merges arrays are truncated or +summarized for readability. +.TP +.B \-\-tensors +Print every tensor present in the file with its full name, GGML quantization +type (e.g., \fINVFP4\fR, \fIBF16\fR, \fIF16\fR, \fIQ4_K\fR), shape dimensions, +and byte offset within the file. +.SH SUPPORTED QUANTIZATION TYPES +.B gguf_info +recognizes all standard GGML types including: +.PP +.BR F32 ", " F16 ", " BF16 ", " F64 ", " +.BR Q4_0 ", " Q4_1 ", " Q5_0 ", " Q5_1 ", " Q8_0 ", " Q8_1 ", " +.BR Q2_K ", " Q3_K ", " Q4_K ", " Q5_K ", " Q6_K ", " Q8_K ", " +.BR IQ1_S ", " IQ1_M ", " IQ2_XXS ", " IQ2_XS ", " IQ2_S ", " +.BR IQ3_XXS ", " IQ3_S ", " IQ4_XS ", " IQ4_NL ", " +.BR TQ1_0 ", " TQ2_0 ", " MXFP4 ", " NVFP4 ", " Q1_0 ", " Q2_0 . +.SH EXAMPLES +.TP +.B Inspect a single GGUF model: +.RS +.nf +gguf_info model-NVFP4.gguf +.fi +.RE +.TP +.B Dump all metadata keys and parameters: +.RS +.nf +gguf_info \-\-metadata model-NVFP4.gguf +.fi +.RE +.TP +.B List all tensors and their formats: +.RS +.nf +gguf_info \-\-tensors model-NVFP4.gguf +.fi +.RE +.TP +.B Inspect multiple models in batch: +.RS +.nf +gguf_info model-f16.gguf model-NVFP4.gguf +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +An error occurred (e.g. file not found, invalid magic bytes, or unsupported GGUF version). +.SH SEE ALSO +.BR hf_info (1), +.BR gguf_to_nvfp4_gguf (1), +.BR hf_to_nvfp4_gguf (1), +.BR ollama_nvfp4 (1) + diff --git a/man/man1/gguf_info.py.1 b/man/man1/gguf_info.py.1 new file mode 120000 index 0000000..1838e3c --- /dev/null +++ b/man/man1/gguf_info.py.1 @@ -0,0 +1 @@ +gguf_info.1 \ No newline at end of file diff --git a/man/man1/gguf_to_nvfp4_gguf.1 b/man/man1/gguf_to_nvfp4_gguf.1 new file mode 100644 index 0000000..353df44 --- /dev/null +++ b/man/man1/gguf_to_nvfp4_gguf.1 @@ -0,0 +1,216 @@ +.TH GGUF_TO_NVFP4_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +gguf_to_nvfp4_gguf \- quantize or repack an existing GGUF model to NVFP4 GGUF +.SH SYNOPSIS +.B gguf_to_nvfp4_gguf +[\fB\-o\fR \fIOUTPUT\fR] +[\fB\-\-modelfile\-only\fR] +[\fB\-\-overwrite\fR] +[\fB\-\-mmproj\fR \fIMMPROJ\fR] +[\fB\-\-context\-length\fR \fILEN\fR] +[\fB\-\-no\-quant\fR] +[\fB\-\-outtype\fR \fITYPE\fR] +[\fB\-\-llama\-cpp\fR \fIPATH\fR] +[\fB\-\-convert\-to\-quant\fR \fIPATH\fR] +[\fB\-\-model\-name\fR \fINAME\fR] +[\fB\-\-device\fR \fIDEVICE\fR] +[\fB\-\-simple\fR] +[\fB\-\-optimizer\fR \fIOPT\fR] +[\fB\-\-num\-iter\fR \fIN\fR] +[\fB\-\-lr\fR \fILR\fR] +[\fB\-\-lr\-patience\fR \fIN\fR] +[\fB\-\-lr\-min\fR \fIMIN\fR] +[\fB\-\-lr\-schedule\fR \fISCHED\fR] +[\fB\-\-scale\-optimization\fR \fIMODE\fR] +[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR] +[\fB\-\-full\-matrix\fR] +[\fB\-\-top\-p\fR \fIP\fR] +[\fB\-\-min\-k\fR \fIK\fR] +[\fB\-\-max\-k\fR \fIK\fR] +[\fB\-\-early\-stop\-loss\fR \fILOSS\fR] +[\fB\-\-early\-stop\-stall\fR \fISTALL\fR] +[\fB\-\-target\-loss\fR \fITARGET\fR] +[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR] +[\fB\-\-calib\-samples\fR \fISAMPLES\fR] +[\fB\-\-seed\fR \fISEED\fR] +[\fB\-\-no\-bias\-correction\fR] +[\fB\-\-quantize\-output\fR] +[\fB\-\-exclude\fR \fIREGEX\fR] +[\fB\-\-checkpoint\-dir\fR \fIDIR\fR] +[\fB\-\-fresh\fR] +[\fB\-\-keep\-checkpoints\fR] +[\fB\-\-eager\fR] +[\fB\-\-no\-temp\-file\fR] +[\fB\-\-verbose\fR] +[\fB\-\-self\-test\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fIgguf\fR +.SH DESCRIPTION +.B gguf_to_nvfp4_gguf +reads an existing GGUF model (such as an F16, BF16, or existing quantized checkpoint) +and converts it to NVIDIA FP4 (NVFP4) format, or repacks an existing NVFP4 GGUF file. +.PP +Vocabulary, tokenizer settings, architecture metadata, and tensor names are copied directly +from the source GGUF header. When quantizing unquantized (float) tensors, learned rounding +or simple grid rounding is applied using the same optimization flags as \fBhf_to_nvfp4_gguf\fR. +.PP +An accompanying Ollama \fBModelfile\fR is written alongside the output GGUF, configured with +the source model's parameters and templates. +.SH POSITIONAL ARGUMENTS +.TP +.I gguf +Input GGUF file path. +.SH OPTIONS +.SS Output & File Control +.TP +.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT +Output directory (default: \fI./-NVFP4\fR, or \fI./\fR with \fB\-\-no\-quant\fR and \fBauto\fR). +If ending in \fI.gguf\fR, specifies the destination file directly. +.TP +.B \-\-overwrite +Overwrite existing output GGUF, Modelfile, or copied projector without prompting. +.TP +.B \-\-modelfile\-only +Write the Ollama Modelfile and skip quantization and GGUF generation. +.TP +.BI \-\-mmproj " " MMPROJ +Path to a vision projector GGUF to bundle into the output directory and add as another \fBFROM\fR layer. +May be repeated for multi-modal models. +.TP +.BI \-\-context\-length " " LEN +Override maximum context length stored in the GGUF metadata. +.TP +.BI \-\-model\-name " " NAME +Override the model name stored in the GGUF metadata. +.SS Quantization Mode +.TP +.B \-\-no\-quant +Skip learned rounding. Writes a float GGUF, or copies/repacks an existing NVFP4 GGUF when combined with \fB\-\-outtype nvfp4\fR. +.TP +.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4} +Target data type when learned rounding is skipped. \fBnvfp4\fR copies an already NVFP4 GGUF without re-optimizing; +\fBauto\fR leaves tensors as stored in the source file (default: \fBauto\fR). +.TP +.B \-\-simple +Round each value directly onto the E2M1 grid without learned rounding (fast, non-iterative). +.TP +.B \-\-quantize\-output +Also quantize the output (\fIlm_head\fR) matrix. By default, output projection layers are left unquantized. +.TP +.BI \-\-exclude " " REGEX +Regular expression matching GGUF tensor names that should remain in their source precision. +.SS Optimization & Learned Rounding +.TP +.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original} +Learned-rounding optimization algorithm (default: \fBauto\fR; selects Prodigy if available, else AdamW). +.TP +.BI \-\-num\-iter " " N +Learned-rounding optimization iterations per matrix (default: 4000). +.TP +.BI \-\-lr " " LR +Initial learning rate (default: 1.0). +.TP +.BI \-\-lr\-patience " " N +Patience steps before decaying the learning rate (default: 5). +.TP +.BI \-\-lr\-min " " MIN +Lowest learning rate floor (default: 1e-8). +.TP +.BI \-\-lr\-schedule " " {plateau,exponential,adaptive} +Learning rate decay schedule (default: \fBplateau\fR). +.TP +.BI \-\-scale\-optimization " " {fixed,iterative,joint} +Block scale tuning strategy: \fBfixed\fR, \fBiterative\fR, or \fBjoint\fR (default: \fBjoint\fR). +.TP +.BI \-\-scale\-refinement\-rounds " " N +Scale refresh iterations when using \fBiterative\fR scale optimization (default: 2). +.TP +.B \-\-full\-matrix +Use full SVD instead of low-rank SVD for reconstruction objective. +.TP +.BI \-\-top\-p " " P ", " \-\-min\-k " " K ", " \-\-max\-k " " K +Singular value decomposition rank selection bounds. +.TP +.BI \-\-target\-loss " " TARGET +Target reconstruction loss threshold for early exit (default: 1e-5; 0 disables). +.TP +.BI \-\-early\-stop\-stall " " STALL +Early stop threshold in steps without loss reduction (default: 2000). +.TP +.BI \-\-early\-stop\-floor\-stall " " FLOOR +Steps without improvement after reaching \fB\-\-lr\-min\fR before terminating matrix search (default: 64; 0 disables). +.TP +.BI \-\-calib\-samples " " SAMPLES +Random calibration vectors used for sibling bias correction (default: 3072). +.TP +.BI \-\-seed " " SEED +Random seed (default: 42). +.TP +.B \-\-no\-bias\-correction +Disable sibling bias correction. +.SS System & Execution +.TP +.BI \-\-device " " DEVICE +Compute device: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR). +.TP +.BI \-\-llama\-cpp " " PATH +Path to local llama.cpp repository checkout (default: \fI~/llama.cpp\fR). +.TP +.BI \-\-convert\-to\-quant " " PATH +Path to local convert_to_quant repository checkout. +.TP +.BI \-\-checkpoint\-dir " " DIR +Directory for per-tensor resume checkpoints. +.TP +.B \-\-fresh +Delete checkpoint directory before starting. +.TP +.B \-\-keep\-checkpoints +Keep per-tensor checkpoints on disk after completion. +.TP +.B \-\-eager +Map and load tensors up front. +.TP +.B \-\-no\-temp\-file +Keep finished GGUF tensors in RAM rather than a temporary file. +.TP +.B \-\-verbose +Enable verbose logging. +.TP +.B \-\-self\-test +Verify NVFP4 packing against llama.cpp dequantizer and exit. +.SH EXAMPLES +.TP +.B Quantize an F16 GGUF to NVFP4: +.RS +.nf +gguf_to_nvfp4_gguf -o ./llama3-nvfp4 ./llama-3-8b-f16.gguf +.fi +.RE +.TP +.B Fast conversion with simple grid rounding: +.RS +.nf +gguf_to_nvfp4_gguf --simple -o ./qwen-nvfp4 ./qwen2.5-7b-bf16.gguf +.fi +.RE +.TP +.B Repack existing NVFP4 GGUF into a new directory with Modelfile: +.RS +.nf +gguf_to_nvfp4_gguf --no-quant --outtype nvfp4 -o ./repacked ./model-nvfp4.gguf +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +An error occurred during quantization or repacking. +.SH SEE ALSO +.BR hf_to_nvfp4_gguf (1), +.BR safetensors_to_nvfp4_gguf (1), +.BR gguf_info (1), +.BR ollama_nvfp4 (1) + diff --git a/man/man1/gguf_to_nvfp4_gguf.py.1 b/man/man1/gguf_to_nvfp4_gguf.py.1 new file mode 120000 index 0000000..3d1cd77 --- /dev/null +++ b/man/man1/gguf_to_nvfp4_gguf.py.1 @@ -0,0 +1 @@ +gguf_to_nvfp4_gguf.1 \ No newline at end of file diff --git a/man/man1/heretic.1 b/man/man1/heretic.1 new file mode 100644 index 0000000..8cbaebb --- /dev/null +++ b/man/man1/heretic.1 @@ -0,0 +1,142 @@ +.TH HERETIC 1 "November 2026" "heretic-llm" "User Commands" +.SH NAME +heretic \- automated directional ablation and refusal suppression for large language models +.SH SYNOPSIS +.B heretic +\fB\-\-model\fR \fIMODEL\fR +[\fB\-\-quantization\fR \fIMETHOD\fR] +[\fB\-\-n\-trials\fR \fIN\fR] +[\fB\-\-export\-strategy\fR \fISTRATEGY\fR] +[\fB\-\-save\-directory\fR \fIDIR\fR] +[\fB\-\-upload\-repo\-id\fR \fIREPO\fR] +[\fB\-\-reproduce\fR \fIFILE\fR] +[\fIoptions...\fR] +.SH DESCRIPTION +.B heretic +(\fBheretic-llm\fR) is an automated optimization and directional ablation tool for +modifying and un-aligning Large Language Models (LLMs). It isolates refusal vectors +and behavioral directions across internal residual streams and attention/MLP layers, +optimizing modifications via genetic multi-objective search (Optuna Pareto front analysis). +.PP +The tool balances refusal suppression ("abliteration" / "decensoring") against model capability +preservation by evaluating candidate modifications against built-in language benchmarks +(MMLU, GSM8K, IFEval, AGIEval, BBH, etc.). +.PP +Modified models can be saved to disk as merged safetensors checkpoints or adapter weights, +or uploaded directly to Hugging Face with complete reproducibility logs (\fIreproduce.json\fR). +.SH OPTIONS +.SS Model & Input +.TP +.BI \-\-model " " MODEL +Hugging Face model repository ID (e.g. \fIgoogle/gemma-2-9b-it\fR) or local filesystem path (required). +.TP +.BI \-\-model\-commit " " HASH +Specific Git commit hash of the Hugging Face model repository. +.TP +.BI \-\-quantization " " {NONE,BNB_4BIT} +Quantization mode used when loading model weights into VRAM: \fBNONE\fR (native precision) or \fBBNB_4BIT\fR (4-bit bitsandbytes). Default: \fBNONE\fR. +.TP +.BI \-\-dtypes " " LIST +List of PyTorch data types to attempt when loading tensors (default: \fI['auto', 'float16', 'bfloat16', 'float32']\fR). +.TP +.BI \-\-system\-prompt " " STR +System prompt passed to the model during evaluation (default: \fI"You are a helpful assistant."\fR). +.SS Execution & Memory +.TP +.BI \-\-device\-map " " MAP +Device allocation strategy passed to Hugging Face Accelerate (default: \fBauto\fR). +.TP +.BI \-\-max\-memory " " DICT +Maximum VRAM/RAM limits per device (e.g. \fI'{"0": "22GB", "cpu": "64GB"}'\fR). +.TP +.BR \-\-offload\-outputs\-to\-cpu ", " \-\-no\-offload\-outputs\-to\-cpu +Offload intermediate residual tensors and logits to CPU RAM immediately to reduce peak VRAM usage (default: enabled). +.TP +.BI \-\-batch\-size " " N +Batch size for evaluation sequences. Set to 0 for automatic detection (default: 0). +.TP +.BI \-\-max\-batch\-size " " N +Upper bound during automatic batch size discovery (default: 128). +.TP +.BI \-\-max\-response\-length " " N +Maximum token generation length during prompt scoring. +.SS Optimization & Genetic Search +.TP +.BI \-\-n\-trials " " N +Total number of optimization trials to explore across the Pareto front. +.TP +.BI \-\-n\-startup\-trials " " N +Number of initial random trials before heuristic exploration begins. +.TP +.BI \-\-seed " " SEED +Random seed for reproducibility. +.TP +.BI \-\-study\-checkpoint\-dir " " DIR +Directory where Optuna study progression and intermediate trials are saved. +.TP +.BI \-\-scorers " " JSON +Custom or overriding scoring configurations (loss metrics, refusal detectors, perplexity monitors). +.TP +.BI \-\-modifiers " " JSON +Custom modification operators (directional projection, orthogonalization, rank cuts). +.SS Benchmarks & Evaluation +.TP +.BI \-\-benchmarks " " LIST +List of evaluation benchmarks to verify model integrity. Supported benchmarks include: +\fIagieval\fR, \fIbbh\fR, \fIcommonsense_qa\fR, \fIeq_bench\fR, \fIgsm8k\fR, \fIhellaswag\fR, \fIifeval\fR, \fImmlu\fR, \fImmlu_pro\fR, \fIpiqa\fR, \fIwinogrande\fR. +.TP +.BI \-\-evaluate\-model " " MODEL +Evaluate an existing model relative to the base model instead of running search trials. +.SS Export & Publishing +.TP +.BI \-\-export\-strategy " " {MERGE,ADAPTER} +Model export strategy: \fBMERGE\fR (bake modifications directly into output safetensors weights) +or \fBADAPTER\fR (export as differential weight delta). +.TP +.BI \-\-save\-directory " " DIR +Local directory path where modified safetensors weights and tokenizer files are written. +.TP +.BI \-\-max\-shard\-size " " SIZE +Maximum file size for exported safetensors shards (default: \fI5GB\fR). +.TP +.BI \-\-upload\-repo\-id " " REPO +Hugging Face repository ID to upload the final modified model to. +.TP +.B \-\-upload\-repo\-private +Make the destination Hugging Face repository private. +.TP +.BI \-\-upload\-tags " " LIST +Tags added to the Hugging Face model card (default: \fI['uncensored', 'decensored', 'abliterated']\fR). +.SS Reproduction +.TP +.BI \-\-reproduce " " PATH_OR_URL +Load parameters from an existing \fIreproduce.json\fR file or URL and reproduce the exact model modification. +.TP +.BI \-\-collect\-reproducibles " " DIR +Download all \fIreproduce.json\fR files from public Heretic model repositories on Hugging Face into \fIDIR\fR for archival. +.SH EXAMPLES +.TP +.B Run automated abliteration search on a local model: +.RS +.nf +heretic --model ./meta-llama-3-8b-instruct --n-trials 50 --save-directory ./llama3-abliterated +.fi +.RE +.TP +.B Optimize with 4-bit base weights to reduce VRAM: +.RS +.nf +heretic --model mistralai/Mistral-7B-Instruct-v0.3 --quantization BNB_4BIT --save-directory ./mistral-heretic +.fi +.RE +.TP +.B Reproduce an existing published Heretic model: +.RS +.nf +heretic --model google/gemma-2-9b-it --reproduce https://huggingface.co/owner/repo/raw/main/reproduce.json --save-directory ./gemma2-reproduced +.fi +.RE +.SH SEE ALSO +.BR ctq (1), +.BR hf_to_nvfp4_gguf (1) + diff --git a/man/man1/hf_info.1 b/man/man1/hf_info.1 new file mode 100644 index 0000000..a4d6f36 --- /dev/null +++ b/man/man1/hf_info.1 @@ -0,0 +1,86 @@ +.TH HF_INFO 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +hf_info \- print architecture, quantization, and tensor layout from Hugging Face checkpoints +.SH SYNOPSIS +.B hf_info +[\fB\-\-metadata\fR] +[\fB\-\-tensors\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fImodel\fR [\fImodel\fR ...] +.SH DESCRIPTION +.B hf_info +inspects one or more Hugging Face model directories or standalone \fI.safetensors\fR files. +It reads configuration files (\fIconfig.json\fR, \fIgeneration_config.json\fR, +\fItokenizer_config.json\fR) and safetensors file headers without loading weight bytes +into RAM, allowing rapid inspection of multi-gigabyte or sharded checkpoints. +.PP +By default, +.B hf_info +displays a comprehensive summary, including: +.IP \(bu 2 +Model architecture class, text/vision config hierarchy, and model type. +.IP \(bu 2 +Model dimensions: context length, hidden size, intermediate size, layer counts, attention heads, and key-value heads. +.IP \(bu 2 +Parameter counts: total parameters, active parameters, and size on disk. +.IP \(bu 2 +Chat template configuration and capability detection (such as tool/function calling support). +.IP \(bu 2 +Quantization format detection: Hugging Face quantization configs (bitsandbytes, AWQ, GPTQ, FP8), +embedded NVFP4 scales, and ComfyUI NVFP4 metadata (\fI_quantization_metadata\fR, \fIweight_scale\fR, \fI.comfy_quant\fR). +.IP \(bu 2 +Sharding structure: list of shard files, header sizes, and index verification against \fImodel.safetensors.index.json\fR. +.SH OPTIONS +.TP +.BR \-h ", " \-\-help +Show command-line usage help and exit. +.TP +.B \-\-metadata +Print complete config, generation configuration, tokenizer configuration, and safetensors header metadata. +Long strings and large arrays are shortened for readability. +.TP +.B \-\-tensors +Print every tensor name, data type (e.g. \fIBF16\fR, \fIF16\fR, \fIU8\fR), shape, byte size, +and the shard file in which it resides. +.SH EXAMPLES +.TP +.B Inspect a Hugging Face model directory: +.RS +.nf +hf_info ./gemma-4-12B-it +.fi +.RE +.TP +.B Inspect a standalone safetensors file: +.RS +.nf +hf_info ./model-00001-of-00004.safetensors +.fi +.RE +.TP +.B Dump full config and chat template metadata: +.RS +.nf +hf_info \-\-metadata ./meta-llama-3-8b-instruct +.fi +.RE +.TP +.B List all tensors across all checkpoint shards: +.RS +.nf +hf_info \-\-tensors ./mistral-7b-v0.3 +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +An error occurred (e.g. model directory not found or missing configuration). +.SH SEE ALSO +.BR gguf_info (1), +.BR hf_to_nvfp4_gguf (1), +.BR hf_to_nvfp4_safetensors (1), +.BR safetensors_to_nvfp4_gguf (1) + diff --git a/man/man1/hf_info.py.1 b/man/man1/hf_info.py.1 new file mode 120000 index 0000000..2c2a92f --- /dev/null +++ b/man/man1/hf_info.py.1 @@ -0,0 +1 @@ +hf_info.1 \ No newline at end of file diff --git a/man/man1/hf_to_nvfp4_gguf.1 b/man/man1/hf_to_nvfp4_gguf.1 new file mode 100644 index 0000000..bec7068 --- /dev/null +++ b/man/man1/hf_to_nvfp4_gguf.1 @@ -0,0 +1,235 @@ +.TH HF_TO_NVFP4_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +hf_to_nvfp4_gguf \- quantize Hugging Face models to NVFP4 GGUF for llama.cpp and Ollama +.SH SYNOPSIS +.B hf_to_nvfp4_gguf +[\fB\-o\fR \fIOUTPUT\fR] +[\fB\-\-modelfile\-only\fR] +[\fB\-\-overwrite\fR] +[\fB\-\-mmproj\fR \fIMMPROJ\fR] +[\fB\-\-context\-length\fR \fILEN\fR] +[\fB\-\-no\-quant\fR] +[\fB\-\-outtype\fR \fITYPE\fR] +[\fB\-\-llama\-cpp\fR \fIPATH\fR] +[\fB\-\-convert\-to\-quant\fR \fIPATH\fR] +[\fB\-\-model\-name\fR \fINAME\fR] +[\fB\-\-device\fR \fIDEVICE\fR] +[\fB\-\-simple\fR] +[\fB\-\-optimizer\fR \fIOPT\fR] +[\fB\-\-num\-iter\fR \fIN\fR] +[\fB\-\-lr\fR \fILR\fR] +[\fB\-\-lr\-patience\fR \fIN\fR] +[\fB\-\-lr\-min\fR \fIMIN\fR] +[\fB\-\-lr\-schedule\fR \fISCHED\fR] +[\fB\-\-scale\-optimization\fR \fIMODE\fR] +[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR] +[\fB\-\-full\-matrix\fR] +[\fB\-\-top\-p\fR \fIP\fR] +[\fB\-\-min\-k\fR \fIK\fR] +[\fB\-\-max\-k\fR \fIK\fR] +[\fB\-\-early\-stop\-loss\fR \fILOSS\fR] +[\fB\-\-early\-stop\-stall\fR \fISTALL\fR] +[\fB\-\-target\-loss\fR \fITARGET\fR] +[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR] +[\fB\-\-calib\-samples\fR \fISAMPLES\fR] +[\fB\-\-seed\fR \fISEED\fR] +[\fB\-\-no\-bias\-correction\fR] +[\fB\-\-quantize\-output\fR] +[\fB\-\-exclude\fR \fIREGEX\fR] +[\fB\-\-checkpoint\-dir\fR \fIDIR\fR] +[\fB\-\-fresh\fR] +[\fB\-\-keep\-checkpoints\fR] +[\fB\-\-eager\fR] +[\fB\-\-no\-temp\-file\fR] +[\fB\-\-verbose\fR] +[\fB\-\-self\-test\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fImodel\fR +.SH DESCRIPTION +.B hf_to_nvfp4_gguf +quantizes a Hugging Face model directory (\fIconfig.json\fR + safetensors) to +NVIDIA FP4 (NVFP4) format packaged into a GGUF file suitable for \fBllama.cpp\fR +and \fBOllama\fR. +.PP +The model is loaded through \fBllama.cpp\fR's architecture conversion logic so that +all vocabulary, tensor naming conventions, tokenization metadata, and architectural +parameters match native GGUF standards. Weight matrices are quantized using learned +rounding (via \fBconvert_to_quant\fR) or simple nearest-grid projection onto the E2M1 +floating point grid with two-level scaling (per-block E4M3/E8M0 scales and per-tensor scales). +.PP +Along with the output GGUF, an Ollama \fBModelfile\fR is generated automatically, +preserving the model's chat template, stop sequences, parameter defaults, and vision projectors. +.SH POSITIONAL ARGUMENTS +.TP +.I model +Path to a Hugging Face model directory containing \fIconfig.json\fR and one or more \fI.safetensors\fR files. +.SH OPTIONS +.SS Output & File Control +.TP +.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT +Output destination directory (default: \fI./-NVFP4\fR, or \fI./\fR with +\fB\-\-no\-quant\fR and \fBauto\fR). If the path ends in \fI.gguf\fR, it specifies the destination +GGUF file itself. +.TP +.B \-\-overwrite +Overwrite an existing GGUF, Modelfile, or copied projector without prompting. +.TP +.B \-\-modelfile\-only +Generate only the Ollama \fBModelfile\fR and exit without quantizing or writing GGUF tensors. +.TP +.BI \-\-mmproj " " MMPROJ +Path to a vision projector GGUF. The file is copied into the output directory and added as an +additional \fBFROM\fR layer in the Modelfile. May be specified multiple times for multimodal models. +.TP +.BI \-\-context\-length " " LEN +Override the maximum context length recorded in the GGUF metadata. Replaces checkpoint value +or defines one if missing. +.TP +.BI \-\-model\-name " " NAME +Override the model name recorded in the GGUF metadata. +.SS Quantization Mode +.TP +.B \-\-no\-quant +Skip learned rounding. Writes a float GGUF, or repacks an already NVFP4 checkpoint. +.TP +.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4} +Data type for non-quantized weights when learned rounding is skipped. \fBnvfp4\fR repacks a +checkpoint that already has packed 4-bit weights. \fBauto\fR preserves the source float types (default: \fBauto\fR). +.TP +.B \-\-simple +Round weights directly onto the E2M1 grid without iterative learned rounding (fast, non-iterative). +.TP +.B \-\-quantize\-output +Also quantize the final output projection (\fIlm_head\fR) matrix. By default, output layers are +kept in BF16/float for generation quality. +.TP +.BI \-\-exclude " " REGEX +Regular expression matching GGUF tensor names that should remain unquantized in BF16. +.SS Optimization & Learned Rounding +.TP +.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original} +Learned-rounding optimization algorithm. \fBauto\fR selects Prodigy if installed, otherwise AdamW (default: \fBauto\fR). +.TP +.BI \-\-num\-iter " " N +Learned-rounding optimization iterations per matrix (default: 4000). +.TP +.BI \-\-lr " " LR +Initial learning rate (default: 1.0). +.TP +.BI \-\-lr\-patience " " N +Number of steps without improvement before decaying the learning rate (default: 5). +.TP +.BI \-\-lr\-min " " MIN +Minimum learning rate floor. The flat-loss termination check begins once this floor is reached (default: 1e-8). +.TP +.BI \-\-lr\-schedule " " {plateau,exponential,adaptive} +Learning rate decay schedule (default: \fBplateau\fR). +.TP +.BI \-\-scale\-optimization " " {fixed,iterative,joint} +Block scale tuning strategy. \fBfixed\fR keeps initial scales; \fBiterative\fR refreshes scales periodically +using the original optimizer; \fBjoint\fR trains block scales concurrently with the codes (default: \fBjoint\fR). +.TP +.BI \-\-scale\-refinement\-rounds " " N +Number of scale refreshes when \fB\-\-scale\-optimization\fR is set to \fBiterative\fR (default: 2). +.TP +.B \-\-full\-matrix +Use full singular value decomposition (SVD) rather than a low-rank approximation for the loss objective. +.TP +.BI \-\-top\-p " " P +Cumulative energy cutoff for low-rank SVD (default: 0.2). +.TP +.BI \-\-min\-k " " K ", " \-\-max\-k " " K +Minimum and maximum rank limits for low-rank SVD (default: 128 and 1280). +.TP +.BI \-\-target\-loss " " TARGET +Stop optimization when best reconstruction loss reaches this threshold (default: 1e-5; 0 disables). +.TP +.BI \-\-early\-stop\-stall " " STALL +Stop optimization after this many steps without reaching a new best loss (default: 2000). +.TP +.BI \-\-early\-stop\-floor\-stall " " FLOOR +Stop optimization after this many steps without improvement once the learning rate has reached \fB\-\-lr\-min\fR (default: 64; 0 disables). +.TP +.BI \-\-calib\-samples " " SAMPLES +Number of random Gaussian calibration inputs used to calculate sibling bias correction (default: 3072). +.TP +.BI \-\-seed " " SEED +Random seed for calibration vectors (default: 42). +.TP +.B \-\-no\-bias\-correction +Disable sibling bias correction on downstream biases. +.SS System & Execution +.TP +.BI \-\-device " " DEVICE +Device to perform optimization on: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR). +.TP +.BI \-\-llama\-cpp " " PATH +Path to local \fBllama.cpp\fR repository checkout (default: \fI~/llama.cpp\fR). +.TP +.BI \-\-convert\-to\-quant " " PATH +Path to local \fBconvert_to_quant\fR repository checkout, if not installed as a Python package. +.TP +.BI \-\-checkpoint\-dir " " DIR +Directory used to store and resume per-tensor quantization checkpoints. +.TP +.B \-\-fresh +Remove existing checkpoint directory before starting. +.TP +.B \-\-keep\-checkpoints +Preserve per-tensor checkpoints after successful conversion. +.TP +.B \-\-eager +Load all model tensors into memory up front rather than streaming (requires more system RAM). +.TP +.B \-\-no\-temp\-file +Keep finished GGUF tensors in memory rather than buffering through a temporary scratch file. +.TP +.B \-\-verbose +Enable detailed verbose logging. +.TP +.B \-\-self\-test +Run internal verification of NVFP4 packing against llama.cpp's reference dequantizer and exit immediately. +.SH EXAMPLES +.TP +.B Full learned-rounding conversion to GGUF: +.RS +.nf +hf_to_nvfp4_gguf -o ./llama3-8b-NVFP4 ./meta-llama-3-8b +.fi +.RE +.TP +.B Fast quantization with simple grid rounding: +.RS +.nf +hf_to_nvfp4_gguf --simple -o ./qwen-nvfp4 ./Qwen2.5-7B-Instruct +.fi +.RE +.TP +.B Multi-modal model with vision projector: +.RS +.nf +hf_to_nvfp4_gguf --mmproj mmproj-model-f16.gguf -o ./vision-nvfp4 ./Qwen2-VL-7B +.fi +.RE +.TP +.B Specify CUDA GPU and custom llama.cpp directory: +.RS +.nf +hf_to_nvfp4_gguf --llama-cpp /opt/llama.cpp --device cuda:0 -o ./output ./model +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +An error occurred during conversion or validation. +.SH SEE ALSO +.BR gguf_to_nvfp4_gguf (1), +.BR safetensors_to_nvfp4_gguf (1), +.BR hf_to_nvfp4_safetensors (1), +.BR hf_info (1), +.BR gguf_info (1), +.BR ollama_nvfp4 (1) + diff --git a/man/man1/hf_to_nvfp4_gguf.py.1 b/man/man1/hf_to_nvfp4_gguf.py.1 new file mode 120000 index 0000000..4181c80 --- /dev/null +++ b/man/man1/hf_to_nvfp4_gguf.py.1 @@ -0,0 +1 @@ +hf_to_nvfp4_gguf.1 \ No newline at end of file diff --git a/man/man1/hf_to_nvfp4_safetensors.1 b/man/man1/hf_to_nvfp4_safetensors.1 new file mode 100644 index 0000000..616b431 --- /dev/null +++ b/man/man1/hf_to_nvfp4_safetensors.1 @@ -0,0 +1,169 @@ +.TH HF_TO_NVFP4_SAFETENSORS 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +hf_to_nvfp4_safetensors \- quantize a Hugging Face directory to a ComfyUI NVFP4 safetensors file +.SH SYNOPSIS +.B hf_to_nvfp4_safetensors +[\fB\-o\fR \fIOUTPUT\fR] +[\fB\-\-overwrite\fR] +[\fB\-\-output\-dtype\fR \fIDTYPE\fR] +[\fB\-\-low\-memory\fR] +[\fB\-\-heur\fR] +[\fB\-\-exclude\-layers\fR \fIREGEX\fR] +[\fB\-\-convert\-to\-quant\fR \fIPATH\fR] +[\fB\-\-simple\fR] +[\fB\-\-optimizer\fR \fIOPT\fR] +[\fB\-\-num\-iter\fR \fIN\fR] +[\fB\-\-lr\fR \fILR\fR] +[\fB\-\-lr\-min\fR \fIMIN\fR] +[\fB\-\-lr\-patience\fR \fIN\fR] +[\fB\-\-lr\-schedule\fR \fISCHED\fR] +[\fB\-\-scale\-optimization\fR \fIMODE\fR] +[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR] +[\fB\-\-full\-matrix\fR] +[\fB\-\-top\-p\fR \fIP\fR] +[\fB\-\-min\-k\fR \fIK\fR] +[\fB\-\-max\-k\fR \fIK\fR] +[\fB\-\-early\-stop\-loss\fR \fILOSS\fR] +[\fB\-\-early\-stop\-stall\fR \fISTALL\fR] +[\fB\-\-calib\-samples\fR \fISAMPLES\fR] +[\fB\-\-seed\fR \fISEED\fR] +[\fB\-\-use\-speed\fR] +[\fB\-\-verbose\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fImodel\fR +.SH DESCRIPTION +.B hf_to_nvfp4_safetensors +quantizes a Hugging Face model directory (\fIconfig.json\fR and sharded \fI.safetensors\fR files) +into a single consolidated \fI.safetensors\fR file formatted for ComfyUI's native NVFP4 loader. +.PP +For each quantized weight matrix, the script outputs packed NVFP4 bytes alongside +companion scaling tensors: +.IP \(bu 2 +\fI.weight\fR (packed NVFP4 data, two 4-bit values per byte) +.IP \(bu 2 +\fI.weight_scale\fR (per-block E4M3 scale factors) +.IP \(bu 2 +\fI.weight_scale_2\fR (per-tensor scale factor) +.IP \(bu 2 +\fI.comfy_quant\fR (metadata tensor specifying quantization type) +.PP +A unified \fI_quantization_metadata\fR header is embedded in the output safetensors header. +When the source model is split across multiple shards, each shard is quantized in a temporary +workspace and then efficiently merged into the final single output file. +.PP +Normalization layers, embeddings, biases, and layers matching \fB\-\-exclude\-layers\fR +remain in the target float format (controlled by \fB\-\-output\-dtype\fR). +.SH POSITIONAL ARGUMENTS +.TP +.I model +Hugging Face model directory containing \fIconfig.json\fR and \fI.safetensors\fR weights. +.SH OPTIONS +.SS Output & Memory Control +.TP +.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT +Output \fI.safetensors\fR file path (default: \fI./-NVFP4.safetensors\fR). +.TP +.B \-\-overwrite +Overwrite an existing output file without prompting. +.TP +.BI \-\-output\-dtype " " {bfloat16,float16} +Data type recorded for quantized layer scales and used for unquantized 2D weights (default: \fBbfloat16\fR). +.TP +.B \-\-low\-memory +Stream each shard sequentially through quantization rather than loading the entire model shard into RAM at once. +.TP +.B \-\-heur +Skip layers that convert_to_quant's built-in size heuristic rejects. +.TP +.BI \-\-exclude\-layers " " REGEX +Regular expression matching layer or tensor names to leave unquantized in float format. +.SS Quantization & Optimization +.TP +.B \-\-simple +Round onto the E2M1 grid directly without iterative learned rounding. +.TP +.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original} +Learned-rounding optimizer. \fBauto\fR uses Prodigy when installed, otherwise AdamW (default: \fBauto\fR). +.TP +.BI \-\-num\-iter " " N +Learned-rounding iterations per matrix (default: 4000). +.TP +.BI \-\-lr " " LR +Initial learning rate (default: 1.0). +.TP +.BI \-\-lr\-min " " MIN +Lowest learning rate floor (default: 1e-8). +.TP +.BI \-\-lr\-patience " " N +Patience steps before decaying learning rate (default: 5). +.TP +.BI \-\-lr\-schedule " " {plateau,exponential,adaptive} +Learning rate decay schedule (default: \fBplateau\fR). +.TP +.BI \-\-scale\-optimization " " {fixed,iterative,joint} +Scale tuning strategy: \fBfixed\fR, \fBiterative\fR, or \fBjoint\fR (default: \fBjoint\fR). +.TP +.BI \-\-scale\-refinement\-rounds " " N +Scale refresh iterations when using iterative scale optimization (default: 2). +.TP +.B \-\-full\-matrix +Use full SVD instead of low-rank SVD. +.TP +.BI \-\-top\-p " " P ", " \-\-min\-k " " K ", " \-\-max\-k " " K +Singular value decomposition rank selection bounds. +.TP +.BI \-\-early\-stop\-loss " " LOSS +Early stopping loss target (default: 5e-9). +.TP +.BI \-\-early\-stop\-stall " " STALL +Stop after this many steps without loss improvement (default: 2000). +.TP +.BI \-\-calib\-samples " " SAMPLES +Random Gaussian calibration inputs for bias correction (default: 3072). +.TP +.BI \-\-seed " " SEED +Random seed (default: 42). +.TP +.B \-\-use\-speed +Enable Prodigy optimizer's \fIuse_speed\fR mode. +.SS System & Execution +.TP +.BI \-\-convert\-to\-quant " " PATH +Path to local convert_to_quant repository checkout. +.TP +.B \-\-verbose +Enable verbose output during shard quantization and merge. +.SH EXAMPLES +.TP +.B Quantize a model directory into a ComfyUI NVFP4 safetensors file: +.RS +.nf +hf_to_nvfp4_safetensors -o gemma-4-12B-it-NVFP4.safetensors ./gemma-4-12B-it +.fi +.RE +.TP +.B Low-memory streaming mode with float16 weights: +.RS +.nf +hf_to_nvfp4_safetensors --low-memory --output-dtype float16 -o ./model-nvfp4.safetensors ./model +.fi +.RE +.TP +.B Exclude specific projection layers: +.RS +.nf +hf_to_nvfp4_safetensors --exclude-layers ".*gate_proj.*" -o ./model-nvfp4.safetensors ./model +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +An error occurred during quantization or shard merging. +.SH SEE ALSO +.BR safetensors_to_nvfp4_gguf (1), +.BR hf_to_nvfp4_gguf (1), +.BR hf_info (1) + diff --git a/man/man1/hf_to_nvfp4_safetensors.py.1 b/man/man1/hf_to_nvfp4_safetensors.py.1 new file mode 120000 index 0000000..fd587f7 --- /dev/null +++ b/man/man1/hf_to_nvfp4_safetensors.py.1 @@ -0,0 +1 @@ +hf_to_nvfp4_safetensors.1 \ No newline at end of file diff --git a/man/man1/ollama_nvfp4.1 b/man/man1/ollama_nvfp4.1 new file mode 100644 index 0000000..7e5ec64 --- /dev/null +++ b/man/man1/ollama_nvfp4.1 @@ -0,0 +1,181 @@ +.TH OLLAMA_NVFP4 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +ollama_nvfp4 \- inspect Ollama sha256 model stores and quantize models to NVFP4 GGUF +.SH SYNOPSIS +.B ollama_nvfp4 +[\fB\-\-models\fR \fIDIR\fR] +.I subcommand +[\fIoptions...\fR] +.br +.B ollama_nvfp4 +[\fB\-\-models\fR \fIDIR\fR] +.B \-\-info +\fImodel\fR +[\fIgguf_info_args...\fR] +.SH DESCRIPTION +.B ollama_nvfp4 +is a management and quantization utility for local \fBOllama\fR model installations. +Ollama stores model artifacts as content-addressed blobs (\fIblobs/sha256-\fR) +referenced by JSON manifests (\fImanifests/\fR). +.PP +.B ollama_nvfp4 +resolves these hashes into human-readable model names, displays model capabilities and +parameters, exports native GGUF weights alongside reconstructed \fBModelfile\fRs, +and directly quantizes models to NVIDIA FP4 (NVFP4) format. Quantized models can +optionally be registered straight back into Ollama's local library. +.PP +The generated \fBModelfile\fR preserves the original prompt template, stop sequences, +system message, renderer, and parser settings so that thinking/reasoning modes and +generation behavior remain intact. +.SH GLOBAL OPTIONS +.TP +.BI \-\-models " " DIR +Path to the Ollama models store directory containing \fIblobs/\fR and \fImanifests/\fR. +If not provided, the location is resolved via the \fBOLLAMA_MODELS\fR environment variable, +\fI/models/ollama\fR, or \fI~/.ollama/models\fR. +.TP +.BR \-h ", " \-\-help +Show command help and exit. +.SH SUBCOMMANDS +.SS list +.B ollama_nvfp4 +[\fB\-\-models\fR \fIDIR\fR] +.B list +[\fB\-\-full\-scan\fR] +[\fB\-\-ctx\-scan\fR] +.PP +Display every manifest in the store as a readable model entry with family, quantization type, +and capability badges (chat, tool call, thinking, vision). +.TP +.B \-\-full\-scan +Read the underlying GGUF header of each model blob. This is slower on large stores but reveals +capabilities and vision encoders not declared in manifest metadata. +.TP +.B \-\-ctx\-scan +Scan parameter blobs and report models whose Modelfile explicitly configures \fInum_ctx\fR. +.SS show +.B ollama_nvfp4 +[\fB\-\-models\fR \fIDIR\fR] +.B show +\fImodel\fR +.PP +Display detailed information for a specified model, including manifest layers, sha256 blob +hashes, component media types, GGUF architecture metadata, context length, and system settings. +The \fImodel\fR argument can be a full name (e.g. \fIllama3.2:latest\fR) or a unique substring. +.SS export +.B ollama_nvfp4 +[\fB\-\-models\fR \fIDIR\fR] +.B export +[\fB\-o\fR \fIDIR\fR] +[\fB\-\-modelfile\-only\fR] +[\fB\-\-overwrite\fR|\fB\-\-force\fR] +\fImodel\fR +.PP +Export the model's underlying GGUF weight blob and generate a fully configured \fBModelfile\fR +into the destination directory (default: \fI./\fR). +.TP +.BI \-o " " DIR ", " \-\-output " " DIR +Output directory path. +.TP +.B \-\-modelfile\-only +Write only the Modelfile without copying the GGUF or vision projector blobs. +.TP +.BR \-\-overwrite ", " \-\-force +Replace existing files in the output directory. +.SS quant +.B ollama_nvfp4 +[\fB\-\-models\fR \fIDIR\fR] +.B quant +[\fB\-o\fR \fIDIR\fR] +[\fB\-\-name\fR \fINAME\fR] +[\fB\-\-install\fR] +[\fB\-\-float\-only\fR] +[\fB\-\-context\-length\fR \fILEN\fR] +[\fIquant_opts...\fR] +\fImodel\fR +.PP +Quantize an Ollama model to NVFP4 GGUF format and generate an accompanying Modelfile. +.TP +.BI \-o " " DIR ", " \-\-output " " DIR +Output directory (default: \fI./-NVFP4\fR). +.TP +.BI \-\-name " " NAME +Model tag used when registering with \fB\-\-install\fR (default: \fI:nvfp4\fR). +.TP +.B \-\-install +Automatically register the quantized model back into Ollama using \fIollama create\fR. +.TP +.B \-\-float\-only +Refuse to requantize a model if its source blob is already quantized (e.g. \fIQ4_K_M\fR). +Quantizing from float16/BF16 produces superior NVFP4 fidelity. +.TP +.BI \-\-context\-length " " LEN +Override maximum context length stored in the GGUF. +.TP +.I quant_opts +All standard learned-rounding and execution options supported by \fBgguf_to_nvfp4_gguf\fR +are supported: \fB\-\-simple\fR, \fB\-\-optimizer\fR, \fB\-\-lr\fR, \fB\-\-num\-iter\fR, +\fB\-\-scale\-optimization\fR, \fB\-\-device\fR, \fB\-\-checkpoint\-dir\fR, etc. +.SS \-\-info +.B ollama_nvfp4 +[\fB\-\-models\fR \fIDIR\fR] +.B \-\-info +\fImodel\fR +[\fIgguf_info_args...\fR] +.PP +Resolve \fImodel\fR to its sha256 GGUF weight blob in the store and invoke +.BR gguf_info (1) +directly on the file. Any subsequent flags (such as \fB\-\-metadata\fR or \fB\-\-tensors\fR) +are forwarded directly to \fBgguf_info\fR. +.SH ENVIRONMENT VARIABLES +.TP +.B OLLAMA_MODELS +Path to the default Ollama models directory. Overridden by \fB\-\-models\fR. +.SH EXAMPLES +.TP +.B List installed models with capability summary: +.RS +.nf +ollama_nvfp4 list +.fi +.RE +.TP +.B Inspect detailed manifest and parameters of a model: +.RS +.nf +ollama_nvfp4 show llama3.2:3b +.fi +.RE +.TP +.B Run gguf_info on an Ollama model's weight blob: +.RS +.nf +ollama_nvfp4 --info llama3.2:3b --metadata +.fi +.RE +.TP +.B Export a model and its Modelfile: +.RS +.nf +ollama_nvfp4 export llama3.2:3b -o ./llama3-export +.fi +.RE +.TP +.B Quantize an Ollama FP16 model to NVFP4 and install it: +.RS +.nf +ollama_nvfp4 quant --install llama3.2:3b-instruct-fp16 +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +An error occurred (e.g. model not found, invalid store directory, or quantization failure). +.SH SEE ALSO +.BR gguf_info (1), +.BR gguf_to_nvfp4_gguf (1), +.BR hf_to_nvfp4_gguf (1) + diff --git a/man/man1/ollama_nvfp4.py.1 b/man/man1/ollama_nvfp4.py.1 new file mode 120000 index 0000000..4291a0d --- /dev/null +++ b/man/man1/ollama_nvfp4.py.1 @@ -0,0 +1 @@ +ollama_nvfp4.1 \ No newline at end of file diff --git a/man/man1/safetensors_to_nvfp4_gguf.1 b/man/man1/safetensors_to_nvfp4_gguf.1 new file mode 100644 index 0000000..74f6a50 --- /dev/null +++ b/man/man1/safetensors_to_nvfp4_gguf.1 @@ -0,0 +1,219 @@ +.TH SAFETENSORS_TO_NVFP4_GGUF 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +safetensors_to_nvfp4_gguf \- quantize a safetensors file to an NVFP4 GGUF weight container +.SH SYNOPSIS +.B safetensors_to_nvfp4_gguf +[\fB\-o\fR \fIOUTPUT\fR] +[\fB\-\-all\-weights\fR] +[\fB\-\-no\-quant\fR] +[\fB\-\-outtype\fR \fITYPE\fR] +[\fB\-\-llama\-cpp\fR \fIPATH\fR] +[\fB\-\-convert\-to\-quant\fR \fIPATH\fR] +[\fB\-\-overwrite\fR|\fB\-\-force\fR] +[\fB\-\-mmproj\fR \fIMMPROJ\fR] +[\fB\-\-context\-length\fR \fILEN\fR] +[\fB\-\-checkpoint\-dir\fR \fIDIR\fR] +[\fB\-\-fresh\fR] +[\fB\-\-keep\-checkpoints\fR] +[\fB\-\-no\-temp\-file\fR] +[\fB\-\-verbose\fR] +[\fB\-\-self\-test\fR] +[\fB\-\-device\fR \fIDEVICE\fR] +[\fB\-\-simple\fR] +[\fB\-\-optimizer\fR \fIOPT\fR] +[\fB\-\-num\-iter\fR \fIN\fR] +[\fB\-\-lr\fR \fILR\fR] +[\fB\-\-lr\-schedule\fR \fISCHED\fR] +[\fB\-\-scale\-optimization\fR \fIMODE\fR] +[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR] +[\fB\-\-full\-matrix\fR] +[\fB\-\-top\-p\fR \fIP\fR] +[\fB\-\-min\-k\fR \fIK\fR] +[\fB\-\-max\-k\fR \fIK\fR] +[\fB\-\-early\-stop\-loss\fR \fILOSS\fR] +[\fB\-\-early\-stop\-lr\fR \fILR\fR] +[\fB\-\-early\-stop\-stall\fR \fISTALL\fR] +[\fB\-\-target\-loss\fR \fITARGET\fR] +[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR] +[\fB\-\-lr\-min\fR \fIMIN\fR] +[\fB\-\-lr\-patience\fR \fIN\fR] +[\fB\-\-use\-speed\fR] +[\fB\-\-calib\-samples\fR \fISAMPLES\fR] +[\fB\-\-seed\fR \fISEED\fR] +[\fB\-\-no\-bias\-correction\fR] +[\fB\-\-quantize\-output\fR] +[\fB\-\-exclude\fR \fIREGEX\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fIsafetensors\fR +.SH DESCRIPTION +.B safetensors_to_nvfp4_gguf +quantizes the tensor weights stored in a single \fI.safetensors\fR file directly into +an NVFP4 GGUF weight container, without requiring a Hugging Face model directory or +tokenizer configuration. +.PP +This tool is particularly useful for standalone diffusion models (e.g. FLUX, SDXL), +isolated LoRA weights, or intermediate checkpoints. All original tensor names are +preserved. +.PP +By default, layers on \fBconvert_to_quant\fR's standard exclusion list (normalization +layers, biases, embeddings, and output heads) remain unquantized in their original precision. +Passing \fB\-\-all\-weights\fR overrides this heuristic and quantizes every eligible 2D +matrix whose trailing dimension is a multiple of 64. +.PP +The tool also supports repacking ComfyUI-style NVFP4 \fI.safetensors\fR files (which contain +\fIweight_scale\fR and \fIweight_scale_2\fR) into GGUF format without re-optimization using +\fB\-\-no\-quant \-\-outtype nvfp4\fR. +.SH POSITIONAL ARGUMENTS +.TP +.I safetensors +Path to the input \fI.safetensors\fR file. +.SH OPTIONS +.SS Output & Packaging +.TP +.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT +Output GGUF file path (default: \fI-NVFP4.gguf\fR, or \fI.gguf\fR with \fB\-\-no\-quant\fR and \fBauto\fR). +.TP +.BR \-\-overwrite ", " \-\-force +Replace an existing GGUF or auxiliary output file without prompting. +.TP +.B \-\-all\-weights +Quantize every 2D weight matrix eligible for 4-bit block packing, ignoring convert_to_quant's +default exclusion list. +.TP +.B \-\-no\-quant +Skip learned rounding. Writes a float GGUF or repacks ComfyUI NVFP4 weights. +.TP +.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4} +Data type when learned rounding is skipped: \fBnvfp4\fR repacks ComfyUI NVFP4 weights; \fBauto\fR preserves original dtypes (default: \fBauto\fR). +.TP +.BI \-\-mmproj " " MMPROJ +Path to a vision projector GGUF to copy next to output. May be specified multiple times. +.TP +.BI \-\-context\-length " " LEN +Maximum context length to store in the GGUF metadata. +.SS Quantization & Optimization +.TP +.B \-\-simple +Round each value directly onto the E2M1 grid without learned rounding. +.TP +.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original} +Learned-rounding optimizer (default: \fBauto\fR). +.TP +.BI \-\-num\-iter " " N +Optimization iterations per matrix (default: 4000). +.TP +.BI \-\-lr " " LR +Initial learning rate (default: 1.0). +.TP +.BI \-\-lr\-schedule " " {plateau,exponential,adaptive} +Learning rate decay schedule (default: \fBplateau\fR). +.TP +.BI \-\-lr\-min " " MIN +Lowest learning rate floor (default: 1e-8). +.TP +.BI \-\-lr\-patience " " N +Patience steps before decaying the learning rate (default: 5). +.TP +.B \-\-use\-speed +Enable Prodigy optimizer's \fIuse_speed\fR mode. +.TP +.BI \-\-scale\-optimization " " {fixed,iterative,joint} +Block scale tuning method (default: \fBjoint\fR). +.TP +.BI \-\-scale\-refinement\-rounds " " N +Scale refresh iterations for iterative mode (default: 2). +.TP +.B \-\-full\-matrix +Use full SVD instead of low-rank SVD. +.TP +.BI \-\-top\-p " " P ", " \-\-min\-k " " K ", " \-\-max\-k " " K +Singular value decomposition rank selection bounds. +.TP +.BI \-\-target\-loss " " TARGET +Target reconstruction loss threshold (default: 1e-5; 0 disables). +.TP +.BI \-\-early\-stop\-stall " " STALL +Early stop stall steps without reaching a new best loss (default: 2000). +.TP +.BI \-\-early\-stop\-floor\-stall " " FLOOR +Stall steps at minimum learning rate before stopping (default: 64; 0 disables). +.TP +.BI \-\-early\-stop\-lr " " LR +Learning rate threshold for early stop check. +.TP +.BI \-\-calib\-samples " " SAMPLES +Random Gaussian calibration inputs for sibling bias correction (default: 3072). +.TP +.BI \-\-seed " " SEED +Random seed (default: 42). +.TP +.B \-\-no\-bias\-correction +Disable sibling bias correction. +.TP +.B \-\-quantize\-output +Also quantize \fIoutput.weight\fR. +.TP +.BI \-\-exclude " " REGEX +Regex matching tensor names to leave unquantized. +.SS System & Execution +.TP +.BI \-\-device " " DEVICE +Device for quantization computations: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR). +.TP +.BI \-\-llama\-cpp " " PATH +Path to llama.cpp repository checkout (default: \fI~/llama.cpp\fR). +.TP +.BI \-\-convert\-to\-quant " " PATH +Path to convert_to_quant checkout. +.TP +.BI \-\-checkpoint\-dir " " DIR +Directory for per-tensor resume checkpoints. +.TP +.B \-\-fresh +Delete checkpoint directory before starting. +.TP +.B \-\-keep\-checkpoints +Keep per-tensor checkpoints after completion. +.TP +.B \-\-no\-temp\-file +Keep finished GGUF tensors in memory rather than a temporary file. +.TP +.B \-\-verbose +Enable verbose logging. +.TP +.B \-\-self\-test +Verify NVFP4 packing against llama.cpp dequantizer and exit. +.SH EXAMPLES +.TP +.B Quantize a standalone diffusion model: +.RS +.nf +safetensors_to_nvfp4_gguf -o flux1-dev-nvfp4.gguf flux1-dev.safetensors +.fi +.RE +.TP +.B Repack ComfyUI NVFP4 safetensors to GGUF: +.RS +.nf +safetensors_to_nvfp4_gguf --no-quant --outtype nvfp4 -o repacked.gguf comfy-nvfp4.safetensors +.fi +.RE +.TP +.B Quantize every eligible weight matrix with simple grid rounding: +.RS +.nf +safetensors_to_nvfp4_gguf --simple --all-weights -o fast-nvfp4.gguf model.safetensors +.fi +.RE +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +An error occurred during conversion or packaging. +.SH SEE ALSO +.BR hf_to_nvfp4_safetensors (1), +.BR hf_to_nvfp4_gguf (1), +.BR gguf_to_nvfp4_gguf (1) + diff --git a/man/man1/safetensors_to_nvfp4_gguf.py.1 b/man/man1/safetensors_to_nvfp4_gguf.py.1 new file mode 120000 index 0000000..37baed7 --- /dev/null +++ b/man/man1/safetensors_to_nvfp4_gguf.py.1 @@ -0,0 +1 @@ +safetensors_to_nvfp4_gguf.1 \ No newline at end of file