fix missing gguf-py

Dockerfile

@@ -72,8 +72,8 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install torch
 RUN git clone --depth=1 https://github.com/silveroxides/convert_to_quant && \
     cd convert_to_quant && pip3 install -r requirements.txt && pip3 install .
 
-# Install heretic-llm
+# Install heretic-llm and compressed-tensors
-RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm
+RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm compressed-tensors
 
 # Install zefie's nvfp4 quant requirements
 COPY --chmod=0644 requirements.txt /workspace/requirements-nvfp4.txt
@@ -105,14 +105,14 @@ WORKDIR /opt/nvfp4
 
 RUN mkdir nvfp4_gguf_quant
 COPY --exclude=__pycache__ nvfp4gguf ./nvfp4_gguf_quant/nvfp4gguf
-COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py safetensors_to_nvfp4_gguf.py hf_info.py gguf_info.py ./nvfp4_gguf_quant/
+COPY --chmod=0644 --exclude=ollama_nvfp4.py *.py ./nvfp4_gguf_quant/
 
 COPY --chmod=0644 docker/README.txt /workspace/README.txt
 COPY --chmod=0755 --exclude=README.txt docker/* /usr/local/bin/
 
 # Install manpages for zefie's nvfp4 quant scripts, ctq and heretic
 RUN mkdir -p /usr/local/share/man/man1
-COPY --chmod=0644 man/man1/* /usr/local/share/man/man1/
+COPY --chmod=0644 --exclude=man/man1/ollama_nvfp4.1 --exclude=man/man1/ollama_nvfp4.py.1 man/man1/* /usr/local/share/man/man1/
 
 # Copy llama.cpp compiled binaries, manpages and shared libraries from builder
 COPY --from=builder /install/bin/ /usr/local/bin/
@@ -122,6 +122,7 @@ COPY --from=builder /install/share/man/ /usr/share/man/
 # Set up a minimal llama.cpp source for conversions
 RUN mkdir /opt/nvfp4/llama.cpp
 COPY --from=builder /workspace/conversion /opt/nvfp4/llama.cpp/conversion/
+COPY --from=builder /workspace/gguf-py /opt/nvfp4/llama.cpp/gguf-py/
 COPY --from=builder --chmod=0644 /workspace/*.py /opt/nvfp4/llama.cpp/
 
 COPY --from=venv-builder /workspace/py-venv /opt/nvfp4/py-venv

README.md

@@ -0,0 +1,33 @@
+These scripts require [llama.cpp](https://github.com/ggml-org/llama.cpp) and [Convert-to-Quant](https://github.com/silveroxides/convert_to_quant).
+
+`hf`, the Hugging Face CLI command is also suggested, but not required.
+
+For quantizations or conversions, try:
+- `hf_to_nvfp4_gguf` (huggingface structure to GGUF)
+- `gguf_to_nvfp4_gguf` (GGUF to GGUF)
+- `safetensors_to_nvfp4_gguf` (ComfyUI style safetensors to GGUF (untested))
+- `hf_to_nvfp4_safetensors` (huggingface structure to ComfyUI style safetensors)
+- `hf_to_nvfp4_hf` (huggingface structure to NVFP4 Hugging Face structure)
+- `gguf_to_nvfp4_hf` (GGUF to NVFP4 Hugging Face structure)
+
+For model info, try:
+- `hf_info` (huggingface structure)
+- `gguf_info` (GGUF structure)
+---
+
+Example:
+
+```bash
+hf download coder3101/gemma-4-12B-it-heretic --local-dir gemma-4-12B-it-heretic
+hf_to_nvfp4_gguf -o gemma-4-12B-it-heretic_nvfp4 gemma-4-12B-it-heretic
+```
+
+This would create a NVFP4 GGUF for Ollama/llama.cpp from `https://huggingface.co/coder3101/gemma-4-12B-it-heretic`
+
+The same directory can be quantized to one ComfyUI NVFP4 safetensors file:
+
+`hf_to_nvfp4_safetensors -o gemma-4-12B-it-heretic-NVFP4.safetensors gemma-4-12B-it-heretic`
+
+Conversions between formats without quantization is available in the custom commands with `--no-quant`.
+
+Currently the only supported modes are "Quantize to NVFP4" or "Copy without quantizing" (`--no-quant`). I may add other quantization options in the future.

docker.sh

@@ -8,6 +8,7 @@ get_latest() {
 	latest_image=$(docker images --format '{{.CreatedAt}}\t{{.Repository}}:{{.Tag}}' zefie/nvfp4_quant \
 	  | sort -r \
 	  | cut -f2- \
+	  | grep -v latest \
 	  | head -n 1)
 
 	if [[ -z "$latest_image" ]]; then
@@ -44,7 +45,9 @@ elif [ "${1}" == "push" ]; then
 	# push dated one first, then latest
 	"${0}" mklatest
 	IMAGE=$(get_latest)
+	echo "Pushing ${IMAGE}"
 	docker push "${IMAGE}"
+	echo "Pushing zefie/nvfp4_quant:latest"
 	docker push "zefie/nvfp4_quant:latest"
 else
 	echo "Usage: ${0} [build|run|mklatest]"

docker/README.txt

@@ -1,11 +1,13 @@
 Convert-to-Quant (ctq) is installed for normal quantizations.
 
-For GGUF quantizations, try:
+For quantizations and conversions, try:
 
-hf_to_nvfp4_quant
+- hf_to_nvfp4_gguf (Hugging Face to GGUF)
-gguf_to_nvfp4_quant
+- gguf_to_nvfp4_gguf (GGUF to GGUF)
-safetensors_to_nvfp4_quant
+- safetensors_to_nvfp4_gguf (ComfyUI safetensors to GGUF)
-hf_to_nvfp4_safetensors
+- hf_to_nvfp4_safetensors (Hugging Face to ComfyUI safetensors)
+- hf_to_nvfp4_hf (Hugging Face to NVFP4 Hugging Face)
+- gguf_to_nvfp4_hf (GGUF to NVFP4 Hugging Face)
 
 Depending on the source material.
 
@@ -15,8 +17,14 @@ hf cli command is also installed, eg you can:
 hf download coder3101/gemma-4-12B-it-heretic --local-dir gemma-4-12B-it-heretic
 hf_to_nvfp4_gguf -o gemma-4-12B-it-heretic_nvfp4 gemma-4-12B-it-heretic
 
-The same directory can be quantized to one ComfyUI NVFP4 safetensors file:
+The same directory can be quantized to NVFP4 Hugging Face format:
+
+hf_to_nvfp4_hf -o gemma-4-12B-it-heretic-NVFP4-HF gemma-4-12B-it-heretic
+
+Or to one ComfyUI NVFP4 safetensors file:
 
 hf_to_nvfp4_safetensors -o gemma-4-12B-it-heretic-NVFP4.safetensors gemma-4-12B-it-heretic
 
-This would create a NVFP4 GGUF for Ollama/llama.cpp from https://huggingface.co/coder3101/gemma-4-12B-it-heretic
+A GGUF file can also be converted to NVFP4 Hugging Face format:
+
+gguf_to_nvfp4_hf -o gemma-4-12B-it-heretic-NVFP4-HF gemma-4-12B-it-heretic.gguf

docker/gguf_to_nvfp4_hf

@@ -0,0 +1,3 @@
+#!/bin/bash
+python /opt/nvfp4/nvfp4_gguf_quant/gguf_to_nvfp4_hf.py --llama-cpp /opt/nvfp4/llama.cpp --convert-to-quant /opt/nvfp4/convert_to_quant "${@}"
+

docker/hf_to_nvfp4_hf

@@ -0,0 +1,3 @@
+#!/bin/bash
+python /opt/nvfp4/nvfp4_gguf_quant/hf_to_nvfp4_hf.py --llama-cpp /opt/nvfp4/llama.cpp --convert-to-quant /opt/nvfp4/convert_to_quant "${@}"
+

gguf_to_nvfp4_hf.py

@@ -0,0 +1,761 @@
+"""Convert and quantize a GGUF model to NVFP4 Hugging Face format.
+
+Reads a GGUF file and writes a Hugging Face format model directory (config.json +
+safetensors). By default (--format hf), writes standard Hugging Face compressed-tensors
+format with .weight_packed, .weight_scale, and .weight_global_scale natively supported
+by Hugging Face transformers, vLLM, SGLang, and llama.cpp.
+
+Optionally (--format modelopt / comfy), outputs raw weight tensors (.weight),
+weight_scale, weight_scale_2, and .comfy_quant for ModelOpt and ComfyUI.
+
+If the source GGUF contains float or non-NVFP4 weights, weight matrices are quantized
+to NVFP4 using convert_to_quant's learned rounding (or simple round-to-grid with --simple).
+If the source GGUF is already NVFP4, weights are repacked into safetensors without loss
+of precision.
+
+If --no-quant is specified, weights are dequantized and written to safetensors in the
+target float format (--outtype f32, f16, bf16, or auto).
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import logging
+import os
+import re
+import shutil
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+from safetensors.torch import save_file
+
+from nvfp4gguf import (
+    CTQ_NUM_ITER,
+    DEFAULT_CONVERT_TO_QUANT,
+    DEFAULT_LLAMA_CPP,
+    DEFAULT_EARLY_STOP_FLOOR_STALL,
+    DEFAULT_TARGET_LOSS,
+    DEFAULT_EARLY_STOP_STALL,
+    DEFAULT_LR_FLOOR,
+    DEFAULT_LR_PATIENCE,
+    DEFAULT_LR_THRESHOLD,
+    DEFAULT_SCALE_OPTIMIZATION,
+    DEFAULT_SCALE_REFINEMENT_ROUNDS,
+    QUANT_VERSION,
+    TensorQuantizer,
+    add_quant_arguments,
+    bind_exclude,
+    coordinate_descent_refine,
+    ctq_skips_key,
+    ensure_cuda_interpreter,
+    load_learned_converter,
+    quant_profile,
+    report_reconstruction,
+    resolve_optimizer,
+    sweep_per_tensor_scale,
+    ue4m3_bits,
+)
+
+INDEX_NAME = "model.safetensors.index.json"
+MAX_SHARD_BYTES = 5 * 1024 * 1024 * 1024  # 5 GB max shard size
+
+log = logging.getLogger("gguf_to_nvfp4_hf")
+
+
+def _checkpoint_file(directory: Path, name: str) -> Path:
+    digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:16]
+    safe = re.sub(r"[^A-Za-z0-9._-]+", "_", name)[:80]
+    return directory / f"{safe}.{digest}.npz"
+
+
+def _load_checkpoint(directory: Path, name: str, shape: tuple[int, ...], need_bias: bool):
+    path = _checkpoint_file(directory, name)
+    if not path.is_file():
+        return None
+    with np.load(path) as blob:
+        if "version" in blob.files and int(blob["version"]) != QUANT_VERSION:
+            return None
+        if tuple(int(v) for v in blob["shape"]) != tuple(shape):
+            return None
+        if need_bias and "bias" not in blob.files:
+            return None
+        codes = np.array(blob["codes"], copy=True)
+        block_scale = np.array(blob["block_scale"], copy=True)
+        per_tensor = float(blob["per_tensor"])
+        bias = np.array(blob["bias"], copy=True) if "bias" in blob.files else None
+    return codes, block_scale, per_tensor, bias
+
+
+def _save_checkpoint(directory: Path, name: str, shape: tuple[int, ...], codes, block_scale, per_tensor, bias) -> None:
+    path = _checkpoint_file(directory, name)
+    temporary = path.with_name(path.name + ".tmp")
+    with temporary.open("wb") as handle:
+        save_dict = {
+            "version": np.int32(QUANT_VERSION),
+            "shape": np.asarray(shape, dtype=np.int64),
+            "codes": np.asarray(codes, dtype=np.uint8),
+            "block_scale": np.asarray(block_scale, dtype=np.float32),
+            "per_tensor": np.float32(per_tensor),
+        }
+        if bias is not None:
+            save_dict["bias"] = np.asarray(bias, dtype=np.float32)
+        np.savez_compressed(handle, **save_dict)
+    temporary.replace(path)
+
+
+def _pythonize(value):
+    if isinstance(value, np.generic):
+        return value.item()
+    if isinstance(value, np.ndarray):
+        return value.tolist()
+    if isinstance(value, list):
+        return [_pythonize(item) for item in value]
+    return value
+
+
+def _text(value) -> str:
+    value = _pythonize(value)
+    if isinstance(value, bytes):
+        return value.decode("utf-8", "replace")
+    if isinstance(value, list):
+        return "".join(_text(item) for item in value)
+    return str(value)
+
+
+def _to_float(tensor, gguf_module) -> np.ndarray:
+    qtype = tensor.tensor_type
+    if qtype == gguf_module.GGMLQuantizationType.F32:
+        values = np.asarray(tensor.data, dtype=np.float32)
+    elif qtype == gguf_module.GGMLQuantizationType.F16:
+        values = np.asarray(tensor.data, dtype=np.float32)
+    elif qtype == gguf_module.GGMLQuantizationType.BF16:
+        values = np.asarray(tensor.data, dtype=np.float32)
+    else:
+        values = np.asarray(gguf_module.quants.dequantize(tensor.data, qtype), dtype=np.float32)
+    return np.array(values, dtype=np.float32, copy=True)
+
+
+def _target_dtype(dtype_str: str) -> torch.dtype:
+    mapping = {
+        "float32": torch.float32,
+        "f32": torch.float32,
+        "float16": torch.float16,
+        "f16": torch.float16,
+        "bfloat16": torch.bfloat16,
+        "bf16": torch.bfloat16,
+    }
+    return mapping.get(dtype_str.lower(), torch.bfloat16)
+
+
+def _assign_output(args) -> None:
+    source = Path(args.gguf)
+    stem = source.stem
+    passthrough = args.no_quant or args.outtype == "nvfp4"
+    if not args.output:
+        if passthrough and args.outtype == "auto":
+            directory = Path.cwd() / stem
+        elif passthrough:
+            label = "NVFP4" if args.outtype == "nvfp4" else args.outtype.upper()
+            directory = Path.cwd() / f"{stem}-{label}"
+        else:
+            directory = Path.cwd() / f"{stem}-NVFP4"
+    else:
+        directory = Path(args.output).resolve()
+    args.output = str(directory)
+
+
+def pack_nvfp4_hf(codes: np.ndarray, block_scales: np.ndarray, per_tensor: float):
+    """Pack codes and scales into NVFP4 tensors."""
+    packed = (codes[:, 0::2] & 0x0F) | ((codes[:, 1::2] & 0x0F) << 4)
+    packed_tensor = torch.from_numpy(np.ascontiguousarray(packed, dtype=np.uint8))
+
+    scale_bits = ue4m3_bits(block_scales)
+    scale_tensor = torch.from_numpy(np.ascontiguousarray(scale_bits, dtype=np.uint8)).view(torch.float8_e4m3fn)
+    scale_2_tensor = torch.tensor(float(per_tensor), dtype=torch.float32)
+
+    return packed_tensor, scale_tensor, scale_2_tensor
+
+
+def unpack_gguf_nvfp4_to_hf(raw: np.ndarray, per_tensor: float):
+    """Convert GGUF NVFP4 super-blocks (36 bytes / 64 values) into ModelOpt tensors."""
+    rows = raw.shape[0]
+    n_super = raw.shape[1] // 36
+    n_blocks = n_super * 4
+    cols = n_blocks * 16
+
+    raw_t = torch.from_numpy(raw).reshape(rows, n_super, 36)
+    d_ue = raw_t[:, :, :4].reshape(rows, n_blocks)
+    qs = raw_t[:, :, 4:].reshape(rows, n_blocks, 8)
+
+    vals = torch.empty((rows, n_blocks, 16), dtype=torch.uint8)
+    vals[:, :, :8] = qs & 0x0F
+    vals[:, :, 8:] = qs >> 4
+
+    w = ((vals[:, :, 0::2] & 0x0F) | ((vals[:, :, 1::2] & 0x0F) << 4)).reshape(rows, cols // 2)
+
+    scale_tensor = d_ue.contiguous().view(torch.float8_e4m3fn)
+    scale_2_tensor = torch.tensor(float(per_tensor), dtype=torch.float32)
+    return w, scale_tensor, scale_2_tensor
+
+
+def unpermute_rope(weights: np.ndarray, n_head: int, n_kv_head: int | None = None) -> np.ndarray:
+    """Invert RoPE permutation applied during HF to GGUF conversion."""
+    head_count = n_kv_head if n_kv_head is not None else n_head
+    if head_count <= 0 or weights.shape[0] % head_count != 0:
+        return weights
+    head_dim = weights.shape[0] // head_count
+    if head_dim % 2 != 0:
+        return weights
+    w = weights.reshape(head_count, head_dim // 2, 2, *weights.shape[1:])
+    w = np.swapaxes(w, 1, 2)
+    return w.reshape(weights.shape)
+
+
+def build_tensor_name_map(arch_name: str, block_count: int, gguf_module) -> dict[str, str]:
+    """Build a mapping from GGUF tensor names to standard Hugging Face names."""
+    arch_enum = getattr(gguf_module.MODEL_ARCH, arch_name.upper(), None)
+    mapping: dict[str, str] = {}
+
+    if arch_enum is not None:
+        try:
+            tm = gguf_module.get_tensor_name_map(arch_enum, block_count)
+            rev: dict[str, list[str]] = {}
+            for hf_key, (_tensor_enum, gguf_name) in tm.mapping.items():
+                if hf_key == gguf_name:
+                    continue
+                if gguf_name not in rev:
+                    rev[gguf_name] = []
+                rev[gguf_name].append(hf_key)
+
+            for g_name, hf_candidates in rev.items():
+                preferred = [c for c in hf_candidates if c.startswith("model.") or c.startswith("transformer.")]
+                mapping[g_name] = preferred[0] if preferred else hf_candidates[0]
+        except Exception:
+            pass
+
+    defaults = {
+        "token_embd": "model.embed_tokens",
+        "output_norm": "model.norm",
+        "output": "lm_head",
+    }
+    for g, h in defaults.items():
+        mapping.setdefault(g, h)
+
+    for bid in range(block_count):
+        bid_defaults = {
+            f"blk.{bid}.attn_q": f"model.layers.{bid}.self_attn.q_proj",
+            f"blk.{bid}.attn_k": f"model.layers.{bid}.self_attn.k_proj",
+            f"blk.{bid}.attn_v": f"model.layers.{bid}.self_attn.v_proj",
+            f"blk.{bid}.attn_output": f"model.layers.{bid}.self_attn.o_proj",
+            f"blk.{bid}.attn_norm": f"model.layers.{bid}.input_layernorm",
+            f"blk.{bid}.ffn_gate": f"model.layers.{bid}.mlp.gate_proj",
+            f"blk.{bid}.ffn_up": f"model.layers.{bid}.mlp.up_proj",
+            f"blk.{bid}.ffn_down": f"model.layers.{bid}.mlp.down_proj",
+            f"blk.{bid}.ffn_norm": f"model.layers.{bid}.post_attention_layernorm",
+            f"blk.{bid}.attn_q_norm": f"model.layers.{bid}.self_attn.q_norm",
+            f"blk.{bid}.attn_k_norm": f"model.layers.{bid}.self_attn.k_norm",
+            f"blk.{bid}.post_ffw_norm": f"model.layers.{bid}.post_feedforward_layernorm",
+            f"blk.{bid}.pre_ffw_norm": f"model.layers.{bid}.pre_feedforward_layernorm",
+        }
+        for g, h in bid_defaults.items():
+            mapping.setdefault(g, h)
+
+    return mapping
+
+
+def _map_gguf_name(gguf_name: str, name_map: dict[str, str]) -> str:
+    base, _, suffix = gguf_name.rpartition(".")
+    if base in name_map:
+        return f"{name_map[base]}.{suffix}"
+    return gguf_name
+
+
+def _synthesize_hf_configs(reader, output_dir: Path, arch: str, args, is_quantized: bool) -> None:
+    """Generate config.json and tokenizer files from GGUF metadata."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    fields = reader.fields
+
+    def get_field(key: str, default=None):
+        f = fields.get(key)
+        return _pythonize(f.contents()) if f is not None else default
+
+    arch_map = {
+        "llama": ("LlamaForCausalLM", "llama"),
+        "qwen2": ("Qwen2ForCausalLM", "qwen2"),
+        "gemma": ("GemmaForCausalLM", "gemma"),
+        "gemma2": ("Gemma2ForCausalLM", "gemma2"),
+        "mistral": ("MistralForCausalLM", "mistral"),
+        "phi3": ("Phi3ForCausalLM", "phi3"),
+        "deepseek2": ("DeepseekV2ForCausalLM", "deepseek_v2"),
+        "gptneox": ("GPTNeoXForCausalLM", "gpt_neox"),
+    }
+    class_name, model_type = arch_map.get(arch.lower(), (f"{arch.capitalize()}ForCausalLM", arch.lower()))
+
+    tokens = get_field("tokenizer.ggml.tokens", [])
+    vocab_size = len(tokens) if tokens else get_field(f"{arch}.vocab_size", 32000)
+
+    config = {
+        "architectures": [class_name],
+        "model_type": model_type,
+        "vocab_size": vocab_size,
+        "hidden_size": get_field(f"{arch}.embedding_length", 4096),
+        "intermediate_size": get_field(f"{arch}.feed_forward_length", 11008),
+        "num_hidden_layers": get_field(f"{arch}.block_count", 32),
+        "num_attention_heads": get_field(f"{arch}.attention.head_count", 32),
+        "num_key_value_heads": get_field(f"{arch}.attention.head_count_kv", get_field(f"{arch}.attention.head_count", 32)),
+        "max_position_embeddings": getattr(args, "context_length", None) or get_field(f"{arch}.context_length", 4096),
+        "rms_norm_eps": get_field(f"{arch}.attention.layer_norm_rms_epsilon", 1e-5),
+        "rope_theta": get_field(f"{arch}.rope.freq_base", 10000.0),
+        "bos_token_id": get_field("tokenizer.ggml.bos_token_id", 1),
+        "eos_token_id": get_field("tokenizer.ggml.eos_token_id", 2),
+        "torch_dtype": getattr(args, "output_dtype", "bfloat16"),
+    }
+
+    fmt = getattr(args, "format", "hf")
+    ignore = []
+    if not getattr(args, "quantize_output", False):
+        ignore.append("lm_head")
+    if getattr(args, "exclude", None):
+        ignore.append(args.exclude)
+
+    if is_quantized:
+        if fmt == "hf":
+            config["quantization_config"] = {
+                "quant_method": "compressed-tensors",
+                "format": "nvfp4-pack-quantized",
+                "quantization_status": "compressed",
+                "ignore": ignore,
+                "config_groups": {
+                    "group_0": {
+                        "weights": {
+                            "num_bits": 4,
+                            "type": "float",
+                            "strategy": "tensor_group",
+                            "group_size": 16,
+                            "symmetric": True,
+                            "dynamic": False,
+                            "actorder": None,
+                        },
+                        "input_activations": None,
+                        "targets": ["Linear"],
+                        "format": "nvfp4-pack-quantized",
+                    }
+                },
+            }
+        else:
+            config["quantization_config"] = {
+                "quant_method": "modelopt",
+                "quant_algo": "NVFP4",
+                "format": "nvfp4-pack-quantized",
+                "group_size": 16,
+                "bits": 4,
+            }
+
+    (output_dir / "config.json").write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
+
+    # Tokenizer configs
+    chat_template = get_field("tokenizer.chat_template")
+    tok_config = {
+        "bos_token_id": config["bos_token_id"],
+        "eos_token_id": config["eos_token_id"],
+        "model_max_length": config["max_position_embeddings"],
+    }
+    if chat_template:
+        tok_config["chat_template"] = _text(chat_template)
+    (output_dir / "tokenizer_config.json").write_text(json.dumps(tok_config, indent=2) + "\n", encoding="utf-8")
+
+    special_tokens = {}
+    if tokens:
+        bos_id = config["bos_token_id"]
+        eos_id = config["eos_token_id"]
+        if 0 <= bos_id < len(tokens):
+            special_tokens["bos_token"] = _text(tokens[bos_id])
+        if 0 <= eos_id < len(tokens):
+            special_tokens["eos_token"] = _text(tokens[eos_id])
+        vocab = {_text(t): i for i, t in enumerate(tokens)}
+        (output_dir / "vocab.json").write_text(json.dumps(vocab, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+    (output_dir / "special_tokens_map.json").write_text(json.dumps(special_tokens, indent=2) + "\n", encoding="utf-8")
+
+    merges = get_field("tokenizer.ggml.merges")
+    if merges:
+        merge_lines = [_text(m) for m in merges]
+        (output_dir / "merges.txt").write_text("#version: 0.2\n" + "\n".join(merge_lines) + "\n", encoding="utf-8")
+
+
+def _copy_template_files(template_dir: Path, output_dir: Path, args, is_quantized: bool) -> None:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    fmt = getattr(args, "format", "hf")
+
+    for path in template_dir.iterdir():
+        if path.is_dir() or path.name.startswith("."):
+            continue
+        if any(path.name.endswith(ext) for ext in (".safetensors", ".bin", ".pt", ".pth", ".h5")):
+            continue
+        if path.name == INDEX_NAME:
+            continue
+        dest = output_dir / path.name
+        if path.name == "config.json":
+            try:
+                cfg = json.loads(path.read_text(encoding="utf-8"))
+                if is_quantized:
+                    if fmt == "hf":
+                        cfg["quantization_config"] = {
+                            "quant_method": "compressed-tensors",
+                            "format": "nvfp4-pack-quantized",
+                            "quantization_status": "compressed",
+                            "ignore": ignore,
+                            "config_groups": {
+                                "group_0": {
+                                    "weights": {
+                                        "num_bits": 4,
+                                        "type": "float",
+                                        "strategy": "tensor_group",
+                                        "group_size": 16,
+                                        "symmetric": True,
+                                        "dynamic": False,
+                                        "actorder": None,
+                                    },
+                                    "input_activations": None,
+                                    "targets": ["Linear"],
+                                    "format": "nvfp4-pack-quantized",
+                                }
+                            },
+                        }
+                    else:
+                        cfg["quantization_config"] = {
+                            "quant_method": "modelopt",
+                            "quant_algo": "NVFP4",
+                            "format": "nvfp4-pack-quantized",
+                            "group_size": 16,
+                            "bits": 4,
+                        }
+                    cfg["torch_dtype"] = getattr(args, "output_dtype", "bfloat16")
+                if getattr(args, "context_length", None):
+                    cfg["max_position_embeddings"] = int(args.context_length)
+                dest.write_text(json.dumps(cfg, indent=2) + "\n", encoding="utf-8")
+                continue
+            except Exception:
+                pass
+        shutil.copy2(path, dest)
+
+
+def export_gguf_to_hf(args) -> None:
+    gguf_path = Path(args.gguf).resolve()
+    if not gguf_path.is_file():
+        raise FileNotFoundError(f"GGUF file does not exist: {gguf_path}")
+
+    _assign_output(args)
+    output_dir = Path(args.output).resolve()
+    if output_dir.exists() and any(output_dir.iterdir()) and not args.overwrite:
+        raise SystemExit(f"Output directory {output_dir} already exists and is not empty. Use --overwrite to replace.")
+
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s")
+    log.info("Loading GGUF file: %s", gguf_path)
+    log.info("Target format: %s", "Hugging Face (compressed-tensors)" if args.format == "hf" else "ModelOpt / ComfyUI")
+
+    import gguf
+    reader = gguf.GGUFReader(gguf_path)
+    arch = _text(reader.fields.get("general.architecture", "llama").contents())
+    block_count = 32
+    for k in (f"{arch}.block_count", "general.block_count"):
+        if k in reader.fields:
+            block_count = int(_pythonize(reader.fields[k].contents()))
+            break
+
+    n_head = int(_pythonize(reader.fields.get(f"{arch}.attention.head_count", 32).contents()))
+    n_kv_head = int(_pythonize(reader.fields.get(f"{arch}.attention.head_count_kv", n_head).contents()))
+    should_unpermute = arch.lower() in ("llama", "mistral", "nanbeige")
+
+    name_map = build_tensor_name_map(arch, block_count, gguf)
+    profile = quant_profile(arch)
+    if profile is not None:
+        if not args.exclude:
+            args.exclude = profile.skip
+            log.info("%s profile: leaving %s unquantized.", profile.arch, profile.label)
+        else:
+            log.info("%s profile: --exclude replaces default skip of %s.", profile.arch, profile.label)
+    args.exclude_re = re.compile(args.exclude) if args.exclude else None
+
+    is_passthrough = args.no_quant or args.outtype == "nvfp4"
+    is_quantized = not is_passthrough
+    float_dtype = _target_dtype(args.output_dtype if not is_passthrough else (args.outtype if args.outtype != "auto" else "bfloat16"))
+
+    # Config & Tokenizer generation / copying
+    if args.hf_template and Path(args.hf_template).is_dir():
+        log.info("Copying configs and tokenizers from template: %s", args.hf_template)
+        _copy_template_files(Path(args.hf_template), output_dir, args, is_quantized=is_quantized)
+    else:
+        log.info("Synthesizing Hugging Face configs and tokenizer from GGUF metadata")
+        _synthesize_hf_configs(reader, output_dir, arch, args, is_quantized=is_quantized)
+
+    # Scale tensors cache for NVFP4 repacking
+    scale_tensors: dict[str, float] = {}
+    for tensor in reader.tensors:
+        if tensor.name.endswith(".scale"):
+            base = tensor.name.removesuffix(".scale")
+            scale_val = float(np.asarray(tensor.data, dtype=np.float32).reshape(-1)[0])
+            scale_tensors[base] = scale_val
+
+    # Checkpoint dir
+    checkpoint_dir = Path(args.checkpoint_dir) if args.checkpoint_dir else output_dir / ".nvfp4-work"
+    if args.fresh and checkpoint_dir.exists():
+        shutil.rmtree(checkpoint_dir)
+    checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+    quantizer = None
+    if is_quantized:
+        ensure_cuda_interpreter(args.device)
+        args.optimizer = resolve_optimizer(args.optimizer)
+        quantizer = TensorQuantizer(args, gguf, load_learned_converter(args.convert_to_quant)[1])
+
+    quant_count = 0
+    passthrough_count = 0
+    bias_count = 0
+    all_quant_meta: dict[str, dict] = {}
+    all_hf_tensors: dict[str, torch.Tensor] = {}
+
+    log.info("Processing %d tensor(s) from GGUF", len(reader.tensors))
+
+    for tensor in reader.tensors:
+        g_name = tensor.name
+        if g_name.endswith(".scale") or g_name.endswith(".input_scale"):
+            continue
+
+        hf_name = _map_gguf_name(g_name, name_map)
+        base_name = hf_name.removesuffix(".weight")
+        shape = tuple(int(d) for d in tensor.shape[::-1])  # GGUF shapes are reversed
+
+        # If already NVFP4 in GGUF
+        if tensor.tensor_type == gguf.GGMLQuantizationType.NVFP4:
+            log.info("  Repacking NVFP4 [%s] %s -> %s", "x".join(str(d) for d in shape), g_name, hf_name)
+            raw = np.asarray(tensor.data, dtype=np.uint8).reshape(shape[0], -1)
+            per_tensor = scale_tensors.get(g_name, 1.0)
+            packed_w, scale_1, scale_2 = unpack_gguf_nvfp4_to_hf(raw, per_tensor)
+
+            if args.format == "hf":
+                all_hf_tensors[f"{base_name}.weight_packed"] = packed_w
+                all_hf_tensors[f"{base_name}.weight_scale"] = scale_1
+                all_hf_tensors[f"{base_name}.weight_global_scale"] = torch.tensor(1.0 / float(per_tensor), dtype=torch.float32)
+            else:
+                all_hf_tensors[hf_name] = packed_w
+                all_hf_tensors[f"{base_name}.weight_scale"] = scale_1
+                all_hf_tensors[f"{base_name}.weight_scale_2"] = scale_2
+                all_hf_tensors[f"{base_name}.comfy_quant"] = torch.tensor([1], dtype=torch.uint8)
+
+            all_quant_meta[base_name] = {"quant_type": "nvfp4", "block_size": 16, "scale_type": "float8_e4m3fn"}
+            quant_count += 1
+            continue
+
+        # Convert tensor data to float
+        data_float = _to_float(tensor, gguf)
+
+        # Unpermute RoPE for Q and K if needed
+        if should_unpermute:
+            if g_name.endswith("attn_q.weight"):
+                data_float = unpermute_rope(data_float, n_head, n_head)
+            elif g_name.endswith("attn_k.weight"):
+                data_float = unpermute_rope(data_float, n_head, n_kv_head)
+
+        # Check eligibility for NVFP4 quantization
+        is_eligible = (
+            is_quantized
+            and len(shape) == 2
+            and hf_name.endswith(".weight")
+            and shape[-1] >= 64
+            and shape[-1] % 64 == 0
+            and not ("norm" in hf_name.lower() or "embed_tokens" in hf_name.lower())
+            and (hf_name != "lm_head.weight" or getattr(args, "quantize_output", False))
+            and not (args.exclude_re and args.exclude_re.search(hf_name))
+            and not ctq_skips_key(hf_name, args.convert_to_quant)
+        )
+
+        if is_eligible and quantizer is not None:
+            log.info("  Quantizing [%s] %s -> %s", "x".join(str(d) for d in shape), g_name, hf_name)
+            has_bias = False
+
+            cached = _load_checkpoint(checkpoint_dir, hf_name, shape, need_bias=has_bias)
+            if cached is not None:
+                codes, block_scale, per_tensor, bias_corr = cached
+                log.info("    Loaded from checkpoint: %s", hf_name)
+            else:
+                rounds = getattr(args, "scale_refinement_rounds", DEFAULT_SCALE_REFINEMENT_ROUNDS)
+                codes, block_scale, per_tensor, dequant = quantizer.quantize_matrix(data_float, hf_name)
+                codes, block_scale = coordinate_descent_refine(
+                    data_float, codes, block_scale, per_tensor, bake=False, rounds=rounds, device=quantizer.device,
+                )
+                codes, block_scale, per_tensor = sweep_per_tensor_scale(
+                    data_float, codes, block_scale, per_tensor, bake=False, device=quantizer.device,
+                )
+                _save_checkpoint(checkpoint_dir, hf_name, shape, codes, block_scale, per_tensor, None)
+
+            packed_w, scale_1, scale_2 = pack_nvfp4_hf(codes, block_scale, per_tensor)
+
+            if args.format == "hf":
+                all_hf_tensors[f"{base_name}.weight_packed"] = packed_w
+                all_hf_tensors[f"{base_name}.weight_scale"] = scale_1
+                all_hf_tensors[f"{base_name}.weight_global_scale"] = torch.tensor(1.0 / float(per_tensor), dtype=torch.float32)
+            else:
+                all_hf_tensors[hf_name] = packed_w
+                all_hf_tensors[f"{base_name}.weight_scale"] = scale_1
+                all_hf_tensors[f"{base_name}.weight_scale_2"] = scale_2
+                all_hf_tensors[f"{base_name}.comfy_quant"] = torch.tensor([1], dtype=torch.uint8)
+
+            all_quant_meta[base_name] = {"quant_type": "nvfp4", "block_size": 16, "scale_type": "float8_e4m3fn"}
+            quant_count += 1
+        else:
+            # Float tensor passthrough
+            tensor_t = torch.from_numpy(data_float)
+            if tensor_t.ndim <= 1 and hf_name.endswith("norm.weight"):
+                tensor_t = tensor_t.float()
+            else:
+                tensor_t = tensor_t.to(dtype=float_dtype)
+            all_hf_tensors[hf_name] = tensor_t
+            passthrough_count += 1
+
+        # Shard writing logic
+    total_bytes = sum(t.numel() * t.element_size() for t in all_hf_tensors.values())
+    metadata = {}
+    if args.format != "hf":
+        metadata["_quantization_metadata"] = json.dumps({"format_version": "1.0", "layers": all_quant_meta}, separators=(",", ":"))
+
+    if total_bytes < MAX_SHARD_BYTES:
+        out_file = output_dir / "model.safetensors"
+        save_file(all_hf_tensors, out_file, metadata=metadata if metadata else None)
+        log.info("Saved single shard: %s", out_file)
+    else:
+        # Split into shards
+        shards_list: list[dict[str, torch.Tensor]] = [{}]
+        current_shard_size = 0
+        for name, tensor in all_hf_tensors.items():
+            t_size = tensor.numel() * tensor.element_size()
+            if current_shard_size + t_size > MAX_SHARD_BYTES and shards_list[-1]:
+                shards_list.append({})
+                current_shard_size = 0
+            shards_list[-1][name] = tensor
+            current_shard_size += t_size
+
+        num_shards = len(shards_list)
+        weight_map: dict[str, str] = {}
+        for idx, shard_dict in enumerate(shards_list, start=1):
+            shard_filename = f"model-{idx:05d}-of-{num_shards:05d}.safetensors"
+            shard_path = output_dir / shard_filename
+            save_file(shard_dict, shard_path, metadata=metadata if metadata else None)
+            for k in shard_dict:
+                weight_map[k] = shard_filename
+            log.info("Saved shard %d/%d: %s", idx, num_shards, shard_path)
+
+        index_payload = {
+            "metadata": {"total_size": total_bytes},
+            "weight_map": weight_map,
+        }
+        (output_dir / INDEX_NAME).write_text(json.dumps(index_payload, indent=2) + "\n", encoding="utf-8")
+
+    if not args.keep_checkpoints and quant_count:
+        shutil.rmtree(checkpoint_dir, ignore_errors=True)
+
+    log.info(
+        "\nWrote %s (%d NVFP4 tensors, %d other tensors)",
+        output_dir,
+        quant_count,
+        passthrough_count,
+    )
+
+
+def self_test(args) -> None:
+    """Self-test NVFP4 unpacking and roundtripping."""
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    from nvfp4gguf import FP4_TABLE
+    import gguf
+
+    log.info("Running gguf_to_nvfp4_hf self-test...")
+    _converter_cls, unpack_uint4 = load_learned_converter(args.convert_to_quant)
+    quantizer = TensorQuantizer(args, gguf, unpack_uint4)
+
+    generator = np.random.default_rng(0)
+    for shape in ((64, 128), (128, 256)):
+        weight = generator.normal(size=shape).astype(np.float32) * 0.05
+        codes, block_scale, per_tensor, dequant = quantizer.quantize_matrix(weight)
+        packed_w, scale_1, scale_2 = pack_nvfp4_hf(codes, block_scale, per_tensor)
+
+        out_features = shape[0]
+        n_blocks = shape[1] // 16
+        n_super = shape[1] // 64
+
+        w_b = packed_w.reshape(out_features, n_blocks, 8)
+        vals = torch.stack([w_b & 0x0F, w_b >> 4], dim=-1).reshape(out_features, n_blocks, 16)
+        d_ue = scale_1.view(torch.uint8).reshape(out_features, n_blocks) & 0x7F
+        qs = (vals[:, :, :8] | (vals[:, :, 8:] << 4)).to(torch.uint8)
+        d_grouped = d_ue.reshape(out_features, n_super, 4)
+        qs_grouped = qs.reshape(out_features, n_super, 32)
+        raw = torch.cat([d_grouped, qs_grouped], dim=-1).reshape(out_features, n_super * 36).numpy()
+
+        w_rec, s1_rec, s2_rec = unpack_gguf_nvfp4_to_hf(raw, per_tensor)
+        assert torch.equal(packed_w, w_rec), "GGUF unpack weights mismatch!"
+        assert torch.equal(scale_1.view(torch.uint8) & 0x7F, s1_rec.view(torch.uint8) & 0x7F), "GGUF unpack scales mismatch!"
+        log.info("Shape %s GGUF-to-HF unpack roundtrip ok", shape)
+
+    log.info("self-test passed successfully.")
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Convert and quantize a GGUF model to NVFP4 Hugging Face format.",
+    )
+    parser.add_argument("gguf", nargs="?", help="Input GGUF file")
+    parser.add_argument("-o", "--output", help="Output directory. Default: ./<model-name>-NVFP4")
+    parser.add_argument("--format", default="hf", choices=("hf", "modelopt", "comfy"),
+                        help="Safetensors tensor layout: 'hf' (default) uses Hugging Face compressed-tensors (.weight_packed, .weight_scale, .weight_global_scale). 'modelopt' (or 'comfy') uses raw .weight.")
+    parser.add_argument("--overwrite", action="store_true", help="Replace existing files in the output directory")
+    parser.add_argument("--no-quant", action="store_true", help="Skip learned rounding; write float HF model")
+    parser.add_argument("--outtype", default="auto", choices=("auto", "f32", "f16", "bf16", "nvfp4"),
+                        help="Float type for tensors when --no-quant is used (default: auto)")
+    parser.add_argument("--output-dtype", default="bfloat16", choices=("bfloat16", "float16"),
+                        help="Dtype for unquantized 2D weights and embeddings (default: bfloat16)")
+    parser.add_argument("--hf-template", type=Path, default=None,
+                        help="Reference Hugging Face model directory to copy config and tokenizer files from")
+    parser.add_argument("--context-length", type=int, default=None,
+                        help="Override context length stored in config.json")
+    parser.add_argument("--model-name", default=None, help="Override model name")
+    parser.add_argument("--checkpoint-dir", type=Path, default=None, help="Per-tensor resume directory")
+    parser.add_argument("--fresh", action="store_true", help="Delete checkpoint directory before starting")
+    parser.add_argument("--keep-checkpoints", action="store_true", help="Keep per-tensor checkpoints upon completion")
+    parser.add_argument("--eager", action="store_true", help="Eager loading")
+    parser.add_argument("--low-memory", action="store_true", help="Stream each tensor instead of loading all at once")
+    parser.add_argument("--heur", action="store_true", help="Skip layers heuristic rejects")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose logging")
+    parser.add_argument("--self-test", action="store_true", help="Run NVFP4 packing self-test and exit")
+    parser.add_argument("--llama-cpp", type=Path, default=DEFAULT_LLAMA_CPP,
+                        help="llama.cpp checkout directory (default: ~/llama.cpp)")
+
+    add_quant_arguments(parser)
+    args = parser.parse_args(argv)
+
+    if args.self_test:
+        if args.device == "auto":
+            args.device = "cpu"
+        return args
+
+    if not args.gguf:
+        parser.error("GGUF model file is required")
+
+    if args.convert_to_quant is None and DEFAULT_CONVERT_TO_QUANT.is_dir():
+        args.convert_to_quant = DEFAULT_CONVERT_TO_QUANT
+
+    return args
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = parse_args(argv)
+    if args.self_test:
+        self_test(args)
+        return
+    export_gguf_to_hf(args)
+
+
+if __name__ == "__main__":
+    main()

hf_to_nvfp4_hf.py

@@ -0,0 +1,620 @@
+"""Quantize a Hugging Face model directory to NVFP4 Hugging Face format.
+
+Reads a Hugging Face model directory (config.json + safetensors) and writes a
+quantized Hugging Face model directory. By default (--format hf), writes standard
+Hugging Face compressed-tensors format with .weight_packed, .weight_scale, and
+.weight_global_scale natively supported by Hugging Face transformers, vLLM, SGLang,
+and llama.cpp.
+
+Optionally (--format modelopt / comfy), outputs raw weight tensors (.weight),
+weight_scale, weight_scale_2, and .comfy_quant for ModelOpt and ComfyUI.
+
+Weight matrices are quantized using convert_to_quant's learned rounding (or simple
+round-to-grid with --simple). Biases are adjusted using calibration data unless
+--no-bias-correction is specified. Embeddings, layer norms, and excluded layers
+stay in their float format.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import logging
+import os
+import re
+import shutil
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+from nvfp4gguf import (
+    CTQ_NUM_ITER,
+    DEFAULT_CONVERT_TO_QUANT,
+    DEFAULT_LLAMA_CPP,
+    DEFAULT_EARLY_STOP_FLOOR_STALL,
+    DEFAULT_TARGET_LOSS,
+    DEFAULT_EARLY_STOP_STALL,
+    DEFAULT_LR_FLOOR,
+    DEFAULT_LR_PATIENCE,
+    DEFAULT_LR_THRESHOLD,
+    DEFAULT_SCALE_OPTIMIZATION,
+    DEFAULT_SCALE_REFINEMENT_ROUNDS,
+    QUANT_VERSION,
+    TensorQuantizer,
+    add_quant_arguments,
+    bind_exclude,
+    coordinate_descent_refine,
+    ctq_skips_key,
+    ensure_cuda_interpreter,
+    load_learned_converter,
+    quant_profile,
+    report_reconstruction,
+    resolve_optimizer,
+    sweep_per_tensor_scale,
+    ue4m3_bits,
+)
+
+INDEX_NAME = "model.safetensors.index.json"
+NON_WEIGHT_PATTERNS = {
+    ".safetensors",
+    ".bin",
+    ".pt",
+    ".pth",
+    ".h5",
+    ".msgpack",
+    ".ot",
+    ".tmp",
+}
+
+log = logging.getLogger("hf_to_nvfp4_hf")
+
+
+def _checkpoint_file(directory: Path, name: str) -> Path:
+    digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:16]
+    safe = re.sub(r"[^A-Za-z0-9._-]+", "_", name)[:80]
+    return directory / f"{safe}.{digest}.npz"
+
+
+def _load_checkpoint(directory: Path, name: str, shape: tuple[int, ...], need_bias: bool):
+    path = _checkpoint_file(directory, name)
+    if not path.is_file():
+        return None
+    with np.load(path) as blob:
+        if "version" in blob.files and int(blob["version"]) != QUANT_VERSION:
+            return None
+        if tuple(int(v) for v in blob["shape"]) != tuple(shape):
+            return None
+        if need_bias and "bias" not in blob.files:
+            return None
+        codes = np.array(blob["codes"], copy=True)
+        block_scale = np.array(blob["block_scale"], copy=True)
+        per_tensor = float(blob["per_tensor"])
+        bias = np.array(blob["bias"], copy=True) if "bias" in blob.files else None
+    return codes, block_scale, per_tensor, bias
+
+
+def _save_checkpoint(directory: Path, name: str, shape: tuple[int, ...], codes, block_scale, per_tensor, bias) -> None:
+    path = _checkpoint_file(directory, name)
+    temporary = path.with_name(path.name + ".tmp")
+    with temporary.open("wb") as handle:
+        save_dict = {
+            "version": np.int32(QUANT_VERSION),
+            "shape": np.asarray(shape, dtype=np.int64),
+            "codes": np.asarray(codes, dtype=np.uint8),
+            "block_scale": np.asarray(block_scale, dtype=np.float32),
+            "per_tensor": np.float32(per_tensor),
+        }
+        if bias is not None:
+            save_dict["bias"] = np.asarray(bias, dtype=np.float32)
+        np.savez_compressed(handle, **save_dict)
+    temporary.replace(path)
+
+
+def discover_shards(model_dir: Path) -> list[str]:
+    """Return safetensors shard filenames in the checkpoint directory."""
+    index_path = model_dir / INDEX_NAME
+    if index_path.is_file():
+        try:
+            payload = json.loads(index_path.read_text(encoding="utf-8"))
+            weight_map = payload.get("weight_map")
+            if isinstance(weight_map, dict) and weight_map:
+                names = sorted(set(weight_map.values()))
+                missing = [name for name in names if not (model_dir / name).is_file()]
+                if not missing:
+                    return names
+        except Exception:
+            pass
+
+    single = model_dir / "model.safetensors"
+    if single.is_file():
+        return ["model.safetensors"]
+
+    shards = sorted(path.name for path in model_dir.glob("*.safetensors") if path.is_file())
+    if not shards:
+        raise SystemExit(f"No safetensors shards found in {model_dir}")
+    return shards
+
+
+def _assign_output(args) -> None:
+    source = Path(args.model).resolve()
+    stem = source.name
+    passthrough = args.no_quant or args.outtype == "nvfp4"
+    if not args.output:
+        if passthrough and args.outtype == "auto":
+            directory = Path.cwd() / stem
+        elif passthrough:
+            label = "NVFP4" if args.outtype == "nvfp4" else args.outtype.upper()
+            directory = Path.cwd() / f"{stem}-{label}"
+        else:
+            directory = Path.cwd() / f"{stem}-NVFP4"
+    else:
+        directory = Path(args.output).resolve()
+    args.output = str(directory)
+
+
+def _apply_directory_profile(args, source: Path) -> None:
+    config_path = source / "config.json"
+    if not config_path.is_file():
+        return
+    try:
+        config = json.loads(config_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return
+    names: list[str] = []
+    architectures = config.get("architectures")
+    if isinstance(architectures, list):
+        names.extend(str(item) for item in architectures)
+    elif isinstance(architectures, str):
+        names.append(architectures)
+    model_type = config.get("model_type")
+    if isinstance(model_type, str) and model_type:
+        names.append(model_type)
+    for name in names:
+        profile = quant_profile(name)
+        if profile is None:
+            continue
+        if not args.exclude:
+            args.exclude = profile.skip
+            log.info("%s profile: leaving %s unquantized.", profile.arch, profile.label)
+        else:
+            log.info("%s profile: --exclude replaces default skip of %s.", profile.arch, profile.label)
+        return
+
+
+def _copy_non_weight_files(source_dir: Path, output_dir: Path) -> None:
+    """Copy configs, tokenizers, chat templates, and documentation into output."""
+    output_dir.mkdir(parents=True, exist_ok=True)
+    for path in source_dir.iterdir():
+        if path.is_dir() or path.name.startswith("."):
+            continue
+        if path.name == "config.json" or path.name == INDEX_NAME:
+            continue
+        if any(path.name.endswith(ext) for ext in NON_WEIGHT_PATTERNS):
+            continue
+        dest = output_dir / path.name
+        if not dest.exists() or dest.stat().st_mtime < path.stat().st_mtime:
+            shutil.copy2(path, dest)
+
+
+def _write_config_json(source_dir: Path, output_dir: Path, args, is_quantized: bool) -> None:
+    config_path = source_dir / "config.json"
+    if not config_path.is_file():
+        return
+    try:
+        config = json.loads(config_path.read_text(encoding="utf-8"))
+    except Exception as exc:
+        log.warning("Could not parse config.json: %s", exc)
+        return
+
+    fmt = getattr(args, "format", "hf")
+
+    def _build_ct_quant_config():
+        ignore = []
+        if not getattr(args, "quantize_output", False):
+            ignore.append("lm_head")
+        if getattr(args, "exclude", None):
+            ignore.append(args.exclude)
+        return {
+            "quant_method": "compressed-tensors",
+            "format": "nvfp4-pack-quantized",
+            "quantization_status": "compressed",
+            "ignore": ignore,
+            "config_groups": {
+                "group_0": {
+                    "weights": {
+                        "num_bits": 4,
+                        "type": "float",
+                        "strategy": "tensor_group",
+                        "group_size": 16,
+                        "symmetric": True,
+                        "dynamic": False,
+                        "actorder": None,
+                    },
+                    "input_activations": None,
+                    "targets": ["Linear"],
+                    "format": "nvfp4-pack-quantized",
+                }
+            },
+        }
+
+    if is_quantized:
+        if fmt == "hf":
+            config["quantization_config"] = _build_ct_quant_config()
+        else:
+            config["quantization_config"] = {
+                "quant_method": "modelopt",
+                "quant_algo": "NVFP4",
+                "format": "nvfp4-pack-quantized",
+                "group_size": 16,
+                "bits": 4,
+            }
+        config["torch_dtype"] = getattr(args, "output_dtype", "bfloat16")
+    elif args.no_quant:
+        if args.outtype in ("f32", "f16", "bf16"):
+            config.pop("quantization_config", None)
+            config["torch_dtype"] = "bfloat16" if args.outtype == "bf16" else ("float16" if args.outtype == "f16" else "float32")
+        elif args.outtype == "nvfp4":
+            if fmt == "hf":
+                config["quantization_config"] = _build_ct_quant_config()
+            else:
+                config.setdefault("quantization_config", {
+                    "quant_method": "modelopt",
+                    "quant_algo": "NVFP4",
+                    "format": "nvfp4-pack-quantized",
+                    "group_size": 16,
+                    "bits": 4,
+                })
+
+    if getattr(args, "context_length", None):
+        for key in ("max_position_embeddings", "seq_length", "max_sequence_length"):
+            if key in config:
+                config[key] = int(args.context_length)
+
+    out_file = output_dir / "config.json"
+    out_file.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
+
+
+def pack_nvfp4_hf(codes: np.ndarray, block_scales: np.ndarray, per_tensor: float):
+    """Pack codes and scales into NVFP4 tensors."""
+    # Packed weights: low nibble is element 0, high nibble is element 1
+    packed = (codes[:, 0::2] & 0x0F) | ((codes[:, 1::2] & 0x0F) << 4)
+    packed_tensor = torch.from_numpy(np.ascontiguousarray(packed, dtype=np.uint8))
+
+    scale_bits = ue4m3_bits(block_scales)
+    scale_tensor = torch.from_numpy(np.ascontiguousarray(scale_bits, dtype=np.uint8)).view(torch.float8_e4m3fn)
+    scale_2_tensor = torch.tensor(float(per_tensor), dtype=torch.float32)
+
+    return packed_tensor, scale_tensor, scale_2_tensor
+
+
+def _should_quantize(
+    key: str,
+    shape: tuple[int, ...],
+    args,
+    convert_to_quant_dir: Path | None,
+) -> bool:
+    if len(shape) != 2:
+        return False
+    if not (key.endswith(".weight") or key.endswith(".weight_packed")):
+        return False
+    if shape[-1] % 64 != 0 or shape[-1] < 64:
+        return False
+    if key in ("lm_head.weight", "lm_head.weight_packed") and not getattr(args, "quantize_output", False):
+        return False
+    lowered = key.lower()
+    if "embed_tokens" in lowered or "wte" in lowered or "word_embeddings" in lowered:
+        return False
+    if "norm" in lowered or "ln" in lowered:
+        return False
+    if getattr(args, "exclude_re", None) and args.exclude_re.search(key):
+        return False
+    if ctq_skips_key(key, convert_to_quant_dir):
+        return False
+    return True
+
+
+def _target_dtype(dtype_str: str) -> torch.dtype:
+    mapping = {
+        "float32": torch.float32,
+        "f32": torch.float32,
+        "float16": torch.float16,
+        "f16": torch.float16,
+        "bfloat16": torch.bfloat16,
+        "bf16": torch.bfloat16,
+    }
+    return mapping.get(dtype_str.lower(), torch.bfloat16)
+
+
+def export_directory(args) -> None:
+    source_dir = Path(args.model).resolve()
+    if not source_dir.is_dir():
+        raise FileNotFoundError(f"Model directory does not exist: {source_dir}")
+
+    _assign_output(args)
+    output_dir = Path(args.output).resolve()
+    if output_dir == source_dir:
+        raise SystemExit("Output directory cannot be the same as the input directory.")
+    if output_dir.exists() and any(output_dir.iterdir()) and not args.overwrite:
+        raise SystemExit(f"Output directory {output_dir} already exists and is not empty. Use --overwrite to replace.")
+
+    shards = discover_shards(source_dir)
+    _apply_directory_profile(args, source_dir)
+    args.exclude_re = re.compile(args.exclude) if args.exclude else None
+
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s")
+    log.info("Loading %s", source_dir)
+    log.info("Found %d safetensors shard(s)", len(shards))
+    log.info("Target format: %s", "Hugging Face (compressed-tensors)" if args.format == "hf" else "ModelOpt / ComfyUI")
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    _copy_non_weight_files(source_dir, output_dir)
+
+    is_passthrough = args.no_quant or args.outtype == "nvfp4"
+    float_dtype = _target_dtype(args.output_dtype if not is_passthrough else (args.outtype if args.outtype != "auto" else "bfloat16"))
+
+    if is_passthrough:
+        log.info("Quantization skipped. Converting tensors to %s", args.outtype)
+        weight_map: dict[str, str] = {}
+        total_size = 0
+        for shard_name in shards:
+            log.info("Processing %s", shard_name)
+            src_shard = source_dir / shard_name
+            out_shard = output_dir / shard_name
+            out_tensors: dict[str, torch.Tensor] = {}
+            with safe_open(src_shard, framework="pt", device="cpu") as handle:
+                for key in handle.keys():
+                    t = handle.get_tensor(key)
+                    if args.outtype in ("f32", "f16", "bf16") and t.is_floating_point():
+                        t = t.to(dtype=float_dtype)
+                    out_tensors[key] = t
+                    weight_map[key] = shard_name
+                    total_size += t.numel() * t.element_size()
+            save_file(out_tensors, out_shard)
+        _write_config_json(source_dir, output_dir, args, is_quantized=False)
+        if len(shards) > 1:
+            (output_dir / INDEX_NAME).write_text(
+                json.dumps({"metadata": {"total_size": total_size}, "weight_map": weight_map}, indent=2) + "\n",
+                encoding="utf-8",
+            )
+        log.info("Wrote %s", output_dir)
+        return
+
+    # Learned NVFP4 Quantization
+    ensure_cuda_interpreter(args.device)
+    args.optimizer = resolve_optimizer(args.optimizer)
+
+    # Prepare checkpoint directory for per-tensor resume
+    checkpoint_dir = Path(args.checkpoint_dir) if args.checkpoint_dir else output_dir / ".nvfp4-work"
+    if args.fresh and checkpoint_dir.exists():
+        shutil.rmtree(checkpoint_dir)
+    checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+    # Initialize quantizer
+    import gguf
+    quantizer = TensorQuantizer(args, gguf, load_learned_converter(args.convert_to_quant)[1])
+
+    # First pass: collect all tensor keys and shapes across shards
+    all_keys: dict[str, tuple[str, tuple[int, ...]]] = {}
+    for shard_name in shards:
+        with safe_open(source_dir / shard_name, framework="pt", device="cpu") as handle:
+            for key in handle.keys():
+                slice_header = handle.get_slice(key)
+                all_keys[key] = (shard_name, tuple(int(d) for d in slice_header.get_shape()))
+
+    quant_count = 0
+    passthrough_count = 0
+    bias_count = 0
+    weight_map: dict[str, str] = {}
+    total_size = 0
+    all_quant_meta: dict[str, dict] = {}
+
+    for s_idx, shard_name in enumerate(shards, start=1):
+        log.info("\n[%d/%d] Processing shard: %s", s_idx, len(shards), shard_name)
+        src_shard = source_dir / shard_name
+        out_shard = output_dir / shard_name
+        shard_tensors: dict[str, torch.Tensor] = {}
+
+        with safe_open(src_shard, framework="pt", device="cpu") as handle:
+            keys_in_shard = list(handle.keys())
+
+            for key in keys_in_shard:
+                tensor = handle.get_tensor(key)
+                shape = tuple(int(d) for d in tensor.shape)
+                base_key = key.removesuffix(".weight_packed").removesuffix(".weight")
+
+                if _should_quantize(key, shape, args, args.convert_to_quant):
+                    log.info("  Quantizing [%s] %s", "x".join(str(d) for d in shape), key)
+                    weight_np = tensor.detach().float().cpu().numpy()
+
+                    # Check for sibling bias
+                    bias_key = f"{base_key}.bias"
+                    has_bias = bias_key in all_keys and not args.no_bias_correction
+                    bias_shard = all_keys[bias_key][0] if has_bias else None
+
+                    # Check checkpoint cache
+                    cached = _load_checkpoint(checkpoint_dir, key, shape, need_bias=has_bias)
+                    if cached is not None:
+                        codes, block_scale, per_tensor, bias_corr = cached
+                        log.info("    Loaded from checkpoint: %s", key)
+                    else:
+                        rounds = getattr(args, "scale_refinement_rounds", DEFAULT_SCALE_REFINEMENT_ROUNDS)
+                        codes, block_scale, per_tensor, dequant = quantizer.quantize_matrix(weight_np, key)
+                        codes, block_scale = coordinate_descent_refine(
+                            weight_np, codes, block_scale, per_tensor, bake=False, rounds=rounds, device=quantizer.device,
+                        )
+                        codes, block_scale, per_tensor = sweep_per_tensor_scale(
+                            weight_np, codes, block_scale, per_tensor, bake=False, device=quantizer.device,
+                        )
+
+                        bias_corr = None
+                        if has_bias:
+                            bias_corr = quantizer.bias_correction(weight_np, dequant)
+
+                        _save_checkpoint(checkpoint_dir, key, shape, codes, block_scale, per_tensor, bias_corr)
+
+                    packed_w, scale_1, scale_2 = pack_nvfp4_hf(codes, block_scale, per_tensor)
+
+                    if args.format == "hf":
+                        # Standard Hugging Face compressed-tensors layout
+                        shard_tensors[f"{base_key}.weight_packed"] = packed_w
+                        shard_tensors[f"{base_key}.weight_scale"] = scale_1
+                        shard_tensors[f"{base_key}.weight_global_scale"] = torch.tensor(1.0 / float(per_tensor), dtype=torch.float32)
+                    else:
+                        # ModelOpt / ComfyUI layout
+                        shard_tensors[f"{base_key}.weight"] = packed_w
+                        shard_tensors[f"{base_key}.weight_scale"] = scale_1
+                        shard_tensors[f"{base_key}.weight_scale_2"] = scale_2
+                        shard_tensors[f"{base_key}.comfy_quant"] = torch.tensor([1], dtype=torch.uint8)
+
+                    quant_count += 1
+                    all_quant_meta[base_key] = {
+                        "quant_type": "nvfp4",
+                        "block_size": 16,
+                        "scale_type": "float8_e4m3fn",
+                    }
+
+                    # Bias correction application if bias is in the same shard
+                    if has_bias and bias_shard == shard_name and bias_corr is not None:
+                        bias_t = handle.get_tensor(bias_key).float()
+                        corrected_bias = bias_t - torch.from_numpy(bias_corr).to(bias_t.device)
+                        shard_tensors[bias_key] = corrected_bias.to(dtype=float_dtype)
+                        bias_count += 1
+                else:
+                    # Non-quantized tensors (or bias already processed)
+                    if key in shard_tensors:
+                        continue
+                    if tensor.is_floating_point():
+                        if tensor.ndim <= 1 and key.endswith("norm.weight"):
+                            tensor = tensor.float()
+                        else:
+                            tensor = tensor.to(dtype=float_dtype)
+                    shard_tensors[key] = tensor
+                    passthrough_count += 1
+
+        shard_metadata = {}
+        if args.format != "hf":
+            shard_metadata["_quantization_metadata"] = json.dumps({"format_version": "1.0", "layers": all_quant_meta}, separators=(",", ":"))
+
+        for k, t in shard_tensors.items():
+            weight_map[k] = shard_name
+            total_size += t.numel() * t.element_size()
+
+        save_file(shard_tensors, out_shard, metadata=shard_metadata if shard_metadata else None)
+
+    _write_config_json(source_dir, output_dir, args, is_quantized=True)
+
+    if len(shards) > 1:
+        index_payload = {
+            "metadata": {"total_size": total_size},
+            "weight_map": weight_map,
+        }
+        (output_dir / INDEX_NAME).write_text(json.dumps(index_payload, indent=2) + "\n", encoding="utf-8")
+
+    if not args.keep_checkpoints and quant_count:
+        shutil.rmtree(checkpoint_dir, ignore_errors=True)
+
+    log.info(
+        "\nWrote %s (%d NVFP4 tensors, %d biases adjusted, %d other tensors)",
+        output_dir,
+        quant_count,
+        bias_count,
+        passthrough_count,
+    )
+
+
+def self_test(args) -> None:
+    """Check that packed blocks roundtrip accurately with dequantizer."""
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    from nvfp4gguf import FP4_TABLE
+    import gguf
+
+    log.info("Running NVFP4 roundtrip self-test...")
+    _converter_cls, unpack_uint4 = load_learned_converter(args.convert_to_quant)
+    quantizer = TensorQuantizer(args, gguf, unpack_uint4)
+
+    generator = np.random.default_rng(0)
+    for shape in ((64, 128), (128, 256)):
+        weight = generator.normal(size=shape).astype(np.float32) * 0.05
+        codes, block_scale, per_tensor, dequant = quantizer.quantize_matrix(weight)
+        packed_w, scale_1, scale_2 = pack_nvfp4_hf(codes, block_scale, per_tensor)
+
+        # Dequantize packed tensor
+        w_u = packed_w.numpy()
+        c0 = w_u & 0x0F
+        c1 = (w_u >> 4) & 0x0F
+        unpacked_codes = np.empty(shape, dtype=np.uint8)
+        unpacked_codes[:, 0::2] = c0
+        unpacked_codes[:, 1::2] = c1
+        assert np.array_equal(codes, unpacked_codes), "Nibble unpacking mismatch!"
+
+        # Check values
+        fp4_vals = FP4_TABLE[unpacked_codes]
+        b_scales = block_scale.repeat(16, axis=1)
+        recon = fp4_vals * b_scales * per_tensor
+        max_diff = np.max(np.abs(recon - dequant))
+        assert max_diff < 1e-4, f"Reconstruction difference {max_diff} exceeded tolerance!"
+        log.info("Shape %s roundtrip ok (max diff %.3e)", shape, max_diff)
+
+    log.info("self-test passed successfully.")
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Quantize a Hugging Face model directory to NVFP4 Hugging Face format.",
+    )
+    parser.add_argument("model", nargs="?", help="Hugging Face model directory (config.json + safetensors)")
+    parser.add_argument("-o", "--output", help="Output directory. Default: ./<model-name>-NVFP4")
+    parser.add_argument("--format", default="hf", choices=("hf", "modelopt", "comfy"),
+                        help="Safetensors tensor layout: 'hf' (default) uses Hugging Face compressed-tensors (.weight_packed, .weight_scale, .weight_global_scale). 'modelopt' (or 'comfy') uses raw .weight.")
+    parser.add_argument("--overwrite", action="store_true", help="Replace existing files in the output directory")
+    parser.add_argument("--no-quant", action="store_true", help="Skip learned rounding. Rewrites or repacks tensors")
+    parser.add_argument("--outtype", default="auto", choices=("auto", "f32", "f16", "bf16", "nvfp4"),
+                        help="Float type for tensors when --no-quant is used (default: auto)")
+    parser.add_argument("--output-dtype", default="bfloat16", choices=("bfloat16", "float16"),
+                        help="Dtype for unquantized 2D weights and embeddings (default: bfloat16)")
+    parser.add_argument("--context-length", type=int, default=None,
+                        help="Override context length stored in config.json")
+    parser.add_argument("--model-name", default=None, help="Override model name")
+    parser.add_argument("--checkpoint-dir", type=Path, default=None, help="Per-tensor resume directory")
+    parser.add_argument("--fresh", action="store_true", help="Delete checkpoint directory before starting")
+    parser.add_argument("--keep-checkpoints", action="store_true", help="Keep per-tensor checkpoints upon completion")
+    parser.add_argument("--eager", action="store_true", help="Eager loading")
+    parser.add_argument("--low-memory", action="store_true", help="Stream each shard instead of loading all at once")
+    parser.add_argument("--heur", action="store_true", help="Skip layers heuristic rejects")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose logging")
+    parser.add_argument("--self-test", action="store_true", help="Run NVFP4 packing self-test and exit")
+    parser.add_argument("--llama-cpp", type=Path, default=DEFAULT_LLAMA_CPP,
+                        help="llama.cpp checkout directory (default: ~/llama.cpp)")
+
+    add_quant_arguments(parser)
+    args = parser.parse_args(argv)
+
+    if args.self_test:
+        if args.device == "auto":
+            args.device = "cpu"
+        return args
+
+    if not args.model:
+        parser.error("model directory is required")
+
+    if args.convert_to_quant is None and DEFAULT_CONVERT_TO_QUANT.is_dir():
+        args.convert_to_quant = DEFAULT_CONVERT_TO_QUANT
+
+    return args
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = parse_args(argv)
+    if args.self_test:
+        self_test(args)
+        return
+    export_directory(args)
+
+
+if __name__ == "__main__":
+    main()

man/man1/gguf_to_nvfp4_hf.1

@@ -0,0 +1,241 @@
+.TH GGUF_TO_NVFP4_HF 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
+.SH NAME
+gguf_to_nvfp4_hf \- convert and quantize GGUF models to NVFP4 Hugging Face format
+.SH SYNOPSIS
+.B gguf_to_nvfp4_hf
+[\fB\-o\fR \fIOUTPUT\fR]
+[\fB\-\-overwrite\fR]
+[\fB\-\-format\fR \fIFORMAT\fR]
+[\fB\-\-no\-quant\fR]
+[\fB\-\-outtype\fR \fITYPE\fR]
+[\fB\-\-output\-dtype\fR \fIDTYPE\fR]
+[\fB\-\-hf\-template\fR \fIDIR\fR]
+[\fB\-\-context\-length\fR \fILEN\fR]
+[\fB\-\-model\-name\fR \fINAME\fR]
+[\fB\-\-checkpoint\-dir\fR \fIDIR\fR]
+[\fB\-\-fresh\fR]
+[\fB\-\-keep\-checkpoints\fR]
+[\fB\-\-eager\fR]
+[\fB\-\-low\-memory\fR]
+[\fB\-\-heur\fR]
+[\fB\-\-verbose\fR]
+[\fB\-\-self\-test\fR]
+[\fB\-\-llama\-cpp\fR \fIPATH\fR]
+[\fB\-\-convert\-to\-quant\fR \fIPATH\fR]
+[\fB\-\-device\fR \fIDEVICE\fR]
+[\fB\-\-simple\fR]
+[\fB\-\-optimizer\fR \fIOPT\fR]
+[\fB\-\-num\-iter\fR \fIN\fR]
+[\fB\-\-lr\fR \fILR\fR]
+[\fB\-\-lr\-patience\fR \fIN\fR]
+[\fB\-\-lr\-min\fR \fIMIN\fR]
+[\fB\-\-lr\-schedule\fR \fISCHED\fR]
+[\fB\-\-scale\-optimization\fR \fIMODE\fR]
+[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR]
+[\fB\-\-full\-matrix\fR]
+[\fB\-\-top\-p\fR \fIP\fR]
+[\fB\-\-min\-k\fR \fIK\fR]
+[\fB\-\-max\-k\fR \fIK\fR]
+[\fB\-\-early\-stop\-loss\fR \fILOSS\fR]
+[\fB\-\-early\-stop\-stall\fR \fISTALL\fR]
+[\fB\-\-target\-loss\fR \fITARGET\fR]
+[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR]
+[\fB\-\-calib\-samples\fR \fISAMPLES\fR]
+[\fB\-\-seed\fR \fISEED\fR]
+[\fB\-\-no\-bias\-correction\fR]
+[\fB\-\-quantize\-output\fR]
+[\fB\-\-exclude\fR \fIREGEX\fR]
+[\fB\-h\fR|\fB\-\-help\fR]
+\fIgguf\fR
+.SH DESCRIPTION
+.B gguf_to_nvfp4_hf
+converts a GGUF model file into a standard Hugging Face model directory
+(\fIconfig.json\fR + safetensors) with NVIDIA FP4 (NVFP4) quantized weights.
+.PP
+GGUF tensor names are reverse-mapped to canonical Hugging Face conventions
+(e.g., \fIblk.0.attn_q.weight\fR maps to \fImodel.layers.0.self_attn.q_proj.weight\fR),
+and RoPE permutations for Q and K are inverted.
+.PP
+If the source GGUF contains float or non-NVFP4 weights, 2D weight matrices are quantized
+using convert_to_quant's learned rounding (or simple round-to-grid with \fB\-\-simple\fR).
+If the source GGUF already contains NVFP4 super-blocks, they are unpacked and repacked
+into ModelOpt-compatible safetensors without loss of precision.
+.PP
+If \fB\-\-no\-quant\fR is specified, the model is dequantized to standard float format
+(\fI--outtype f32, f16, bf16, or auto\fR).
+.PP
+An optional reference Hugging Face model directory can be provided with \fB\-\-hf\-template\fR
+to copy exact upstream tokenizer files, special token maps, and configs. If omitted,
+synthesized configuration and tokenizer files are extracted directly from the GGUF metadata.
+.SH POSITIONAL ARGUMENTS
+.TP
+.I gguf
+Path to the input GGUF model file.
+.SH OPTIONS
+.SS Output & File Control
+.TP
+.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT
+Output destination directory (default: \fI./<model-name>-NVFP4\fR, or \fI./<model-name>\fR with
+\fB\-\-no\-quant\fR and \fBauto\fR).
+.TP
+.B \-\-overwrite
+Overwrite existing files in the output directory without prompting.
+.TP
+.BI \-\-format " " {hf,modelopt,comfy}
+Safetensors layout format. \fBhf\fR (default) produces standard Hugging Face
+\fBcompressed-tensors\fR layout (\fI<name>.weight_packed\fR, \fI<name>.weight_scale\fR,
+and \fI<name>.weight_global_scale\fR) natively supported by Hugging Face transformers,
+vLLM, SGLang, and llama.cpp. \fBmodelopt\fR (or \fBcomfy\fR) produces raw tensors
+(\fI<name>.weight\fR, \fI<name>.weight_scale\fR, \fI<name>.weight_scale_2\fR, and \fI<name>.comfy_quant\fR).
+.TP
+.BI \-\-hf\-template " " DIR
+Reference Hugging Face model directory to copy config, tokenizer, and template files from.
+.TP
+.BI \-\-context\-length " " LEN
+Override the maximum sequence/context length recorded in \fIconfig.json\fR.
+.TP
+.BI \-\-model\-name " " NAME
+Override the model name.
+.SS Quantization Mode
+.TP
+.B \-\-no\-quant
+Skip learned rounding. Rewrites tensors as float weights.
+.TP
+.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4}
+Data type for tensors when \fB\-\-no\-quant\fR is used (default: \fBauto\fR).
+.TP
+.BI \-\-output\-dtype " " {bfloat16,float16}
+Data type used for unquantized 2D weights, layer norms, and embeddings (default: \fBbfloat16\fR).
+.TP
+.B \-\-simple
+Round weights directly onto the E2M1 grid without iterative learned rounding.
+.SS Optimization & Learning Rate
+.TP
+.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original}
+Optimizer for learned rounding. \fBauto\fR selects Prodigy if installed, otherwise AdamW (default: \fBauto\fR).
+.TP
+.BI \-\-num\-iter " " N
+Maximum optimization iterations per weight matrix (default: 4000).
+.TP
+.BI \-\-lr " " LR
+Initial learning rate (default: 1.0).
+.TP
+.BI \-\-lr\-patience " " N
+Patience steps before decaying the learning rate (default: 5).
+.TP
+.BI \-\-lr\-min " " MIN
+Minimum learning rate floor (default: 1e-8).
+.TP
+.BI \-\-lr\-schedule " " {plateau,exponential,adaptive}
+Learning rate decay schedule (default: \fBplateau\fR).
+.TP
+.BI \-\-scale\-optimization " " {fixed,iterative,joint}
+Scale factor optimization strategy. \fBjoint\fR optimizes block scales simultaneously with codes (default: \fBjoint\fR).
+.TP
+.BI \-\-scale\-refinement\-rounds " " N
+Number of scale refinement passes when using iterative scale optimization (default: 2).
+.SS SVD Low-Rank Approximation
+.TP
+.B \-\-full\-matrix
+Perform full SVD instead of randomized low-rank SVD.
+.TP
+.BI \-\-top\-p " " P
+Cumulative energy threshold for singular value truncation (default: 0.2).
+.TP
+.BI \-\-min\-k " " K
+Minimum number of singular vectors retained (default: 128).
+.TP
+.BI \-\-max\-k " " K
+Maximum number of singular vectors retained (default: 1280).
+.SS Early Stopping
+.TP
+.BI \-\-target\-loss " " TARGET
+Stop optimization once reconstruction loss reaches this threshold (default: 1e-5). Set 0 to disable.
+.TP
+.BI \-\-early\-stop\-stall " " STALL
+Stop after this many steps without loss improvement (default: 2000).
+.TP
+.BI \-\-early\-stop\-floor\-stall " " FLOOR
+Stop after this many steps without improvement once learning rate reaches its floor (default: 64).
+.TP
+.BI \-\-early\-stop\-loss " " LOSS
+Early stopping absolute loss threshold (default: 5e-9).
+.SS Calibration & Bias Correction
+.TP
+.BI \-\-calib\-samples " " SAMPLES
+Number of random normal calibration samples used for bias error compensation (default: 3072).
+.TP
+.BI \-\-seed " " SEED
+Random seed for calibration data generation (default: 42).
+.TP
+.B \-\-no\-bias\-correction
+Leave bias tensors unaltered without applying calibration-based error correction.
+.SS Layer Selection
+.TP
+.B \-\-quantize\-output
+Also quantize the final language model head (\fIlm_head.weight\fR). By default, \fIlm_head\fR remains unquantized.
+.TP
+.BI \-\-exclude " " REGEX ", " \-\-exclude\-layers " " REGEX
+Regular expression matching tensor names that should remain unquantized in float format.
+.SS Checkpoints & Resuming
+.TP
+.BI \-\-checkpoint\-dir " " DIR
+Directory to store and load intermediate per-tensor quantization checkpoints.
+.TP
+.B \-\-fresh
+Delete existing per-tensor checkpoints before starting.
+.TP
+.B \-\-keep\-checkpoints
+Retain per-tensor checkpoints upon successful completion.
+.SS Execution & Environment
+.TP
+.BI \-\-device " " DEVICE
+Compute device: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR).
+.TP
+.B \-\-eager
+Load tensors eagerly into memory.
+.TP
+.B \-\-low\-memory
+Stream tensors sequentially to minimize memory footprint.
+.TP
+.B \-\-heur
+Skip layers that convert_to_quant's heuristic rejects.
+.TP
+.BI \-\-convert\-to\-quant " " PATH
+Path to convert_to_quant source checkout if not installed in Python environment.
+.TP
+.BI \-\-llama\-cpp " " PATH
+Path to llama.cpp source checkout (default: \fI~/llama.cpp\fR).
+.SS Diagnostics
+.TP
+.B \-\-verbose
+Enable detailed progress and debugging output.
+.TP
+.B \-\-self\-test
+Run GGUF-to-HF unpack and roundtrip self-test and exit.
+.TP
+.B \-h\fR, \fB\-\-help
+Show command-line help and exit.
+.SH EXAMPLES
+.PP
+Convert and quantize a GGUF model to NVFP4 HF format:
+.PP
+.nf
+.RS
+gguf_to_nvfp4_hf -o ./meta-llama-3.1-8b-nvfp4 ./meta-llama-3.1-8b.gguf
+.RE
+.fi
+.PP
+Convert GGUF to HF using an upstream HF repo as template for tokenizers:
+.PP
+.nf
+.RS
+gguf_to_nvfp4_hf --hf-template ./upstream-hf-dir -o ./output-hf-dir ./model.gguf
+.RE
+.fi
+.SH SEE ALSO
+.BR hf_to_nvfp4_hf (1),
+.BR hf_to_nvfp4_gguf (1),
+.BR gguf_to_nvfp4_gguf (1),
+.BR safetensors_to_nvfp4_gguf (1)
+

man/man1/gguf_to_nvfp4_hf.py.1

@@ -0,0 +1 @@
+gguf_to_nvfp4_hf.1

man/man1/hf_to_nvfp4_hf.1

@@ -0,0 +1,231 @@
+.TH HF_TO_NVFP4_HF 1 "November 2026" "nvfp4_gguf_quant" "User Commands"
+.SH NAME
+hf_to_nvfp4_hf \- quantize Hugging Face models to NVFP4 Hugging Face format
+.SH SYNOPSIS
+.B hf_to_nvfp4_hf
+[\fB\-o\fR \fIOUTPUT\fR]
+[\fB\-\-overwrite\fR]
+[\fB\-\-format\fR \fIFORMAT\fR]
+[\fB\-\-no\-quant\fR]
+[\fB\-\-outtype\fR \fITYPE\fR]
+[\fB\-\-output\-dtype\fR \fIDTYPE\fR]
+[\fB\-\-context\-length\fR \fILEN\fR]
+[\fB\-\-model\-name\fR \fINAME\fR]
+[\fB\-\-checkpoint\-dir\fR \fIDIR\fR]
+[\fB\-\-fresh\fR]
+[\fB\-\-keep\-checkpoints\fR]
+[\fB\-\-eager\fR]
+[\fB\-\-low\-memory\fR]
+[\fB\-\-heur\fR]
+[\fB\-\-verbose\fR]
+[\fB\-\-self\-test\fR]
+[\fB\-\-llama\-cpp\fR \fIPATH\fR]
+[\fB\-\-convert\-to\-quant\fR \fIPATH\fR]
+[\fB\-\-device\fR \fIDEVICE\fR]
+[\fB\-\-simple\fR]
+[\fB\-\-optimizer\fR \fIOPT\fR]
+[\fB\-\-num\-iter\fR \fIN\fR]
+[\fB\-\-lr\fR \fILR\fR]
+[\fB\-\-lr\-patience\fR \fIN\fR]
+[\fB\-\-lr\-min\fR \fIMIN\fR]
+[\fB\-\-lr\-schedule\fR \fISCHED\fR]
+[\fB\-\-scale\-optimization\fR \fIMODE\fR]
+[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR]
+[\fB\-\-full\-matrix\fR]
+[\fB\-\-top\-p\fR \fIP\fR]
+[\fB\-\-min\-k\fR \fIK\fR]
+[\fB\-\-max\-k\fR \fIK\fR]
+[\fB\-\-early\-stop\-loss\fR \fILOSS\fR]
+[\fB\-\-early\-stop\-stall\fR \fISTALL\fR]
+[\fB\-\-target\-loss\fR \fITARGET\fR]
+[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR]
+[\fB\-\-calib\-samples\fR \fISAMPLES\fR]
+[\fB\-\-seed\fR \fISEED\fR]
+[\fB\-\-no\-bias\-correction\fR]
+[\fB\-\-quantize\-output\fR]
+[\fB\-\-exclude\fR \fIREGEX\fR]
+[\fB\-h\fR|\fB\-\-help\fR]
+\fImodel\fR
+.SH DESCRIPTION
+.B hf_to_nvfp4_hf
+quantizes a Hugging Face model directory (\fIconfig.json\fR + safetensors) to
+NVIDIA FP4 (NVFP4) format packaged into a standard Hugging Face model directory.
+.PP
+The output directory preserves all non-weight metadata, configuration, tokenizer files,
+vocabulary, chat templates, and generation configs. For each quantized weight matrix,
+the script writes ModelOpt-compatible packed 4-bit weights alongside per-block E4M3 scale
+factors (\fI<name>.weight_scale\fR) and per-tensor scale factors (\fI<name>.weight_scale_2\fR).
+.PP
+Weights are quantized using learned rounding (via \fBconvert_to_quant\fR) or simple nearest-grid
+projection onto the E2M1 floating point grid with two-level scaling. Biases are adjusted
+using simulated calibration data unless disabled. Checkpoint-based resumption is supported
+per tensor so interrupted runs can resume without recomputing.
+.SH POSITIONAL ARGUMENTS
+.TP
+.I model
+Path to a Hugging Face model directory containing \fIconfig.json\fR and one or more \fI.safetensors\fR files.
+.SH OPTIONS
+.SS Output & File Control
+.TP
+.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT
+Output destination directory (default: \fI./<model-name>-NVFP4\fR, or \fI./<model-name>\fR with
+\fB\-\-no\-quant\fR and \fBauto\fR).
+.TP
+.B \-\-overwrite
+Overwrite existing files in the output directory without prompting.
+.TP
+.BI \-\-format " " {hf,modelopt,comfy}
+Safetensors layout format. \fBhf\fR (default) produces standard Hugging Face
+\fBcompressed-tensors\fR layout (\fI<name>.weight_packed\fR, \fI<name>.weight_scale\fR,
+and \fI<name>.weight_global_scale\fR) natively supported by Hugging Face transformers,
+vLLM, SGLang, and llama.cpp. \fBmodelopt\fR (or \fBcomfy\fR) produces raw tensors
+(\fI<name>.weight\fR, \fI<name>.weight_scale\fR, \fI<name>.weight_scale_2\fR, and \fI<name>.comfy_quant\fR).
+.TP
+.BI \-\-context\-length " " LEN
+Override the maximum sequence/context length recorded in \fIconfig.json\fR.
+.TP
+.BI \-\-model\-name " " NAME
+Override the model name.
+.SS Quantization Mode
+.TP
+.B \-\-no\-quant
+Skip learned rounding. Rewrites or repacks tensors without quantizing.
+.TP
+.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4}
+Data type for non-quantized weights when learned rounding is skipped (default: \fBauto\fR).
+.TP
+.BI \-\-output\-dtype " " {bfloat16,float16}
+Data type used for unquantized 2D weights, layer norms, and embeddings (default: \fBbfloat16\fR).
+.TP
+.B \-\-simple
+Round weights directly onto the E2M1 grid without iterative learned rounding.
+.SS Optimization & Learning Rate
+.TP
+.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original}
+Optimizer for learned rounding. \fBauto\fR selects Prodigy if installed, otherwise AdamW (default: \fBauto\fR).
+.TP
+.BI \-\-num\-iter " " N
+Maximum optimization iterations per weight matrix (default: 4000).
+.TP
+.BI \-\-lr " " LR
+Initial learning rate (default: 1.0).
+.TP
+.BI \-\-lr\-patience " " N
+Patience steps before decaying the learning rate (default: 5).
+.TP
+.BI \-\-lr\-min " " MIN
+Minimum learning rate floor (default: 1e-8).
+.TP
+.BI \-\-lr\-schedule " " {plateau,exponential,adaptive}
+Learning rate decay schedule (default: \fBplateau\fR).
+.TP
+.BI \-\-scale\-optimization " " {fixed,iterative,joint}
+Scale factor optimization strategy. \fBjoint\fR optimizes block scales simultaneously with codes (default: \fBjoint\fR).
+.TP
+.BI \-\-scale\-refinement\-rounds " " N
+Number of scale refinement passes when using iterative scale optimization (default: 2).
+.SS SVD Low-Rank Approximation
+.TP
+.B \-\-full\-matrix
+Perform full SVD instead of randomized low-rank SVD.
+.TP
+.BI \-\-top\-p " " P
+Cumulative energy threshold for singular value truncation (default: 0.2).
+.TP
+.BI \-\-min\-k " " K
+Minimum number of singular vectors retained (default: 128).
+.TP
+.BI \-\-max\-k " " K
+Maximum number of singular vectors retained (default: 1280).
+.SS Early Stopping
+.TP
+.BI \-\-target\-loss " " TARGET
+Stop optimization once reconstruction loss reaches this threshold (default: 1e-5). Set 0 to disable.
+.TP
+.BI \-\-early\-stop\-stall " " STALL
+Stop after this many steps without loss improvement (default: 2000).
+.TP
+.BI \-\-early\-stop\-floor\-stall " " FLOOR
+Stop after this many steps without improvement once learning rate reaches its floor (default: 64).
+.TP
+.BI \-\-early\-stop\-loss " " LOSS
+Early stopping absolute loss threshold (default: 5e-9).
+.SS Calibration & Bias Correction
+.TP
+.BI \-\-calib\-samples " " SAMPLES
+Number of random normal calibration samples used for bias error compensation (default: 3072).
+.TP
+.BI \-\-seed " " SEED
+Random seed for calibration data generation (default: 42).
+.TP
+.B \-\-no\-bias\-correction
+Leave bias tensors unaltered without applying calibration-based error correction.
+.SS Layer Selection
+.TP
+.B \-\-quantize\-output
+Also quantize the final language model head (\fIlm_head.weight\fR). By default, \fIlm_head\fR remains unquantized.
+.TP
+.BI \-\-exclude " " REGEX ", " \-\-exclude\-layers " " REGEX
+Regular expression matching tensor names that should remain unquantized in float format.
+.SS Checkpoints & Resuming
+.TP
+.BI \-\-checkpoint\-dir " " DIR
+Directory to store and load intermediate per-tensor quantization checkpoints.
+.TP
+.B \-\-fresh
+Delete existing per-tensor checkpoints before starting.
+.TP
+.B \-\-keep\-checkpoints
+Retain per-tensor checkpoints upon successful completion.
+.SS Execution & Environment
+.TP
+.BI \-\-device " " DEVICE
+Compute device: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR).
+.TP
+.B \-\-eager
+Load tensors eagerly into memory.
+.TP
+.B \-\-low\-memory
+Stream shards sequentially to minimize memory footprint.
+.TP
+.B \-\-heur
+Skip layers that convert_to_quant's heuristic rejects.
+.TP
+.BI \-\-convert\-to\-quant " " PATH
+Path to convert_to_quant source checkout if not installed in Python environment.
+.TP
+.BI \-\-llama\-cpp " " PATH
+Path to llama.cpp source checkout if needed (default: \fI~/llama.cpp\fR).
+.SS Diagnostics
+.TP
+.B \-\-verbose
+Enable detailed progress and debugging output.
+.TP
+.B \-\-self\-test
+Run NVFP4 packing and roundtrip self-test and exit.
+.TP
+.B \-h\fR, \fB\-\-help
+Show command-line help and exit.
+.SH EXAMPLES
+.PP
+Quantize a Hugging Face model directory to NVFP4 HF format:
+.PP
+.nf
+.RS
+hf_to_nvfp4_hf -o ./meta-llama-3.1-8b-nvfp4 ./meta-llama-3.1-8b
+.RE
+.fi
+.PP
+Quantize with fast simple grid rounding without learned iterations:
+.PP
+.nf
+.RS
+hf_to_nvfp4_hf --simple -o ./my-model-nvfp4 ./my-model
+.RE
+.fi
+.SH SEE ALSO
+.BR hf_to_nvfp4_gguf (1),
+.BR gguf_to_nvfp4_hf (1),
+.BR hf_to_nvfp4_safetensors (1),
+.BR gguf_to_nvfp4_gguf (1)
+

man/man1/hf_to_nvfp4_hf.py.1

@@ -0,0 +1 @@
+hf_to_nvfp4_hf.1

nvfp4_quant

@@ -1,9 +1,6 @@
 #!/bin/bash
 TAG="${TAG:-latest}"
 CMD="$(basename "${0}")"
-if [ -z "${USER}" ]; then
-	USER="$(whoami)"
-fi
 
 IMAGE="${IMAGE:-zefie/nvfp4_quant:${TAG}}"
 COMMON=(docker run --user 1000:1000 --rm --gpus all -it -v ./:/workspace "${IMAGE}")