initial commit

Dockerfile

@@ -0,0 +1,20 @@
+FROM runpod/pytorch:1.5.0-cu1320-torch2130-ubuntu2404
+WORKDIR /root
+
+RUN mkdir llama.cpp
+COPY llama.cpp ./llama.cpp/
+RUN git clone https://github.com/silveroxides/convert_to_quant && cd convert_to_quant && pip install -r requirements.txt && pip install .
+
+RUN mkdir nvfp4_gguf_quant
+WORKDIR /root/nvfp4_gguf_quant
+COPY --exclude=__pycache__ nvfp4gguf ./nvfp4gguf
+COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py requirements.txt safetensors_to_nvfp4_gguf.py .
+
+RUN pip install -r requirements.txt
+COPY --chmod=0644 docker/README.txt /workspace/README.txt
+COPY --chmod=0755 docker/hf_to_nvfp4_gguf docker/hf_to_nvfp4_safetensors docker/gguf_to_nvfp4_gguf docker/safetensors_to_nvfp4_gguf /usr/local/bin/
+
+RUN apt remove -y python3-cryptography && pip install heretic-llm
+
+WORKDIR /workspace
+

docker/README.txt

@@ -0,0 +1,22 @@
+Convert-to-Quant (ctq) is installed for normal quantizations.
+
+For GGUF quantizations, try:
+
+hf_to_nvfp4_quant
+gguf_to_nvfp4_quant
+safetensors_to_nvfp4_quant
+hf_to_nvfp4_safetensors
+
+Depending on the source material.
+
+hf cli command is also installed, eg you can:
+
+
+hf download coder3101/gemma-4-12B-it-heretic --local-dir gemma-4-12B-it-heretic
+hf_to_nvfp4_gguf -o gemma-4-12B-it-heretic_nvfp4 gemma-4-12B-it-heretic
+
+The same directory can be quantized to one ComfyUI NVFP4 safetensors file:
+
+hf_to_nvfp4_safetensors -o gemma-4-12B-it-heretic-NVFP4.safetensors gemma-4-12B-it-heretic
+
+This would create a NVFP4 GGUF for Ollama/llama.cpp from https://huggingface.co/coder3101/gemma-4-12B-it-heretic

docker/gguf_to_nvfp4_gguf

@@ -0,0 +1,2 @@
+#!/bin/bash
+python /root/nvfp4_gguf_quant/gguf_to_nvfp4_gguf.py --llama-cpp /root/llama.cpp "${@}"

docker/hf_to_nvfp4_gguf

@@ -0,0 +1,2 @@
+#!/bin/bash
+python /root/nvfp4_gguf_quant/hf_to_nvfp4_gguf.py --llama-cpp /root/llama.cpp "${@}"

docker/hf_to_nvfp4_safetensors

@@ -0,0 +1,2 @@
+#!/bin/bash
+python /root/nvfp4_gguf_quant/hf_to_nvfp4_safetensors.py --convert-to-quant /root/convert_to_quant "${@}"

docker/safetensors_to_nvfp4_gguf

@@ -0,0 +1,2 @@
+#!/bin/bash
+python /root/nvfp4_gguf_quant/safetensors_to_nvfp4_gguf.py --llama-cpp /root/llama.cpp "${@}"

gguf_info.py

@@ -0,0 +1,382 @@
+#!/usr/bin/env python3
+"""Print a summary of a GGUF file from its header.
+
+The weight bytes are not read. Metadata and the tensor directory are enough
+for the architecture, quantization, tokenizer, and tensor list.
+"""
+
+from __future__ import annotations
+
+import argparse
+import struct
+from collections import Counter
+from pathlib import Path
+
+# GGML type id -> name, and (values per block, bytes per block).
+GGML_TYPES = {
+    0: ("F32", 1, 4),
+    1: ("F16", 1, 2),
+    2: ("Q4_0", 32, 18),
+    3: ("Q4_1", 32, 20),
+    6: ("Q5_0", 32, 22),
+    7: ("Q5_1", 32, 24),
+    8: ("Q8_0", 32, 34),
+    9: ("Q8_1", 32, 36),
+    10: ("Q2_K", 256, 84),
+    11: ("Q3_K", 256, 110),
+    12: ("Q4_K", 256, 144),
+    13: ("Q5_K", 256, 176),
+    14: ("Q6_K", 256, 210),
+    15: ("Q8_K", 256, 292),
+    16: ("IQ2_XXS", 256, 66),
+    17: ("IQ2_XS", 256, 74),
+    18: ("IQ3_XXS", 256, 98),
+    19: ("IQ1_S", 256, 50),
+    20: ("IQ4_NL", 32, 18),
+    21: ("IQ3_S", 256, 110),
+    22: ("IQ2_S", 256, 82),
+    23: ("IQ4_XS", 256, 136),
+    24: ("I8", 1, 1),
+    25: ("I16", 1, 2),
+    26: ("I32", 1, 4),
+    27: ("I64", 1, 8),
+    28: ("F64", 1, 8),
+    29: ("IQ1_M", 256, 56),
+    30: ("BF16", 1, 2),
+    34: ("TQ1_0", 256, 54),
+    35: ("TQ2_0", 256, 66),
+    39: ("MXFP4", 32, 17),
+    40: ("NVFP4", 64, 36),
+    41: ("Q1_0", 128, 18),
+    42: ("Q2_0", 64, 18),
+}
+
+FILE_TYPES = {
+    0: "ALL_F32",
+    1: "MOSTLY_F16",
+    2: "MOSTLY_Q4_0",
+    3: "MOSTLY_Q4_1",
+    7: "MOSTLY_Q8_0",
+    8: "MOSTLY_Q5_0",
+    9: "MOSTLY_Q5_1",
+    10: "MOSTLY_Q2_K",
+    11: "MOSTLY_Q3_K_S",
+    12: "MOSTLY_Q3_K_M",
+    13: "MOSTLY_Q3_K_L",
+    14: "MOSTLY_Q4_K_S",
+    15: "MOSTLY_Q4_K_M",
+    16: "MOSTLY_Q5_K_S",
+    17: "MOSTLY_Q5_K_M",
+    18: "MOSTLY_Q6_K",
+    19: "MOSTLY_IQ2_XXS",
+    20: "MOSTLY_IQ2_XS",
+    21: "MOSTLY_Q2_K_S",
+    22: "MOSTLY_IQ3_XS",
+    23: "MOSTLY_IQ3_XXS",
+    24: "MOSTLY_IQ1_S",
+    25: "MOSTLY_IQ4_NL",
+    26: "MOSTLY_IQ3_S",
+    27: "MOSTLY_IQ3_M",
+    28: "MOSTLY_IQ2_S",
+    29: "MOSTLY_IQ2_M",
+    30: "MOSTLY_IQ4_XS",
+    31: "MOSTLY_IQ1_M",
+    32: "MOSTLY_BF16",
+    36: "MOSTLY_TQ1_0",
+    37: "MOSTLY_TQ2_0",
+    38: "MOSTLY_MXFP4_MOE",
+    39: "MOSTLY_NVFP4",
+    40: "MOSTLY_Q1_0",
+    41: "MOSTLY_Q2_0",
+    1024: "GUESSED",
+}
+
+_SCALAR = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8}
+_INTEGER = {0: "<B", 1: "<b", 2: "<H", 3: "<h", 4: "<I", 5: "<i", 7: "<?", 10: "<Q", 11: "<q"}
+_FLOAT = {6: "<f", 12: "<d"}
+
+
+def format_size(num_bytes: int) -> str:
+    size = float(num_bytes)
+    for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+        if size < 1024 or unit == "TiB":
+            return f"{int(size)} B" if unit == "B" else f"{size:.2f} {unit}"
+        size /= 1024
+    return f"{size:.2f} TiB"
+
+
+def _read(handle, n: int) -> bytes:
+    data = handle.read(n)
+    if len(data) != n:
+        raise ValueError("truncated GGUF header")
+    return data
+
+
+def _read_string(handle) -> bytes:
+    size = struct.unpack("<Q", _read(handle, 8))[0]
+    if size > 64 * 1024 * 1024:
+        raise ValueError("GGUF string is too long")
+    return _read(handle, size)
+
+
+def _read_scalar(handle, value_type: int):
+    if value_type == 8:
+        return _read_string(handle).decode("utf-8", "replace")
+    if value_type not in _SCALAR:
+        raise ValueError(f"unknown GGUF value type {value_type}")
+    raw = _read(handle, _SCALAR[value_type])
+    if value_type in _INTEGER:
+        return struct.unpack(_INTEGER[value_type], raw)[0]
+    if value_type in _FLOAT:
+        return struct.unpack(_FLOAT[value_type], raw)[0]
+    return None
+
+
+def _read_array(handle, keep: bool):
+    subtype = struct.unpack("<I", _read(handle, 4))[0]
+    count = struct.unpack("<Q", _read(handle, 8))[0]
+    if subtype in _SCALAR and not keep:
+        handle.seek(_SCALAR[subtype] * count, 1)
+        return f"{_type_name(subtype)}[{count}]"
+    if subtype == 8 and not keep:
+        for _ in range(count):
+            _read_string(handle)
+        return f"string[{count}]"
+    values = []
+    limit = 8 if subtype == 8 else 16
+    for index in range(count):
+        if subtype == 8:
+            text = _read_string(handle).decode("utf-8", "replace")
+            if index < limit:
+                values.append(text)
+            continue
+        if subtype not in _SCALAR:
+            raise ValueError(f"unsupported GGUF array type {subtype}")
+        item = _read_scalar(handle, subtype)
+        if index < limit:
+            values.append(item)
+    if count > limit:
+        values.append(f"... {count} items")
+    return values
+
+
+def _type_name(value_type: int) -> str:
+    names = {
+        0: "u8", 1: "i8", 2: "u16", 3: "i16", 4: "u32", 5: "i32",
+        6: "f32", 7: "bool", 8: "string", 10: "u64", 11: "i64", 12: "f64",
+    }
+    return names.get(value_type, str(value_type))
+
+
+def load_gguf(path: Path) -> tuple[dict, list[dict], int]:
+    metadata: dict = {}
+    tensors: list[dict] = []
+    with path.open("rb") as handle:
+        magic, version = struct.unpack("<II", _read(handle, 8))
+        if magic != 0x46554747:
+            raise ValueError(f"{path} is not a GGUF file")
+        if version < 2:
+            raise ValueError(f"{path} uses GGUF version {version}, which this reader does not support")
+        tensor_count, kv_count = struct.unpack("<QQ", _read(handle, 16))
+        for _ in range(kv_count):
+            key = _read_string(handle).decode("utf-8", "replace")
+            value_type = struct.unpack("<I", _read(handle, 4))[0]
+            if value_type == 9:
+                keep = "token" not in key and "merges" not in key and not key.endswith(".scores")
+                metadata[key] = _read_array(handle, keep)
+            else:
+                metadata[key] = _read_scalar(handle, value_type)
+        for _ in range(tensor_count):
+            name = _read_string(handle).decode("utf-8", "replace")
+            n_dims = struct.unpack("<I", _read(handle, 4))[0]
+            dims = list(struct.unpack("<" + "Q" * n_dims, _read(handle, 8 * n_dims))) if n_dims else []
+            ggml_type = struct.unpack("<I", _read(handle, 4))[0]
+            offset = struct.unpack("<Q", _read(handle, 8))[0]
+            tensors.append({"name": name, "dims": dims, "type": ggml_type, "offset": offset})
+    return metadata, tensors, version
+
+
+def _meta(metadata: dict, suffix: str):
+    if suffix in metadata:
+        return metadata[suffix]
+    for key, value in metadata.items():
+        if key.endswith("." + suffix) and ".rope." not in key:
+            return value
+    return None
+
+
+def _context(metadata: dict) -> str:
+    for key, value in metadata.items():
+        if key.endswith(".context_length") and ".rope." not in key and isinstance(value, int) and value > 0:
+            return str(value)
+    return ""
+
+
+def _tensor_bytes(tensor: dict) -> int | None:
+    spec = GGML_TYPES.get(tensor["type"])
+    if spec is None or not tensor["dims"]:
+        return None
+    _name, block, type_size = spec
+    elements = 1
+    for dim in tensor["dims"]:
+        elements *= dim
+    if elements % block != 0:
+        return None
+    return elements // block * type_size
+
+
+def _preview(value, limit: int = 160) -> str:
+    if isinstance(value, str):
+        flat = value.replace("\n", "\\n")
+        if len(flat) > limit:
+            return f"{flat[:limit]}... ({len(value)} chars)"
+        return flat
+    if isinstance(value, list):
+        text = ", ".join(_preview(item, 40) for item in value[:12])
+        if len(value) > 12:
+            text += f", ... ({len(value)})"
+        return "[" + text + "]"
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, float):
+        return f"{value:.6g}"
+    return str(value)
+
+
+def _filetype(value) -> str:
+    if not isinstance(value, int):
+        return _preview(value)
+    name = FILE_TYPES.get(value)
+    return f"{name} ({value})" if name else str(value)
+
+
+def print_summary(path: Path, metadata: dict, tensors: list[dict], version: int) -> None:
+    arch = str(metadata.get("general.architecture") or "")
+    print(f"file          {path}")
+    print(f"size          {format_size(path.stat().st_size)}")
+    print(f"gguf          version {version}")
+    print(f"architecture  {arch or '-'}")
+    for label, key in (
+        ("name", "general.name"),
+        ("basename", "general.basename"),
+        ("size label", "general.size_label"),
+        ("finetune", "general.finetune"),
+        ("type", "general.type"),
+    ):
+        if key in metadata and metadata[key] not in ("", None):
+            print(f"{label:<14}{_preview(metadata[key])}")
+    if "general.file_type" in metadata:
+        print(f"filetype      {_filetype(metadata['general.file_type'])}")
+    if "general.quantization_version" in metadata:
+        print(f"quant version {metadata['general.quantization_version']}")
+    context = _context(metadata)
+    if context:
+        print(f"context       {context}")
+    fields = (
+        ("blocks", "block_count"),
+        ("embedding", "embedding_length"),
+        ("feed forward", "feed_forward_length"),
+        ("heads", "attention.head_count"),
+        ("kv heads", "attention.head_count_kv"),
+        ("head dim", "attention.key_length"),
+        ("experts", "expert_count"),
+        ("vocab", "vocab_size"),
+        ("sliding window", "attention.sliding_window"),
+    )
+    for label, suffix in fields:
+        value = _meta(metadata, suffix)
+        if isinstance(value, int) and value > 0:
+            print(f"{label:<14}{value}")
+    swa = _meta(metadata, "attention.sliding_window_pattern")
+    if isinstance(swa, list) and swa and all(isinstance(item, (bool, int)) for item in swa):
+        flags = [bool(item) for item in swa]
+        print(f"swa layers    {sum(flags)} sliding, {len(flags) - sum(flags)} full")
+    tok_model = metadata.get("tokenizer.ggml.model")
+    if tok_model:
+        print(f"tokenizer     {tok_model}")
+    for label, key in (
+        ("bos", "tokenizer.ggml.bos_token_id"),
+        ("eos", "tokenizer.ggml.eos_token_id"),
+    ):
+        if key in metadata:
+            print(f"{label:<14}{metadata[key]}")
+    template = metadata.get("tokenizer.chat_template")
+    if isinstance(template, str) and template:
+        print(f"chat template yes ({len(template)} chars)")
+    elif isinstance(template, list) and template:
+        print(f"chat template {len(template)} templates")
+    else:
+        print("chat template no")
+    if isinstance(metadata.get("general.description"), str):
+        print(f"description   {_preview(metadata['general.description'], 200)}")
+
+    counts: Counter[str] = Counter()
+    known_bytes = 0
+    unknown = 0
+    for tensor in tensors:
+        spec = GGML_TYPES.get(tensor["type"])
+        counts[spec[0] if spec else f"type {tensor['type']}"] += 1
+        nbytes = _tensor_bytes(tensor)
+        if nbytes is None:
+            unknown += 1
+        else:
+            known_bytes += nbytes
+    print(f"tensors       {len(tensors)}")
+    for name, count in counts.most_common():
+        print(f"  {name:<10} {count}")
+    print(f"tensor bytes  {format_size(known_bytes)}")
+    if unknown:
+        print(f"unmeasured    {unknown} tensors have an unknown type or shape")
+
+
+def print_metadata(metadata: dict) -> None:
+    print()
+    print("metadata")
+    width = max(len(key) for key in metadata) if metadata else 8
+    for key in metadata:
+        print(f"  {key:<{width}}  {_preview(metadata[key])}")
+
+
+def print_tensors(tensors: list[dict]) -> None:
+    print()
+    print(f"{'NAME':<48}  {'TYPE':<8}  {'SHAPE'}")
+    for tensor in tensors:
+        spec = GGML_TYPES.get(tensor["type"])
+        kind = spec[0] if spec else str(tensor["type"])
+        # File order: the first dimension is the contiguous one.
+        shape = " x ".join(str(dim) for dim in tensor["dims"]) or "-"
+        print(f"  {tensor['name']:<46}  {kind:<8}  {shape}")
+
+
+def inspect(path: Path, metadata_dump: bool, tensor_dump: bool) -> None:
+    if not path.is_file():
+        raise SystemExit(f"Not a file: {path}")
+    try:
+        metadata, tensors, version = load_gguf(path)
+    except (OSError, ValueError, struct.error, UnicodeError) as exc:
+        raise SystemExit(f"{path}: {exc}") from exc
+    print_summary(path, metadata, tensors, version)
+    if metadata_dump:
+        print_metadata(metadata)
+    if tensor_dump:
+        print_tensors(tensors)
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Print architecture, quantization, and tensor info from a GGUF header.")
+    parser.add_argument("gguf", nargs="+", type=Path, help="One or more .gguf files")
+    parser.add_argument("--metadata", action="store_true", help="Print every metadata key. Long strings and the vocabulary are shortened.")
+    parser.add_argument("--tensors", action="store_true", help="Print every tensor name, type, and shape.")
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = parse_args(argv)
+    for index, path in enumerate(args.gguf):
+        if index:
+            print()
+        inspect(path, args.metadata, args.tensors)
+
+
+if __name__ == "__main__":
+    main()

gguf_to_nvfp4_gguf.py

@@ -0,0 +1,667 @@
+#!/usr/bin/env python3
+"""Quantize a GGUF to llama.cpp NVFP4, or repack one that is already NVFP4.
+
+The learned-rounding flags match hf_to_nvfp4_gguf.py. Vocabulary, tensor names,
+and architecture metadata are copied from the source GGUF. --outtype nvfp4
+copies an existing NVFP4 file without fitting it again. --no-quant with
+f32, f16, bf16, or auto rewrites the tensors as float, or leaves them as stored.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import logging
+import re
+import shutil
+from pathlib import Path
+
+import numpy as np
+
+from nvfp4gguf import (
+    CTQ_NUM_ITER,
+    DEFAULT_CONVERT_TO_QUANT,
+    DEFAULT_LLAMA_CPP,
+    DEFAULT_EARLY_STOP_FLOOR_STALL,
+    DEFAULT_TARGET_LOSS,
+    DEFAULT_EARLY_STOP_STALL,
+    DEFAULT_SCALE_OPTIMIZATION,
+    DEFAULT_SCALE_REFINEMENT_ROUNDS,
+    QUANT_VERSION,
+    TensorQuantizer,
+    bind_exclude,
+    eligible_weight,
+    ensure_cuda_interpreter,
+    import_gguf,
+    load_learned_converter,
+    must_bake_nvfp4_scale,
+    mtp_layer_tensor,
+    bake_packed_nvfp4,
+    preserve_scale_tensor,
+    quantize_and_pack,
+    report_reconstruction,
+    require_quant_settings,
+    resolve_optimizer,
+    scale_bake_notice,
+)
+from nvfp4gguf.bundle import (
+    flatten_chat_template,
+    normalize_chat_template,
+    is_model_context_key,
+    outtype as _outtype,
+    refuse_existing_bundle,
+    require_context_length,
+    scalar_context_length,
+    warn_missing_gguf_metadata,
+    write_simple_modelfile,
+)
+
+log = logging.getLogger("gguf_to_nvfp4_gguf")
+
+
+def _checkpoint_path(directory: Path, name: str) -> Path:
+    digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:16]
+    safe = re.sub(r"[^A-Za-z0-9._-]+", "_", name)[:80]
+    return directory / f"{safe}.{digest}.npz"
+
+
+def _pythonize(value):
+    if isinstance(value, np.generic):
+        return value.item()
+    if isinstance(value, np.ndarray):
+        return value.tolist()
+    if isinstance(value, list):
+        return [_pythonize(item) for item in value]
+    return value
+
+
+def _text(value) -> str:
+    value = _pythonize(value)
+    if isinstance(value, bytes):
+        return value.decode("utf-8", "replace")
+    if isinstance(value, list):
+        return "".join(_text(item) for item in value)
+    return str(value)
+
+
+def _logical_shape(tensor) -> tuple[int, ...]:
+    return tuple(int(dim) for dim in tensor.shape[::-1])
+
+
+def _to_float(tensor, gguf_module) -> np.ndarray:
+    qtype = tensor.tensor_type
+    if qtype == gguf_module.GGMLQuantizationType.F32:
+        values = np.asarray(tensor.data, dtype=np.float32)
+    elif qtype == gguf_module.GGMLQuantizationType.F16:
+        values = np.asarray(tensor.data, dtype=np.float32)
+    else:
+        values = np.asarray(gguf_module.quants.dequantize(tensor.data, qtype), dtype=np.float32)
+    # The reader maps the file read-only. Torch refuses a non-writable view.
+    return np.array(values, dtype=np.float32, copy=True)
+
+
+def _open_reader(gguf, path: Path):
+    if not path.is_file():
+        raise SystemExit(f"GGUF file does not exist: {path}")
+    reader = gguf.GGUFReader(path)
+    if reader.byte_order != "I":
+        raise SystemExit(f"{path} is not little-endian GGUF")
+    return reader
+
+
+def _architecture(reader) -> str:
+    field = reader.fields.get("general.architecture")
+    if field is None:
+        return "unknown"
+    return _text(field.contents())
+
+
+def _file_type_name(reader, gguf_module) -> str:
+    field = reader.fields.get("general.file_type")
+    if field is None:
+        return ""
+    try:
+        return gguf_module.LlamaFileType(int(field.contents())).name
+    except (TypeError, ValueError):
+        return str(field.contents())
+
+
+def _is_nvfp4(reader, gguf_module) -> bool:
+    if _file_type_name(reader, gguf_module) == "MOSTLY_NVFP4":
+        return True
+    return any(tensor.tensor_type == gguf_module.GGMLQuantizationType.NVFP4 for tensor in reader.tensors)
+
+
+def _copy_metadata(reader, writer, gguf_module, file_type: int | None, model_name: str | None, context_length: int | None) -> None:
+    requested = require_context_length(context_length)
+    source_context = None
+    wrote_context = False
+    for field in reader.fields.values():
+        if field.name.startswith("GGUF.") or field.name == "general.architecture" or not field.types:
+            continue
+        value = _pythonize(field.contents())
+        if value is None:
+            continue
+        value_type = field.types[0]
+        if value_type == gguf_module.GGUFValueType.ARRAY and not value:
+            continue
+        if field.name == "general.file_type" and file_type is not None:
+            value = int(file_type)
+        if field.name == "general.name" and model_name:
+            value = model_name
+        if field.name == "general.alignment":
+            writer.data_alignment = int(value)
+        subtype = field.types[-1] if value_type == gguf_module.GGUFValueType.ARRAY else None
+        if field.name == "tokenizer.chat_template" or field.name.startswith("tokenizer.chat_template."):
+            value = normalize_chat_template(value)
+        if is_model_context_key(field.name):
+            source_context = scalar_context_length(value)
+            if requested is not None:
+                value = requested
+                value_type = gguf_module.GGUFValueType.UINT32
+                subtype = None
+                wrote_context = True
+        writer.add_key_value(field.name, value, value_type, sub_type=subtype)
+    if file_type is not None and "general.file_type" not in reader.fields:
+        writer.add_file_type(int(file_type))
+    if model_name and "general.name" not in reader.fields:
+        writer.add_name(model_name)
+    if requested is not None and not wrote_context:
+        writer.add_context_length(requested)
+    if requested is not None:
+        log.info("context length %d", requested)
+    elif source_context is not None:
+        log.info("context length %d from the source GGUF", source_context)
+
+
+def _replaced_scale(name: str, quant_names: set[str], architecture: str) -> bool:
+    if not (name.endswith(".scale") or name.endswith(".input_scale")):
+        return False
+    stem = name.rsplit(".", 1)[0]
+    weight = stem + ".weight"
+    if weight not in quant_names:
+        return False
+    return not (name.endswith(".scale") and preserve_scale_tensor(architecture, weight))
+
+
+def _keep_f32(name: str, rank: int) -> bool:
+    return rank < 2 or "norm" in name.lower()
+
+
+def _add_float(writer, gguf_module, name: str, values: np.ndarray, outtype: str) -> None:
+    values = np.ascontiguousarray(values, dtype=np.float32)
+    if outtype == "f16":
+        writer.add_tensor(name, values.astype(np.float16))
+        return
+    if outtype == "bf16":
+        raw = gguf_module.quants.quantize(values, gguf_module.GGMLQuantizationType.BF16)
+        writer.add_tensor(name, raw, raw_dtype=gguf_module.GGMLQuantizationType.BF16)
+        return
+    writer.add_tensor(name, values)
+
+
+def _copy_tensor(writer, tensor) -> None:
+    writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor.tensor_type)
+
+
+def _load_checkpoint(directory: Path, name: str, shape: tuple[int, ...]):
+    path = _checkpoint_path(directory, name)
+    if not path.is_file():
+        return None
+    with np.load(path) as blob:
+        if int(blob["version"]) != QUANT_VERSION:
+            return None
+        if tuple(int(v) for v in blob["shape"]) != tuple(shape):
+            return None
+        raw = np.array(blob["raw"], copy=True)
+        scale = np.array(blob["scale"], copy=True)
+        bias = np.array(blob["bias"], copy=True)
+    return raw, None if scale.size == 0 else scale, None if bias.size == 0 else bias
+
+
+def _save_checkpoint(directory: Path, name: str, shape: tuple[int, ...], raw, scale, bias) -> None:
+    path = _checkpoint_path(directory, name)
+    temporary = path.with_name(path.name + ".tmp")
+    with temporary.open("wb") as handle:
+        np.savez_compressed(
+            handle,
+            version=np.int32(QUANT_VERSION),
+            shape=np.asarray(shape, dtype=np.int64),
+            raw=raw,
+            scale=np.zeros((0,), np.float32) if scale is None else np.asarray(scale, dtype=np.float32),
+            bias=np.zeros((0,), np.float32) if bias is None else np.asarray(bias, dtype=np.float32),
+        )
+    temporary.replace(path)
+
+
+def _assign_output(args) -> None:
+    source = Path(args.gguf)
+    stem = source.stem
+    passthrough = args.no_quant or args.outtype == "nvfp4"
+    if args.output is None:
+        if passthrough and args.outtype == "auto":
+            directory = Path.cwd() / stem
+            gguf_name = f"{stem}.gguf"
+        elif passthrough:
+            label = "NVFP4" if args.outtype == "nvfp4" else args.outtype.upper()
+            directory = Path.cwd() / f"{stem}-{label}"
+            gguf_name = f"{directory.name}.gguf"
+        else:
+            directory = Path.cwd() / f"{stem}-NVFP4"
+            gguf_name = f"{directory.name}.gguf"
+    else:
+        output = Path(args.output)
+        if output.suffix == ".gguf":
+            output.parent.mkdir(parents=True, exist_ok=True)
+            args.output = str(output.resolve())
+            return
+        directory = output
+        gguf_name = f"{directory.name}.gguf"
+    directory = directory.resolve()
+    directory.mkdir(parents=True, exist_ok=True)
+    args.output = str(directory / gguf_name)
+
+
+def _refuse_same_file(args) -> None:
+    if Path(args.output).resolve() == Path(args.gguf).resolve():
+        raise SystemExit("Output GGUF is the same file as the input. Choose a different --output.")
+
+
+def _target_file_type(gguf_module, outtype: str, source_name: str):
+    if outtype == "nvfp4":
+        return gguf_module.LlamaFileType.MOSTLY_NVFP4
+    if outtype == "f32":
+        return gguf_module.LlamaFileType.ALL_F32
+    if outtype == "f16":
+        return gguf_module.LlamaFileType.MOSTLY_F16
+    if outtype == "bf16":
+        return gguf_module.LlamaFileType.MOSTLY_BF16
+    if source_name == "ALL_F32":
+        return gguf_module.LlamaFileType.ALL_F32
+    if source_name == "MOSTLY_F16":
+        return gguf_module.LlamaFileType.MOSTLY_F16
+    if source_name == "MOSTLY_BF16":
+        return gguf_module.LlamaFileType.MOSTLY_BF16
+    if source_name == "MOSTLY_NVFP4":
+        return gguf_module.LlamaFileType.MOSTLY_NVFP4
+    return None
+
+
+def _chat_template(reader) -> str:
+    parts = []
+    for name, field in reader.fields.items():
+        if name == "tokenizer.chat_template" or name.startswith("tokenizer.chat_template."):
+            parts.append(flatten_chat_template(field.contents()))
+    return "\n".join(part for part in parts if part)
+
+
+def _kv_int(reader, key: str) -> int:
+    field = reader.fields.get(key)
+    if field is None:
+        return 0
+    try:
+        return int(field.contents())
+    except (TypeError, ValueError):
+        return 0
+
+
+def _mtp_counts(reader, architecture: str) -> tuple[int, int]:
+    return _kv_int(reader, f"{architecture}.block_count"), _kv_int(reader, f"{architecture}.nextn_predict_layers")
+
+
+def _folded_scales(reader, gguf_module, architecture: str, n_blocks: int, n_nextn: int) -> dict[str, float]:
+    """Per-tensor NVFP4 scales the llama.cpp graph never multiplies."""
+    by_name = {tensor.name: tensor for tensor in reader.tensors}
+    nvfp4 = gguf_module.GGMLQuantizationType.NVFP4
+    scales: dict[str, float] = {}
+    for tensor in reader.tensors:
+        if not tensor.name.endswith(".scale"):
+            continue
+        weight_name = tensor.name[: -len(".scale")] + ".weight"
+        weight = by_name.get(weight_name)
+        if weight is None or weight.tensor_type != nvfp4:
+            continue
+        if preserve_scale_tensor(architecture, weight_name):
+            continue
+        bias_name = weight_name[: -len(".weight")] + ".bias"
+        fold = mtp_layer_tensor(weight_name, n_blocks, n_nextn) or must_bake_nvfp4_scale(
+            architecture, weight_name, bias_name in by_name
+        )
+        if not fold:
+            continue
+        values = np.asarray(tensor.data, dtype=np.float32).reshape(-1)
+        if values.size != 1:
+            log.warning(
+                "Leaving %s in place. Baking expects one per-tensor scale, got %d values.",
+                tensor.name,
+                int(values.size),
+            )
+            continue
+        scales[tensor.name] = float(values[0])
+    return scales
+
+
+def export_copy(args) -> None:
+    """Rewrite the GGUF without learned rounding."""
+    gguf = import_gguf(args.llama_cpp)
+    source = Path(args.gguf).resolve()
+    output = Path(args.output).resolve()
+    reader = _open_reader(gguf, source)
+    if args.outtype == "nvfp4" and not _is_nvfp4(reader, gguf):
+        raise SystemExit(
+            f"{source} is not an NVFP4 GGUF. "
+            "Use --outtype f16, bf16, f32, or auto to write float weights, "
+            "or omit --outtype nvfp4 to fit NVFP4 from these weights."
+        )
+    architecture = _architecture(reader)
+    source_type = _file_type_name(reader, gguf)
+    file_type = _target_file_type(gguf, args.outtype, source_type)
+    if args.outtype == "nvfp4":
+        log.info("Copying existing NVFP4 tensors. Learned rounding is skipped.")
+    elif args.outtype == "auto":
+        log.info("Copying tensors as stored (%s). Learned rounding is skipped.", source_type or "unknown file type")
+    else:
+        log.info("Writing float GGUF (%s). Learned rounding is skipped.", args.outtype)
+    writer = gguf.GGUFWriter(output, architecture, use_temp_file=not args.no_temp_file)
+    _copy_metadata(reader, writer, gguf, None if file_type is None else int(file_type), args.model_name, args.context_length)
+    n_blocks, n_nextn = _mtp_counts(reader, architecture)
+    folded = _folded_scales(reader, gguf, architecture, n_blocks, n_nextn) if args.outtype in ("auto", "nvfp4") else {}
+    if folded:
+        log.info(
+            "Folding %d scale tensors into their NVFP4 blocks. "
+            "llama.cpp does not claim those sidecars.",
+            len(folded),
+        )
+    for tensor in reader.tensors:
+        if tensor.name in folded:
+            continue
+        scale_name = tensor.name[: -len(".weight")] + ".scale" if tensor.name.endswith(".weight") else ""
+        if scale_name in folded and tensor.tensor_type == gguf.GGMLQuantizationType.NVFP4:
+            baked = bake_packed_nvfp4(np.array(tensor.data, copy=True), folded[scale_name])
+            writer.add_tensor(tensor.name, baked, raw_dtype=gguf.GGMLQuantizationType.NVFP4)
+            continue
+        if args.outtype in ("auto", "nvfp4"):
+            _copy_tensor(writer, tensor)
+            continue
+        if _keep_f32(tensor.name, len(_logical_shape(tensor))):
+            if tensor.tensor_type == gguf.GGMLQuantizationType.F32:
+                _copy_tensor(writer, tensor)
+            else:
+                _add_float(writer, gguf, tensor.name, _to_float(tensor, gguf), "f32")
+            continue
+        _add_float(writer, gguf, tensor.name, _to_float(tensor, gguf), args.outtype)
+    log.info("Writing %s", output)
+    warn_missing_gguf_metadata(writer)
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file(progress=True)
+    writer.close()
+    write_simple_modelfile(output, args.overwrite, args.mmproj, _chat_template(reader))
+    log.info("Wrote %s", output)
+
+
+def _emit_bias(writer, gguf_module, tensor, correction: np.ndarray | None) -> bool:
+    if correction is None:
+        _copy_tensor(writer, tensor)
+        return False
+    values = _to_float(tensor, gguf_module)
+    flat = values.reshape(-1)
+    if flat.shape != correction.shape:
+        log.warning("    bias %s shape %s does not match correction %s", tensor.name, values.shape, correction.shape)
+        _copy_tensor(writer, tensor)
+        return False
+    values = (flat - correction).reshape(values.shape)
+    log.info("    bias %s adjusted (mean %+.3e)", tensor.name, float(correction.mean()))
+    if tensor.tensor_type == gguf_module.GGMLQuantizationType.F16:
+        writer.add_tensor(tensor.name, np.ascontiguousarray(values, dtype=np.float32).astype(np.float16))
+        return True
+    if tensor.tensor_type == gguf_module.GGMLQuantizationType.BF16:
+        raw = gguf_module.quants.quantize(
+            np.ascontiguousarray(values, dtype=np.float32),
+            gguf_module.GGMLQuantizationType.BF16,
+        )
+        writer.add_tensor(tensor.name, raw, raw_dtype=gguf_module.GGMLQuantizationType.BF16)
+        return True
+    writer.add_tensor(tensor.name, np.ascontiguousarray(values, dtype=np.float32))
+    return True
+
+
+def export_quant(args) -> None:
+    gguf = import_gguf(args.llama_cpp)
+    source = Path(args.gguf).resolve()
+    output = Path(args.output).resolve()
+    ensure_cuda_interpreter(args.device)
+    reader = _open_reader(gguf, source)
+    architecture = _architecture(reader)
+    bind_exclude(args, architecture)
+    notice = scale_bake_notice(architecture)
+    if notice:
+        log.warning(notice)
+    file_type_name = _file_type_name(reader, gguf)
+    if file_type_name not in {"ALL_F32", "MOSTLY_F16", "MOSTLY_BF16"}:
+        log.warning(
+            "Source file type is %s. NVFP4 will be fit to the dequantized weights, "
+            "so detail already discarded by the source quant stays discarded.",
+            file_type_name or "quantized",
+        )
+    names = {tensor.name for tensor in reader.tensors}
+    n_blocks, n_nextn = _mtp_counts(reader, architecture)
+    quant_names = [
+        tensor.name
+        for tensor in reader.tensors
+        if eligible_weight(tensor.name, _logical_shape(tensor), args, gguf)
+    ]
+    quant_name_set = set(quant_names)
+    log.info("%d matrices queued for NVFP4, %d tensors copied", len(quant_names), len(reader.tensors) - len(quant_names))
+    if not quant_names:
+        log.warning("No matrices are eligible for NVFP4. Tensors are copied and the file type stays unchanged.")
+
+    work = Path(args.checkpoint_dir) if args.checkpoint_dir else Path(str(output) + ".nvfp4-work")
+    if args.fresh and work.exists():
+        shutil.rmtree(work)
+    work.mkdir(parents=True, exist_ok=True)
+    require_quant_settings(work, args)
+
+    _converter_cls, unpack_uint4 = load_learned_converter(args.convert_to_quant)
+    del _converter_cls
+    quantizer = TensorQuantizer(args, gguf, unpack_uint4)
+    writer = gguf.GGUFWriter(output, architecture, use_temp_file=not args.no_temp_file)
+    _copy_metadata(
+        reader, writer, gguf,
+        int(gguf.LlamaFileType.MOSTLY_NVFP4) if quant_names else None,
+        args.model_name,
+        args.context_length,
+    )
+
+    pending: dict[str, object] = {}
+    corrections: dict[str, np.ndarray] = {}
+    done = 0
+    baked = 0
+    corrected = 0
+    try:
+        for tensor in reader.tensors:
+            if _replaced_scale(tensor.name, quant_name_set, architecture):
+                continue
+            weight_name = tensor.name[: -len(".bias")] + ".weight" if tensor.name.endswith(".bias") else ""
+            if weight_name in quant_name_set and not args.no_bias_correction:
+                if tensor.name not in corrections:
+                    pending[tensor.name] = tensor
+                    continue
+                if _emit_bias(writer, gguf, tensor, corrections[tensor.name]):
+                    corrected += 1
+                continue
+            if tensor.name not in quant_name_set:
+                _copy_tensor(writer, tensor)
+                continue
+            done += 1
+            logical = _logical_shape(tensor)
+            bias_name = tensor.name[: -len(".weight")] + ".bias"
+            bake = must_bake_nvfp4_scale(architecture, tensor.name, bias_name in names) or mtp_layer_tensor(
+                tensor.name, n_blocks, n_nextn
+            )
+            cached = _load_checkpoint(work, tensor.name, logical)
+            if cached is not None and (cached[1] is None) != bake:
+                log.info("    checkpoint scale layout does not match %s; recomputing", tensor.name)
+                cached = None
+            log.info(
+                "NVFP4 [%d/%d] %s  %s  from %s%s",
+                done, len(quant_names), tensor.name, logical, tensor.tensor_type.name,
+                "  (checkpoint)" if cached else "",
+            )
+            if cached is None:
+                weight = _to_float(tensor, gguf)
+                raw, scale = quantize_and_pack(quantizer, weight, bake, tensor.name)
+                report_reconstruction(quantizer, weight, raw, scale)
+                bias = None
+                if weight.ndim == 2 and not args.no_bias_correction and bias_name in names:
+                    per_tensor = None if scale is None else float(np.reshape(scale, -1)[0])
+                    runtime = quantizer.runtime_dequant(raw if raw.ndim == 2 else raw[0], per_tensor)
+                    sample = weight if weight.ndim == 2 else weight[0]
+                    bias = quantizer.bias_correction(sample, runtime)
+                _save_checkpoint(work, tensor.name, logical, raw, scale, bias)
+                del weight
+            else:
+                raw, scale, bias = cached
+            if bias is not None:
+                corrections[bias_name] = bias
+                pending_tensor = pending.pop(bias_name, None)
+                if pending_tensor is not None and _emit_bias(writer, gguf, pending_tensor, bias):
+                    corrected += 1
+            if bake and scale is None:
+                baked += 1
+            writer.add_tensor(tensor.name, raw, raw_dtype=gguf.GGMLQuantizationType.NVFP4)
+            if scale is not None:
+                writer.add_tensor(tensor.name[: -len(".weight")] + ".scale", np.ascontiguousarray(scale, dtype=np.float32))
+            del raw
+        for name, tensor in list(pending.items()):
+            if _emit_bias(writer, gguf, tensor, corrections.get(name)):
+                corrected += 1
+        pending.clear()
+    except KeyboardInterrupt:
+        log.warning("Interrupted. Checkpoints are in %s; rerun the same command to resume.", work)
+        raise
+
+    log.info("Writing %s", output)
+    warn_missing_gguf_metadata(writer)
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file(progress=True)
+    writer.close()
+    if not args.keep_checkpoints:
+        shutil.rmtree(work, ignore_errors=True)
+    write_simple_modelfile(output, args.overwrite, args.mmproj, _chat_template(reader))
+    log.info(
+        "Wrote %s  (%d NVFP4 tensors, %d with scale folded into the blocks, %d biases adjusted)",
+        output, done, baked, corrected,
+    )
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Quantize a GGUF to NVFP4, or repack one that is already NVFP4, with the same options as hf_to_nvfp4_gguf.py.",
+    )
+    parser.add_argument("gguf", nargs="?", help="Input GGUF")
+    parser.add_argument("-o", "--output", help="Output directory. Default: ./<name>-NVFP4, or ./<name> with --no-quant and auto. A path ending in .gguf is the GGUF file itself.")
+    parser.add_argument("--modelfile-only", action="store_true", help="Write the Modelfile and skip quantization and GGUF creation")
+    parser.add_argument("--overwrite", action="store_true", help="Replace an existing GGUF, Modelfile, or copied projector")
+    parser.add_argument("--mmproj", action="append", type=Path, default=None,
+                        help="Vision projector GGUF. Copied into the output directory under its own name and added as another FROM line. Repeat for more than one projector.")
+    parser.add_argument("--context-length", type=int, default=None,
+                        help="Max context stored in the GGUF. Replaces a source value, or sets one when the source has no single context length.")
+    parser.add_argument("--no-quant", action="store_true",
+                        help="Skip learned rounding. Writes a float GGUF, or copies an existing NVFP4 GGUF with --outtype nvfp4")
+    parser.add_argument("--outtype", default="auto", type=_outtype, choices=("auto", "f32", "f16", "bf16", "nvfp4"),
+                        help="Type used when learned rounding is skipped. nvfp4 copies a GGUF that is already NVFP4. auto leaves tensors as stored (default: auto)")
+    parser.add_argument("--llama-cpp", type=Path, default=DEFAULT_LLAMA_CPP,
+                        help="llama.cpp checkout used for the GGUF reader and writer (default: ~/llama.cpp)")
+    parser.add_argument("--convert-to-quant", type=Path, default=None,
+                        help="convert_to_quant checkout, if it is not installed in the environment")
+    parser.add_argument("--model-name", default=None, help="Override the model name stored in the GGUF")
+    parser.add_argument("--device", default="auto", help="auto, cpu, cuda, or cuda:N")
+    parser.add_argument("--simple", action="store_true", help="Round each value onto the E2M1 grid without learned rounding")
+    parser.add_argument("--optimizer", default="auto", choices=("auto", "prodigy", "adamw", "radam", "original"),
+                        help="Learned-rounding optimizer. auto uses prodigy when it is installed, otherwise AdamW")
+    parser.add_argument("--num-iter", type=int, default=CTQ_NUM_ITER, help="Learned-rounding iterations per matrix (convert_to_quant default: 4000)")
+    parser.add_argument("--lr", type=float, default=1.0, help="Initial learning rate")
+    parser.add_argument("--lr-min", type=float, default=1e-8, help="Lowest learning rate. The flat-loss check starts here (default: 1e-8).")
+    parser.add_argument("--lr-schedule", default="plateau", choices=("plateau", "exponential", "adaptive"))
+    parser.add_argument(
+        "--scale-optimization", default=DEFAULT_SCALE_OPTIMIZATION, choices=("fixed", "iterative", "joint"),
+        help="fixed keeps the initial scale. iterative refreshes it on the original optimizer. "
+             "joint trains the block scales with the codes (default: joint)",
+    )
+    parser.add_argument(
+        "--scale-refinement-rounds", type=int, default=DEFAULT_SCALE_REFINEMENT_ROUNDS,
+        help="Scale refreshes when --scale-optimization is iterative (default: 2). Ignored for fixed and joint.",
+    )
+    parser.add_argument("--full-matrix", action="store_true", help="Use a full SVD instead of a low-rank SVD")
+    parser.add_argument("--top-p", type=float, default=0.2)
+    parser.add_argument("--min-k", type=int, default=128)
+    parser.add_argument("--max-k", type=int, default=1280)
+    parser.add_argument("--early-stop-loss", type=float, default=5e-9)
+    parser.add_argument(
+        "--early-stop-stall", type=int, default=DEFAULT_EARLY_STOP_STALL,
+        help="Stop after this many steps without a new best, at any learning rate (default: 2000)",
+    )
+    parser.add_argument(
+        "--target-loss", type=float, default=DEFAULT_TARGET_LOSS,
+        help="Stop when the best loss reaches this value, at any learning rate (default: 1e-5). 0 disables it.",
+    )
+    parser.add_argument(
+        "--early-stop-floor-stall", type=int, default=DEFAULT_EARLY_STOP_FLOOR_STALL,
+        help="If the loss never reaches --target-loss, stop after this many steps without a new best once it has otherwise flattened. 0 disables it (default: 64)",
+    )
+    parser.add_argument("--calib-samples", type=int, default=3072, help="Random inputs used to correct a sibling bias")
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--no-bias-correction", action="store_true")
+    parser.add_argument("--quantize-output", action="store_true", help="Also quantize the output (lm_head) matrix")
+    parser.add_argument("--exclude", default=None, help="Regex of GGUF tensor names to leave in their source type")
+    parser.add_argument("--checkpoint-dir", type=Path, default=None, help="Per-tensor resume directory")
+    parser.add_argument("--fresh", action="store_true", help="Delete the checkpoint directory before starting")
+    parser.add_argument("--keep-checkpoints", action="store_true")
+    parser.add_argument("--eager", action="store_true", help="Accepted for the same command line as hf_to_nvfp4_gguf. The GGUF reader maps the file either way.")
+    parser.add_argument("--no-temp-file", action="store_true", help="Hold the finished GGUF tensors in RAM instead of a temp file")
+    parser.add_argument("--verbose", action="store_true")
+    parser.add_argument("--self-test", action="store_true", help="Check NVFP4 packing against llama.cpp's dequantizer and exit")
+    args = parser.parse_args(argv)
+    if args.self_test:
+        if args.device == "auto":
+            args.device = "cpu"
+        return args
+    if not args.gguf:
+        parser.error("GGUF file is required")
+    _assign_output(args)
+    _refuse_same_file(args)
+    if not args.modelfile_only:
+        refuse_existing_bundle(Path(args.output), args.overwrite, args.mmproj)
+    else:
+        refuse_existing_bundle(Path(args.output), args.overwrite, args.mmproj, skip_gguf=True)
+    args.exclude_re = re.compile(args.exclude) if args.exclude else None
+    if args.convert_to_quant is None and DEFAULT_CONVERT_TO_QUANT.is_dir():
+        args.convert_to_quant = DEFAULT_CONVERT_TO_QUANT
+    return args
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = parse_args(argv)
+    if args.self_test:
+        from hf_to_nvfp4_gguf import self_test
+        self_test(args)
+        return
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s")
+    if args.modelfile_only:
+        output = Path(args.output)
+        output.parent.mkdir(parents=True, exist_ok=True)
+        log.info("Skipping quantization and GGUF creation; writing Modelfile only")
+        reader = _open_reader(import_gguf(args.llama_cpp), Path(args.gguf))
+        write_simple_modelfile(output, args.overwrite, args.mmproj, _chat_template(reader))
+        return
+    if args.no_quant or args.outtype == "nvfp4":
+        export_copy(args)
+        return
+    args.optimizer = resolve_optimizer(args.optimizer)
+    export_quant(args)
+
+
+if __name__ == "__main__":
+    main()

hf_to_nvfp4_safetensors.py

@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+"""Quantize a Hugging Face directory to one ComfyUI NVFP4 safetensors file.
+
+convert_to_quant reads one safetensors file and writes one file with packed
+NVFP4 weights, ``weight_scale`` / ``weight_scale_2``, ``.comfy_quant``
+tensors, and a ``_quantization_metadata`` header. A checkpoint is usually
+sharded. This script runs that same conversion on every shard, then merges
+the results into a single ``.safetensors`` file.
+
+A bias is corrected only when it lives in the same shard as its weight,
+which is how convert_to_quant sees it. Norms, embeddings, ``lm_head``, and
+the other names on convert_to_quant's exclusion list stay in their original
+dtype.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+import struct
+from pathlib import Path
+
+from nvfp4gguf import (
+    CTQ_NUM_ITER,
+    DEFAULT_CONVERT_TO_QUANT,
+    DEFAULT_EARLY_STOP_STALL,
+    DEFAULT_SCALE_OPTIMIZATION,
+    DEFAULT_SCALE_REFINEMENT_ROUNDS,
+    load_learned_converter,
+    quant_profile,
+    resolve_optimizer,
+)
+
+INDEX_NAME = "model.safetensors.index.json"
+# safetensors aligns each tensor to its dtype width, not to 8.
+_DTYPE_ALIGN = {
+    "BOOL": 1, "U8": 1, "I8": 1,
+    "F8_E4M3": 1, "F8_E5M2": 1, "F8_E8M0": 1,
+    "F16": 2, "BF16": 2, "I16": 2, "U16": 2,
+    "F32": 4, "I32": 4, "U32": 4,
+    "F64": 8, "I64": 8, "U64": 8,
+}
+
+
+def discover_shards(model_dir: Path) -> list[str]:
+    """Return safetensors shard filenames in the checkpoint directory."""
+    index_path = model_dir / INDEX_NAME
+    if index_path.is_file():
+        payload = json.loads(index_path.read_text(encoding="utf-8"))
+        weight_map = payload.get("weight_map")
+        if not isinstance(weight_map, dict) or not weight_map:
+            raise SystemExit(f"{index_path} has no weight_map")
+        names = sorted(set(weight_map.values()))
+        missing = [name for name in names if not (model_dir / name).is_file()]
+        if missing:
+            raise SystemExit(f"Index entries are missing from {model_dir}: {', '.join(missing)}")
+        return names
+
+    single = model_dir / "model.safetensors"
+    if single.is_file():
+        return ["model.safetensors"]
+
+    shards = sorted(path.name for path in model_dir.glob("*.safetensors") if path.is_file())
+    if not shards:
+        raise SystemExit(f"No safetensors shards in {model_dir}")
+    return shards
+
+
+def output_file(args, source: Path) -> Path:
+    if not args.output:
+        return (Path.cwd() / f"{source.name}-NVFP4.safetensors").resolve()
+    output = Path(args.output)
+    if output.exists() and output.is_dir():
+        return (output / f"{source.name}-NVFP4.safetensors").resolve()
+    if output.suffix != ".safetensors":
+        output = output.with_name(output.name + ".safetensors")
+    return output.resolve()
+
+
+def read_header(path: Path) -> tuple[dict, int]:
+    with path.open("rb") as handle:
+        raw_size = handle.read(8)
+        if len(raw_size) != 8:
+            raise SystemExit(f"{path} is not a safetensors file")
+        header_size = struct.unpack("<Q", raw_size)[0]
+        header = json.loads(handle.read(header_size))
+    if not isinstance(header, dict):
+        raise SystemExit(f"{path} has an unreadable safetensors header")
+    return header, header_size
+
+
+def merge_safetensors(paths: list[Path], dest: Path) -> None:
+    """Concatenate quantized shards into one safetensors file.
+
+    Tensor bytes are copied, not loaded. ``_quantization_metadata`` layers
+    from each shard are combined so ComfyUI still sees every quantized weight.
+    """
+    metadata: dict[str, str] = {}
+    layers: dict = {}
+    pieces: list[tuple[str, dict, Path, int, int]] = []
+    owners: dict[str, str] = {}
+    for path in paths:
+        header, header_size = read_header(path)
+        file_meta = dict(header.pop("__metadata__", {}) or {})
+        raw_quant = file_meta.pop("_quantization_metadata", None)
+        if raw_quant:
+            parsed = json.loads(raw_quant)
+            file_layers = parsed.get("layers", {})
+            overlap = sorted(set(file_layers) & set(layers))
+            if overlap:
+                raise SystemExit(f"Quant metadata for {overlap[0]!r} appears in more than one shard")
+            layers.update(file_layers)
+        for key, value in file_meta.items():
+            metadata.setdefault(key, value)
+        data_base = 8 + header_size
+        for key, entry in header.items():
+            if key in owners:
+                raise SystemExit(f"Tensor {key!r} appears in both {owners[key]} and {path.name}")
+            owners[key] = path.name
+            start, end = entry["data_offsets"]
+            pieces.append((key, entry, path, data_base + int(start), int(end) - int(start)))
+
+    if layers:
+        metadata["_quantization_metadata"] = json.dumps(
+            {"format_version": "1.0", "layers": layers},
+            separators=(",", ":"),
+        )
+
+    # safetensors rejects gaps. Grouping by descending dtype width keeps every
+    # tensor aligned without padding, which is how save_file lays a file out.
+    ordered = sorted(
+        enumerate(pieces),
+        key=lambda item: (-_DTYPE_ALIGN.get(item[1][1].get("dtype", ""), 8), item[0]),
+    )
+    cursor = 0
+    new_header: dict = {}
+    if metadata:
+        new_header["__metadata__"] = metadata
+    layout: list[tuple[Path, int, int]] = []
+    for _index, (key, entry, path, abs_start, length) in ordered:
+        align = _DTYPE_ALIGN.get(entry.get("dtype", ""), 8)
+        if cursor % align != 0:
+            raise SystemExit(f"Cannot align {key} ({entry.get('dtype')}) at offset {cursor}")
+        info = {name: value for name, value in entry.items() if name != "data_offsets"}
+        info["data_offsets"] = [cursor, cursor + length]
+        new_header[key] = info
+        layout.append((path, abs_start, length))
+        cursor += length
+
+    header_bytes = json.dumps(new_header, separators=(",", ":")).encode("utf-8")
+    header_bytes += b" " * ((8 - (len(header_bytes) % 8)) % 8)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    temporary = dest.with_name(dest.name + ".tmp")
+    with temporary.open("wb") as out:
+        out.write(struct.pack("<Q", len(header_bytes)))
+        out.write(header_bytes)
+        for path, abs_start, length in layout:
+            with path.open("rb") as src:
+                src.seek(abs_start)
+                remaining = length
+                while remaining:
+                    chunk = src.read(min(remaining, 8 * 1024 * 1024))
+                    if not chunk:
+                        raise SystemExit(f"Short read from {path}")
+                    out.write(chunk)
+                    remaining -= len(chunk)
+    temporary.replace(dest)
+
+
+def quantize_shard(convert_to_nvfp4, args, optimizer: str, source: Path, destination: Path) -> None:
+    if destination.exists():
+        destination.unlink()
+    convert_to_nvfp4(
+        str(source),
+        str(destination),
+        exclude_layers=args.exclude_layers,
+        output_dtype=args.output_dtype,
+        simple=args.simple,
+        num_iter=args.num_iter,
+        heur=args.heur,
+        calib_samples=args.calib_samples,
+        seed=args.seed,
+        optimizer=optimizer,
+        lr=args.lr,
+        lr_schedule=args.lr_schedule,
+        top_p=args.top_p,
+        min_k=args.min_k,
+        max_k=args.max_k,
+        full_matrix=args.full_matrix,
+        early_stop_loss=args.early_stop_loss,
+        early_stop_stall=args.early_stop_stall,
+        scale_refinement_rounds=args.scale_refinement_rounds,
+        scale_optimization=args.scale_optimization,
+        low_memory=args.low_memory,
+        use_speed=args.use_speed,
+        verbose_output=args.verbose,
+    )
+    if not destination.is_file() or destination.stat().st_size == 0:
+        raise SystemExit(f"convert_to_quant did not write {destination}")
+
+
+def _apply_directory_profile(args, source: Path) -> None:
+    config_path = source / "config.json"
+    if not config_path.is_file():
+        return
+    try:
+        config = json.loads(config_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return
+    names: list[str] = []
+    architectures = config.get("architectures")
+    if isinstance(architectures, list):
+        names.extend(str(item) for item in architectures)
+    elif isinstance(architectures, str):
+        names.append(architectures)
+    model_type = config.get("model_type")
+    if isinstance(model_type, str) and model_type:
+        names.append(model_type)
+    for name in names:
+        profile = quant_profile(name)
+        if profile is None:
+            continue
+        if not args.exclude_layers:
+            args.exclude_layers = profile.skip
+            print(f"{profile.arch} profile: leaving {profile.label} unquantized.")
+        else:
+            print(f"{profile.arch} profile: --exclude-layers replaces the default skip of {profile.label}.")
+        return
+
+
+def convert_directory(args) -> None:
+    source = Path(args.model).resolve()
+    _apply_directory_profile(args, source)
+    if not source.is_dir():
+        raise SystemExit(f"Not a directory: {source}")
+    output = output_file(args, source)
+    if output.is_relative_to(source):
+        raise SystemExit("Write the safetensors file outside the input directory.")
+    if output.exists() and not args.overwrite:
+        raise SystemExit(f"{output} already exists. Pass --overwrite to replace it.")
+    shards = discover_shards(source)
+
+    load_learned_converter(args.convert_to_quant)
+    from convert_to_quant.formats.nvfp4_conversion import convert_to_nvfp4
+    from convert_to_quant.utils.logging import setup_logging
+
+    # Without this, convert_to_quant drops the "(13/256) Processing tensor" lines
+    # and only the inner 4000-step bar is visible.
+    setup_logging("VERBOSE" if args.verbose else "NORMAL")
+
+    optimizer = resolve_optimizer(args.optimizer)
+    print(f"Quantizing {len(shards)} shard(s) from {source}")
+    print(f"Output file: {output}")
+    if len(shards) == 1:
+        quantize_shard(convert_to_nvfp4, args, optimizer, source / shards[0], output)
+        print(f"Wrote {output}")
+        return
+
+    work = output.with_name(output.name + ".shards")
+    if work.exists():
+        shutil.rmtree(work)
+    work.mkdir(parents=True)
+    try:
+        written: list[Path] = []
+        for index, name in enumerate(shards, start=1):
+            destination = work / name
+            print(f"\n[{index}/{len(shards)}] {name}")
+            quantize_shard(convert_to_nvfp4, args, optimizer, source / name, destination)
+            written.append(destination)
+        print(f"\nMerging {len(written)} shards into {output.name}")
+        merge_safetensors(written, output)
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+    print(f"Wrote {output}")
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Quantize a Hugging Face directory to one ComfyUI NVFP4 safetensors file.",
+    )
+    parser.add_argument("model", nargs="?", help="Hugging Face model directory (config.json + safetensors)")
+    parser.add_argument("-o", "--output", help="Output .safetensors file. Default: ./<model-name>-NVFP4.safetensors")
+    parser.add_argument("--overwrite", action="store_true", help="Replace an existing output file")
+    parser.add_argument("--convert-to-quant", type=Path, default=None,
+                        help="convert_to_quant checkout, if it is not installed")
+    parser.add_argument("--simple", action="store_true", help="Round onto the E2M1 grid without learned rounding")
+    parser.add_argument("--optimizer", default="auto", choices=("auto", "prodigy", "adamw", "radam", "original"),
+                        help="auto uses prodigy when it is installed, otherwise AdamW")
+    parser.add_argument("--num-iter", type=int, default=CTQ_NUM_ITER,
+                        help="Learned-rounding iterations per matrix (convert_to_quant default: 4000)")
+    parser.add_argument("--lr", type=float, default=1.0, help="Initial learning rate")
+    parser.add_argument("--lr-min", type=float, default=1e-8, help="Lowest learning rate. The flat-loss check starts here (default: 1e-8).")
+    parser.add_argument("--lr-schedule", default="plateau", choices=("plateau", "exponential", "adaptive"))
+    parser.add_argument(
+        "--scale-optimization", default=DEFAULT_SCALE_OPTIMIZATION, choices=("fixed", "iterative", "joint"),
+        help="fixed keeps the initial scale. iterative refreshes it on the original optimizer. "
+             "joint trains the block scales with the codes (default: joint)",
+    )
+    parser.add_argument(
+        "--scale-refinement-rounds", type=int, default=DEFAULT_SCALE_REFINEMENT_ROUNDS,
+        help="Scale refreshes when --scale-optimization is iterative (default: 2). Ignored for fixed and joint.",
+    )
+    parser.add_argument("--full-matrix", action="store_true", help="Use a full SVD instead of a low-rank SVD")
+    parser.add_argument("--top-p", type=float, default=0.2)
+    parser.add_argument("--min-k", type=int, default=128)
+    parser.add_argument("--max-k", type=int, default=1280)
+    parser.add_argument("--early-stop-loss", type=float, default=5e-9)
+    parser.add_argument(
+        "--early-stop-stall", type=int, default=DEFAULT_EARLY_STOP_STALL,
+        help="Stop after this many steps without progress. At the learning-rate floor, progress means a 1% loss drop (default: 2000)",
+    )
+    parser.add_argument("--calib-samples", type=int, default=3072)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--use-speed", action="store_true", help="Enable Prodigy's use_speed")
+    parser.add_argument("--low-memory", action="store_true", help="Stream each shard instead of loading it all at once")
+    parser.add_argument("--heur", action="store_true", help="Skip layers convert_to_quant's size heuristic rejects")
+    parser.add_argument("--exclude-layers", default=None, help="Regex of tensor names to leave unquantized")
+    parser.add_argument("--output-dtype", default="bfloat16", choices=("bfloat16", "float16"),
+                        help="Dtype recorded for quantized layers and used for unquantized 2D weights")
+    parser.add_argument("--verbose", action="store_true")
+    args = parser.parse_args(argv)
+    if not args.model:
+        parser.error("model directory is required")
+    if args.convert_to_quant is None and DEFAULT_CONVERT_TO_QUANT.is_dir():
+        args.convert_to_quant = DEFAULT_CONVERT_TO_QUANT
+    return args
+
+
+def main(argv: list[str] | None = None) -> None:
+    convert_directory(parse_args(argv))
+
+
+if __name__ == "__main__":
+    main()

nvfp4gguf/__init__.py

@@ -0,0 +1,75 @@
+"""Shared NVFP4 quantization used by the GGUF scripts in this project."""
+
+from .quant import (
+    CTQ_NUM_ITER,
+    DEFAULT_CONVERT_TO_QUANT,
+    DEFAULT_LLAMA_CPP,
+    DEFAULT_EARLY_STOP_FLOOR_STALL,
+    DEFAULT_TARGET_LOSS,
+    DEFAULT_EARLY_STOP_STALL,
+    DEFAULT_SCALE_OPTIMIZATION,
+    DEFAULT_SCALE_REFINEMENT_ROUNDS,
+    QUANT_VERSION,
+    ShapeView,
+    TensorQuantizer,
+    add_quant_arguments,
+    bind_exclude,
+    ctq_skips_key,
+    eligible_weight,
+    ensure_cuda_interpreter,
+    import_gguf,
+    load_gguf_modules,
+    load_learned_converter,
+    looks_like_spark,
+    check_scale_policy,
+    must_bake_nvfp4_scale,
+    mtp_layer_tensor,
+    bake_packed_nvfp4,
+    pack_nvfp4,
+    preserve_scale_tensor,
+    quant_profile,
+    quant_settings,
+    quantize_and_pack,
+    report_reconstruction,
+    require_quant_settings,
+    resolve_optimizer,
+    scale_bake_notice,
+    ue4m3_bits,
+)
+
+__all__ = [
+    "CTQ_NUM_ITER",
+    "DEFAULT_CONVERT_TO_QUANT",
+    "DEFAULT_LLAMA_CPP",
+    "DEFAULT_EARLY_STOP_FLOOR_STALL",
+    "DEFAULT_TARGET_LOSS",
+    "DEFAULT_EARLY_STOP_STALL",
+    "DEFAULT_SCALE_OPTIMIZATION",
+    "DEFAULT_SCALE_REFINEMENT_ROUNDS",
+    "QUANT_VERSION",
+    "ShapeView",
+    "TensorQuantizer",
+    "add_quant_arguments",
+    "bind_exclude",
+    "ctq_skips_key",
+    "eligible_weight",
+    "ensure_cuda_interpreter",
+    "import_gguf",
+    "load_gguf_modules",
+    "load_learned_converter",
+    "looks_like_spark",
+    "check_scale_policy",
+    "must_bake_nvfp4_scale",
+    "mtp_layer_tensor",
+    "bake_packed_nvfp4",
+    "pack_nvfp4",
+    "preserve_scale_tensor",
+    "quant_profile",
+    "quant_settings",
+    "quantize_and_pack",
+    "report_reconstruction",
+    "require_quant_settings",
+    "resolve_optimizer",
+    "scale_bake_notice",
+    "ue4m3_bits",
+]

nvfp4gguf/bundle.py

@@ -0,0 +1,340 @@
+"""Output bundle helpers shared by the GGUF conversion scripts."""
+
+from __future__ import annotations
+
+import logging
+import shutil
+from pathlib import Path
+
+log = logging.getLogger("nvfp4gguf.bundle")
+
+CONTEXT_HPARAMS = (
+    "max_position_embeddings",
+    "n_ctx",
+    "n_positions",
+    "max_length",
+    "max_sequence_length",
+    "model_max_length",
+)
+
+
+def is_model_context_key(name: str) -> bool:
+    """The model's max context, not RoPE's original context length."""
+    return name.endswith(".context_length") and ".rope." not in name
+
+
+def require_context_length(length: int | None) -> int | None:
+    if length is None:
+        return None
+    if length <= 0:
+        raise SystemExit("--context-length must be a positive token count")
+    return int(length)
+
+
+def scalar_context_length(value) -> int | None:
+    if isinstance(value, bool) or isinstance(value, (list, dict)):
+        return None
+    try:
+        length = int(value)
+    except (TypeError, ValueError):
+        return None
+    if length <= 0:
+        return None
+    return length
+
+
+def _hf_context_key(hparams: dict) -> tuple[str | None, int | None]:
+    """Key llama.cpp's find_hparam will see, and its scalar length if it has one."""
+    for key in CONTEXT_HPARAMS:
+        if key in hparams:
+            return key, scalar_context_length(hparams[key])
+    for nested_name in ("text_config", "llm_config", "language_config"):
+        nested = hparams.get(nested_name)
+        if not isinstance(nested, dict):
+            continue
+        for key in CONTEXT_HPARAMS:
+            if key in nested:
+                return key, scalar_context_length(nested[key])
+    return None, None
+
+
+def apply_hf_context_length(hparams: dict, length: int | None) -> int | None:
+    """Return the context llama.cpp will write, after an optional override.
+
+    llama.cpp reads the first usual context key on the top-level config. A
+    list or a missing key is not a usable length. The override is written
+    where that lookup will find it.
+    """
+    requested = require_context_length(length)
+    found_key, found_value = _hf_context_key(hparams)
+    if requested is not None:
+        hparams[found_key or "max_position_embeddings"] = requested
+        log.info("context length %d", requested)
+        return requested
+    if found_value is None:
+        return None
+    if found_key not in hparams:
+        hparams[found_key] = found_value
+        log.info("context length %d from the nested checkpoint config", found_value)
+    else:
+        log.info("context length %d from the checkpoint", found_value)
+    return found_value
+
+OUTTYPES = ("auto", "f32", "f16", "bf16", "nvfp4")
+
+
+def outtype(value: str) -> str:
+    return value.lower()
+
+
+def quote_modelfile_value(text: str) -> str:
+    if text == "" or any(char.isspace() for char in text) or '"' in text:
+        if '"""' in text:
+            raise ValueError("Modelfile value contains a triple quote, which a Modelfile cannot hold")
+        if "\n" in text or '"' in text:
+            return '"""' + text + '"""'
+        return '"' + text + '"'
+    return text
+
+
+def mmproj_destinations(gguf_path: Path, sources: list[Path] | None) -> list[tuple[Path, Path]]:
+    """Pair each projector with the path it will have in the output directory."""
+    dest_dir = gguf_path.parent
+    planned: list[tuple[Path, Path]] = []
+    seen: set[str] = set()
+    for source in sources or []:
+        source = Path(source)
+        if not source.is_file():
+            raise SystemExit(f"mmproj file does not exist: {source}")
+        if source.name == gguf_path.name or source.name in seen:
+            raise SystemExit(
+                f"mmproj {source} would be copied as {source.name}, which collides with another file in {dest_dir}."
+            )
+        seen.add(source.name)
+        planned.append((source, dest_dir / source.name))
+    return planned
+
+
+def place_mmprojs(gguf_path: Path, sources: list[Path] | None, overwrite: bool) -> list[Path]:
+    """Copy projector GGUFs next to the language model, keeping their file names."""
+    copied: list[Path] = []
+    for source, dest in mmproj_destinations(gguf_path, sources):
+        try:
+            same = dest.exists() and dest.resolve() == source.resolve()
+        except OSError:
+            same = False
+        if same:
+            copied.append(dest)
+            continue
+        if dest.exists() and not overwrite:
+            raise SystemExit(f"{dest} already exists. Pass --overwrite to replace it.")
+        shutil.copy2(source, dest)
+        log.info("Copied vision projector to %s", dest)
+        copied.append(dest)
+    return copied
+
+
+def refuse_existing_bundle(gguf_path: Path, overwrite: bool, mmprojs: list[Path] | None, *, skip_gguf: bool = False) -> None:
+    """Stop before a long run when the GGUF, Modelfile, or a copied projector is already there."""
+    if not skip_gguf and gguf_path.exists() and not overwrite:
+        raise SystemExit(f"{gguf_path} already exists. Pass --overwrite to replace it.")
+    modelfile = gguf_path.parent / "Modelfile"
+    if modelfile.exists() and not overwrite:
+        raise SystemExit(f"{modelfile} already exists. Pass --overwrite to replace it.")
+    for source, dest in mmproj_destinations(gguf_path, mmprojs):
+        try:
+            same = dest.exists() and dest.resolve() == source.resolve()
+        except OSError:
+            same = False
+        if dest.exists() and not same and not overwrite:
+            raise SystemExit(f"{dest} already exists. Pass --overwrite to replace it.")
+
+
+def _stored_value(kv: dict, name: str):
+    item = kv.get(name)
+    if item is None:
+        return None
+    return getattr(item, "value", item)
+
+
+def _nonempty(value) -> bool:
+    if value is None:
+        return False
+    if isinstance(value, str):
+        return bool(value.strip())
+    if isinstance(value, (list, tuple)):
+        return len(value) > 0
+    return True
+
+
+def warn_missing_gguf_metadata(writer) -> None:
+    """Warn when the GGUF metadata cannot describe a usable chat model.
+
+    Context length, the chat template, and the tokenizer are the fields that
+    still produce a file but change how it runs. Block count and embedding
+    length are what llama.cpp needs to build the graph.
+    """
+    kv = writer.kv_data[0] if writer.kv_data else {}
+    arch = writer.arch or "unknown"
+    missing: list[str] = []
+
+    context_keys = [name for name in kv if is_model_context_key(name)]
+    if not context_keys:
+        missing.append(f"context length ({arch}.context_length); pass --context-length to set it")
+    elif all(scalar_context_length(_stored_value(kv, name)) is None for name in context_keys):
+        missing.append("a single context length; pass --context-length to replace the stored value")
+
+    template_names = [
+        name for name in kv
+        if name == "tokenizer.chat_template" or name == "tokenizer.chat_templates" or name.startswith("tokenizer.chat_template.")
+    ]
+    if not any(_nonempty(_stored_value(kv, name)) for name in template_names):
+        missing.append("chat template (tokenizer.chat_template)")
+
+    if not _nonempty(_stored_value(kv, "tokenizer.ggml.model")):
+        missing.append("tokenizer model (tokenizer.ggml.model)")
+    if not _nonempty(_stored_value(kv, "tokenizer.ggml.tokens")):
+        missing.append("tokenizer vocabulary (tokenizer.ggml.tokens)")
+    if "tokenizer.ggml.eos_token_id" not in kv:
+        missing.append("end-of-sequence token (tokenizer.ggml.eos_token_id)")
+    if scalar_context_length(_stored_value(kv, f"{arch}.block_count")) is None:
+        missing.append(f"block count ({arch}.block_count)")
+    if scalar_context_length(_stored_value(kv, f"{arch}.embedding_length")) is None:
+        missing.append(f"embedding length ({arch}.embedding_length)")
+
+    if missing:
+        log.warning("GGUF is missing %s.", "; ".join(missing))
+
+
+# llama.cpp enables its DeepSeek-R1 tool parser only when the template source
+# contains this exact expression. The sglang DeepSeek-R1 template renders the
+# same calls but does not contain the expression, so the calls stay in the
+# visible reply.
+_DEEPSEEK_R1_TOOL_MARKER = (
+    "{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>'"
+)
+# A brace was left between the two branches of that template. Jinja prints it
+# after every tool call that has no text.
+_DEEPSEEK_R1_STRAY_BRACE = "'<｜tool▁call▁end｜>'}}\n                }\n                {%- else %}"
+_DEEPSEEK_R1_STRAY_FIXED = "'<｜tool▁call▁end｜>'}}\n                {%- else %}"
+
+
+def normalize_chat_template(value):
+    """Make a DeepSeek-R1 tool template recognizable to llama.cpp's parser."""
+    if isinstance(value, str):
+        return _normalize_one_chat_template(value)
+    if isinstance(value, list):
+        changed = []
+        for item in value:
+            if isinstance(item, str):
+                changed.append(_normalize_one_chat_template(item))
+            elif isinstance(item, dict) and isinstance(item.get("template"), str):
+                item = dict(item)
+                item["template"] = _normalize_one_chat_template(item["template"])
+                changed.append(item)
+            else:
+                changed.append(item)
+        return changed
+    return value
+
+
+def _normalize_one_chat_template(template: str) -> str:
+    if "tool▁calls▁begin" not in template:
+        return template
+    if _DEEPSEEK_R1_STRAY_BRACE in template:
+        template = template.replace(_DEEPSEEK_R1_STRAY_BRACE, _DEEPSEEK_R1_STRAY_FIXED, 1)
+    if _DEEPSEEK_R1_TOOL_MARKER not in template:
+        template = "{# " + _DEEPSEEK_R1_TOOL_MARKER + " #}\n" + template
+        log.info("Adjusted the DeepSeek-R1 tool template so tool calls are parsed instead of printed.")
+    return template
+
+
+def flatten_chat_template(value) -> str:
+    """Join a string or a list of named chat templates into one text blob."""
+    if value is None:
+        return ""
+    if isinstance(value, bytes):
+        return value.decode("utf-8", "replace")
+    if isinstance(value, str):
+        return value
+    if isinstance(value, list):
+        parts: list[str] = []
+        for item in value:
+            if isinstance(item, str):
+                parts.append(item)
+            elif isinstance(item, dict) and isinstance(item.get("template"), str):
+                parts.append(item["template"])
+        return "\n".join(parts)
+    return ""
+
+
+# Ollama accepts a tools request only when the chat template both mentions tools
+# and can render a tool result. A bare "tools" string does not.
+_TOOL_RESULT_MARKERS = (
+    "tool_response",
+    "tool_results",
+    "role'] == 'tool'",
+    "role'] == \"tool\"",
+    "role\"] == 'tool'",
+    "role\"] == \"tool\"",
+    "message.role == 'tool'",
+    "message.role == \"tool\"",
+    "ipython",
+)
+
+
+def chat_template_has_tools(template: str | None) -> bool:
+    """True when the template can render a tool call and the tool's result."""
+    if not template:
+        return False
+    if "tools" not in template and "tool_call" not in template:
+        return False
+    if "tool_calls" not in template and "assistant_tool_call" not in template:
+        return False
+    return any(marker in template for marker in _TOOL_RESULT_MARKERS)
+
+
+def tool_modelfile_lines(template: str | None) -> list[str]:
+    """Declare tools when the stored chat template can round-trip them.
+
+    A template that cannot render a tool result leaves the capability off.
+    Ollama then rejects a request that includes tools instead of dropping the
+    call on the floor.
+    """
+    if chat_template_has_tools(template):
+        return [
+            "CAPABILITY tools",
+            "# Tool calls use the chat template stored in the GGUF.",
+        ]
+    return ["# This chat template has no tool round-trip, so tools stay off."]
+
+
+def write_simple_modelfile(
+    gguf_path: Path,
+    overwrite: bool,
+    mmprojs: list[Path] | None,
+    chat_template: str | None = None,
+) -> Path:
+    """Write a Modelfile that points at a finished GGUF and any projector copies.
+
+    Context is not set here. A PARAMETER num_ctx would override OLLAMA_CONTEXT_LENGTH.
+    """
+    modelfile = gguf_path.parent / "Modelfile"
+    if modelfile.exists() and not overwrite:
+        raise SystemExit(f"{modelfile} already exists. Pass --overwrite to replace it.")
+    projectors = place_mmprojs(gguf_path, mmprojs, overwrite)
+    lines = [f"FROM ./{gguf_path.name}"]
+    for projector in projectors:
+        lines.append(f"FROM ./{projector.name}")
+    lines.append("")
+    lines.append("# The chat template is the one stored in the GGUF.")
+    lines.extend(tool_modelfile_lines(chat_template))
+    if chat_template_has_tools(chat_template):
+        log.info("Chat template supports tools.")
+    elif chat_template:
+        log.info("Chat template has no tool round-trip. Tool requests stay rejected.")
+    lines.append("# Context length comes from OLLAMA_CONTEXT_LENGTH.")
+    lines.append("")
+    modelfile.write_text("\n".join(lines), encoding="utf-8")
+    log.info("Wrote %s", modelfile)
+    return modelfile

nvfp4gguf/quant.py

@@ -0,0 +1,1000 @@
+"""NVFP4 learned rounding and ggml packing.
+
+The optimizer is convert_to_quant's LearnedNVFP4Converter. This module only
+captures the scales that quantizer used, packs them into llama.cpp's NVFP4
+block layout, and keeps the same hyperparameters convert_to_quant uses.
+Matrix multiplies stay in full FP32. TF32 would make the SVD residual drift
+from a convert_to_quant run on the same tensor.
+"""
+
+from __future__ import annotations
+
+import argparse
+import gc
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+import torch
+
+log = logging.getLogger("nvfp4gguf")
+
+QUANT_VERSION = 2
+QK_NVFP4 = 64
+NVFP4_BLOCK_BYTES = 36
+F8_E4M3_MAX = 448.0
+CTQ_NUM_ITER = 4000
+# Joint trains the block scales inside the same loop. Iterative refreshes them
+# only on the original optimizer, so joint is the default that Prodigy, AdamW,
+# and RAdam all honor.
+DEFAULT_SCALE_OPTIMIZATION = "joint"
+DEFAULT_SCALE_REFINEMENT_ROUNDS = 2
+# Patience 1 decays the step on every miss and the lr-floor stop then ends
+# the search while the loss is still the best. Prodigy adapts its own step,
+# so a slower plateau and no floor abort let the budget actually run.
+DEFAULT_LR_PATIENCE = 32
+DEFAULT_EARLY_STOP_LR = 0.0
+# A wobble of 2.000e-7 to 2.002e-7 is not progress, but only once the step
+# size has floored. Before that, any new best has to count or the search
+# stops while the loss is still around 1e-2.
+DEFAULT_LR_THRESHOLD = 0.01
+DEFAULT_EARLY_STOP_STALL = 2000
+# Once the step size is at --lr-min, this many steps without a new best ends
+# the matrix. 64 is about a second at the speed these fits run.
+DEFAULT_EARLY_STOP_FLOOR_STALL = 64
+# ffn_down at 9e-4 already matched a run that reached 3e-7. 1e-5 is below
+# that, so hitting it does not change the packed weights.
+DEFAULT_TARGET_LOSS = 1e-5
+# These projections write the residual or build Q, K, and V. The low-rank
+# objective ignores most of their error, which is where a wrong token comes from.
+_TIGHT_ATTENTION = (
+    "attn_output.weight",
+    "attn_q.weight",
+    "attn_k.weight",
+    "attn_v.weight",
+    "attn_qkv.weight",
+    "o_proj.weight",
+    "q_proj.weight",
+    "k_proj.weight",
+    "v_proj.weight",
+    "q_k_v_proj.weight",
+)
+
+DEFAULT_LLAMA_CPP = Path.home() / "llama.cpp"
+DEFAULT_CONVERT_TO_QUANT = Path("/mnt/d/dev/convert_to_quant")
+
+
+def _ensure_import_path(path: Path, label: str) -> None:
+    path = path.resolve()
+    if not path.exists():
+        raise FileNotFoundError(f"{label} path does not exist: {path}")
+    entry = str(path)
+    if entry not in sys.path:
+        sys.path.insert(0, entry)
+
+
+def load_gguf_modules(llama_cpp: Path):
+    llama_cpp = llama_cpp.resolve()
+    if not (llama_cpp / "convert_hf_to_gguf.py").is_file():
+        raise FileNotFoundError(
+            f"{llama_cpp} does not look like a llama.cpp checkout (missing convert_hf_to_gguf.py)"
+        )
+    _ensure_import_path(llama_cpp, "llama.cpp")
+    _ensure_import_path(llama_cpp / "gguf-py", "llama.cpp gguf-py")
+    os.environ.setdefault("NO_LOCAL_GGUF", "1")
+    import gguf
+    from conversion import ModelBase, ModelType, get_model_architecture, get_model_class
+
+    return gguf, ModelBase, ModelType, get_model_architecture, get_model_class
+
+
+def import_gguf(llama_cpp: Path | None = None):
+    root = Path(llama_cpp) if llama_cpp else DEFAULT_LLAMA_CPP
+    gguf_path = root / "gguf-py"
+    if not gguf_path.is_dir():
+        raise FileNotFoundError(f"llama.cpp gguf-py was not found at {gguf_path}")
+    entry = str(gguf_path)
+    if entry not in sys.path:
+        sys.path.insert(0, entry)
+    import gguf
+    return gguf
+
+
+def load_learned_converter(convert_to_quant_dir: Path | None):
+    try:
+        from convert_to_quant.converters.learned_nvfp4 import LearnedNVFP4Converter
+        from convert_to_quant.utils.float_utils import unpack_uint4
+        return LearnedNVFP4Converter, unpack_uint4
+    except ImportError:
+        pass
+
+    search = []
+    if convert_to_quant_dir is not None:
+        search.append(Path(convert_to_quant_dir))
+    search.append(DEFAULT_CONVERT_TO_QUANT)
+    last_error = None
+    for candidate in search:
+        if not candidate.exists():
+            continue
+        _ensure_import_path(candidate, "convert_to_quant")
+        try:
+            from convert_to_quant.converters.learned_nvfp4 import LearnedNVFP4Converter
+            from convert_to_quant.utils.float_utils import unpack_uint4
+            return LearnedNVFP4Converter, unpack_uint4
+        except ImportError as exc:
+            last_error = exc
+    raise ImportError(
+        "convert_to_quant is not importable. Install it, or pass --convert-to-quant "
+        "pointing at the checkout that provides LearnedNVFP4Converter."
+    ) from last_error
+
+
+def ctq_skips_key(key: str, convert_to_quant_dir: Path | None = None) -> bool:
+    """True when convert_to_quant's default exclusion list would leave this tensor alone."""
+    try:
+        from convert_to_quant.constants import AVOID_KEY_NAMES
+    except ImportError:
+        load_learned_converter(convert_to_quant_dir)
+        from convert_to_quant.constants import AVOID_KEY_NAMES
+    return any(pattern in key for pattern in AVOID_KEY_NAMES)
+
+
+def resolve_optimizer(requested: str) -> str:
+    if requested != "auto":
+        return requested
+    try:
+        import prodigyplus.prodigy_plus_schedulefree  # noqa: F401
+        return "prodigy"
+    except ImportError:
+        log.info(
+            "prodigy-plus-schedule-free is not installed; using AdamW. "
+            "pip install prodigy-plus-schedule-free to match convert_to_quant's default optimizer."
+        )
+        return "adamw"
+
+
+def _opt(args, name: str, default):
+    return getattr(args, name, default)
+
+
+def resolved_lr_min(args) -> float:
+    """Plateau floor from ``--lr-min``. ``--lr`` is only the starting rate."""
+    lr_min = getattr(args, "lr_min", None)
+    if lr_min is None:
+        return 1e-8
+    return float(lr_min)
+
+
+class GgufLearnedNVFP4:
+    """LearnedNVFP4Converter plus the total scale the codes were optimized against."""
+
+    def __init__(self, converter_cls, **kwargs):
+        class _Capturing(converter_cls):
+            def convert(self, *args, **kw):
+                self._captured_total_scale = None
+                self._last_lr = self.lr
+                self._floor_stall = 0
+                self._best_seen = None
+                self._settled = False
+                return super().convert(*args, **kw)
+
+            def _learning_rate_floored(self) -> bool:
+                # lr_min is --lr-min, the plateau floor. --lr is only where the search starts.
+                return getattr(self, "_last_lr", self.lr) <= self.lr_min * 1.01
+
+            def _loss_has_settled(self, loss: float) -> bool:
+                target = float(getattr(self, "target_loss", DEFAULT_TARGET_LOSS) or 0)
+                return target > 0 and 0 < loss <= target
+
+            def _check_improvement(self, current_loss: float, best_loss: float) -> bool:
+                # Sub-1% wiggles stop counting once the fit has settled or the
+                # step has reached --lr-min. Before that they are the descent.
+                settled = getattr(self, "_settled", False)
+                if self.lr_threshold > 0 and (settled or self._learning_rate_floored()):
+                    return super()._check_improvement(current_loss, best_loss)
+                return current_loss < best_loss
+
+            def _check_early_stop(self, current_loss: float, curr_lr: float, worse_loss_counter: int) -> bool:
+                self._last_lr = curr_lr
+                if 0 < current_loss < float("inf"):
+                    best = getattr(self, "_best_seen", None)
+                    self._best_seen = current_loss if best is None else min(best, current_loss)
+                best_seen = getattr(self, "_best_seen", None)
+                self._settled = self._loss_has_settled(best_seen if best_seen is not None else current_loss)
+                limit = int(getattr(self, "early_stop_floor_stall", 0) or 0)
+                # The packed codes are already chosen once the best loss is at
+                # the target. The learning rate does not change that.
+                if self._settled:
+                    log.info("    Loss reached %g. Stopping early.", float(self.target_loss))
+                    return True
+                return super()._check_early_stop(current_loss, curr_lr, worse_loss_counter)
+
+            def _compute_svd_components(self, W_float32, verbose: bool = True):
+                # full_matrices=True builds the square factors and then throws
+                # them away. The loss only uses k directions, and a thin SVD
+                # returns those directly.
+                if not self.full_matrix:
+                    return super()._compute_svd_components(W_float32, verbose)
+                import math
+                rows, cols = W_float32.shape
+                max_rank = min(rows, cols)
+                k = min(self.max_k, max(self.min_k, int(math.floor(self.top_p * max_rank))))
+                k = min(k, max_rank)
+                if verbose:
+                    print(f"    - Tensor shape: [{rows}, {cols}], Max rank: {max_rank}. Using k={k} components.")
+                    print("    - Using torch.linalg.svd with full_matrices=False")
+                left, _, right = torch.linalg.svd(W_float32, full_matrices=False)
+                return left[:, :k], right[:k, :], k
+
+            def _compute_block_scales(self, weight, per_tensor_scale, rows, cols):
+                fp8, total, mask = super()._compute_block_scales(weight, per_tensor_scale, rows, cols)
+                self._captured_total_scale = total.detach()
+                return fp8, total, mask
+
+            def _optimize_nvfp4(self, *args, **kw):
+                codes, total = super()._optimize_nvfp4(*args, **kw)
+                self._captured_total_scale = total.detach()
+                return codes, total
+
+        self.converter = _Capturing(**kwargs)
+
+    def convert(self, weight: torch.Tensor):
+        return self.converter.convert(weight)
+
+
+def pack_nvfp4(codes: np.ndarray, scale_bits: np.ndarray) -> np.ndarray:
+    """Pack E2M1 codes and UE4M3 block scales into ggml NVFP4 bytes.
+
+    ``codes`` is ``(..., K)`` with K a multiple of 64. Each nibble is an E2M1
+    code. ``scale_bits`` is ``(..., K / 16)``. A super-block is 4 scale bytes
+    followed by 32 code bytes, matching ``block_nvfp4``.
+    """
+    codes = np.ascontiguousarray(codes, dtype=np.uint8)
+    scale_bits = np.ascontiguousarray(scale_bits, dtype=np.uint8) & np.uint8(0x7F)
+    if codes.shape[:-1] != scale_bits.shape[:-1]:
+        raise ValueError(f"code shape {codes.shape} does not match scales {scale_bits.shape}")
+    length = codes.shape[-1]
+    if length % QK_NVFP4 != 0:
+        raise ValueError(f"NVFP4 row length {length} is not a multiple of {QK_NVFP4}")
+    if scale_bits.shape[-1] * 16 != length:
+        raise ValueError("expected one scale per 16 values")
+
+    prefix = codes.shape[:-1]
+    n_super = length // QK_NVFP4
+    grouped = codes.reshape(*prefix, n_super, 4, 16)
+    quant_bytes = (grouped[..., :8] | (grouped[..., 8:] << np.uint8(4))).reshape(*prefix, n_super, 32)
+    scales = scale_bits.reshape(*prefix, n_super, 4)
+    return np.ascontiguousarray(
+        np.concatenate([scales, quant_bytes], axis=-1).reshape(*prefix, n_super * NVFP4_BLOCK_BYTES)
+    )
+
+
+def mtp_layer_tensor(name: str, n_blocks: int, n_nextn: int) -> bool:
+    """True for weights stored in the NextN/MTP blocks after the trunk.
+
+    llama.cpp's normal load skips those blocks before it looks for NVFP4
+    ``.scale`` sidecars, so a sidecar on one of them is never claimed and the
+    loader aborts with ``wrong number of tensors``.
+    """
+    if n_nextn <= 0 or n_blocks <= 0 or not name.startswith("blk."):
+        return False
+    layer = name.split(".", 2)[1]
+    if not layer.isdigit():
+        return False
+    return int(layer) >= n_blocks - n_nextn
+
+
+def bake_packed_nvfp4(raw: np.ndarray, per_tensor: float) -> np.ndarray:
+    """Fold a per-tensor scale into the UE4M3 bytes of packed NVFP4 blocks."""
+    raw = np.ascontiguousarray(raw, dtype=np.uint8)
+    if raw.size == 0 or raw.shape[-1] % NVFP4_BLOCK_BYTES != 0:
+        raise ValueError(f"NVFP4 packing has shape {raw.shape}, which is not groups of {NVFP4_BLOCK_BYTES} bytes")
+    if abs(float(per_tensor) - 1.0) <= 1e-6:
+        return raw
+    blocks = raw.reshape(*raw.shape[:-1], -1, NVFP4_BLOCK_BYTES)
+    scales = np.ascontiguousarray(blocks[..., :4])
+    decoded = torch.from_numpy(scales).view(torch.float8_e4m3fn).float().numpy()
+    encoded = ue4m3_bits(decoded * np.float32(per_tensor)).reshape(scales.shape)
+    baked = np.array(blocks, copy=True)
+    baked[..., :4] = encoded
+    return np.ascontiguousarray(baked.reshape(raw.shape))
+
+
+def ue4m3_bits(scale_float: np.ndarray) -> np.ndarray:
+    """Encode positive scales as UE4M3 using torch's E4M3FN bit patterns.
+
+    ggml's own encoder saturates every value from 256 to 448 down to 448.
+    Torch's float8 bits decode back to the right magnitude with ggml's decoder,
+    including the exponent-15 range that NVFP4 block scales actually use.
+    """
+    array = np.ascontiguousarray(np.clip(scale_float, 0.0, F8_E4M3_MAX), dtype=np.float32)
+    bits = torch.from_numpy(array).to(torch.float8_e4m3fn).view(torch.uint8).numpy()
+    return np.ascontiguousarray(bits & np.uint8(0x7F))
+
+
+def _cuda_python_candidates() -> list[Path]:
+    found: list[Path] = []
+    env_root = Path.home() / "anaconda3" / "envs"
+    if env_root.is_dir():
+        for python in sorted(env_root.glob("*/bin/python")):
+            found.append(python)
+    which = shutil.which("python3")
+    if which:
+        found.append(Path(which))
+    return found
+
+
+def _python_sees_cuda(python: Path) -> bool:
+    if not python.is_file():
+        return False
+    probe = subprocess.run(
+        [str(python), "-c", "import torch, sys; sys.exit(0 if torch.cuda.is_available() else 1)"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        check=False,
+    )
+    return probe.returncode == 0
+
+
+def ensure_cuda_interpreter(device: str) -> None:
+    """Re-run under a CUDA torch when this interpreter cannot see the GPU."""
+    wants_cuda = device == "auto" or str(device).startswith("cuda")
+    if not wants_cuda or torch.cuda.is_available() or os.environ.get("HF_NVFP4_REEXEC") == "1":
+        return
+    current = Path(sys.executable).resolve()
+    candidates = _cuda_python_candidates()
+    candidates.sort(key=lambda path: (0 if "envs/torch/" in path.as_posix() else 1, str(path)))
+    for candidate in candidates:
+        try:
+            if candidate.resolve() == current:
+                continue
+        except OSError:
+            continue
+        if not _python_sees_cuda(candidate):
+            continue
+        print(
+            f"This interpreter ({sys.executable}, torch {torch.__version__}) cannot use CUDA. "
+            f"Restarting with {candidate}",
+            flush=True,
+        )
+        os.environ["HF_NVFP4_REEXEC"] = "1"
+        os.execv(str(candidate), [str(candidate), *sys.argv])
+    if device == "auto":
+        return
+    built = torch.version.cuda or "cpu-only"
+    raise RuntimeError(
+        f"{sys.executable} has torch {torch.__version__} (CUDA {built}) and no visible GPU. "
+        "Pass --device cpu, or run this script with a CUDA build of torch."
+    )
+
+
+def configure_matmul(device: str) -> None:
+    """Full FP32 matmul. convert_to_quant does not enable TF32, and TF32 moves the SVD residual."""
+    if not str(device).startswith("cuda"):
+        return
+    torch.backends.cuda.matmul.allow_tf32 = False
+    torch.backends.cudnn.allow_tf32 = False
+    torch.set_float32_matmul_precision("highest")
+
+
+class TensorQuantizer:
+    def __init__(self, args, gguf_module, unpack_uint4):
+        self.args = args
+        self.gguf = gguf_module
+        self.unpack_uint4 = unpack_uint4
+        self.nvfp4_cls = getattr(getattr(gguf_module, "quants", None), "NVFP4", None)
+        optimizer = resolve_optimizer(args.optimizer)
+        self.optimizer_name = optimizer
+        device = args.device
+        if device == "auto":
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+        if str(device).startswith("cuda") and not torch.cuda.is_available():
+            raise RuntimeError(
+                f"{sys.executable} has torch {torch.__version__} (CUDA {torch.version.cuda or 'cpu-only'}) "
+                "and no visible GPU."
+            )
+        self.device = device
+        configure_matmul(device)
+
+        converter_cls, _ = load_learned_converter(getattr(args, "convert_to_quant", None))
+        lr_min = resolved_lr_min(args)
+        self.engine = GgufLearnedNVFP4(
+            converter_cls,
+            optimizer=optimizer,
+            num_iter=args.num_iter,
+            lr=args.lr,
+            top_p=args.top_p,
+            min_k=args.min_k,
+            max_k=args.max_k,
+            full_matrix=args.full_matrix,
+            no_learned_rounding=args.simple,
+            lr_schedule=args.lr_schedule,
+            lr_gamma=_opt(args, "lr_gamma", 0.99),
+            lr_patience=_opt(args, "lr_patience", DEFAULT_LR_PATIENCE),
+            lr_factor=_opt(args, "lr_factor", 0.95),
+            lr_min=lr_min,
+            lr_cooldown=_opt(args, "lr_cooldown", 0),
+            lr_threshold=_opt(args, "lr_threshold", DEFAULT_LR_THRESHOLD),
+            lr_shape_influence=_opt(args, "lr_shape_influence", 1.0),
+            early_stop_loss=args.early_stop_loss,
+            early_stop_lr=_opt(args, "early_stop_lr", DEFAULT_EARLY_STOP_LR),
+            early_stop_stall=_opt(args, "early_stop_stall", DEFAULT_EARLY_STOP_STALL),
+            scale_refinement_rounds=args.scale_refinement_rounds,
+            scale_optimization=args.scale_optimization,
+            use_speed=_opt(args, "use_speed", False),
+            block_size=16,
+            pad_to_16x=True,
+            device=device,
+        )
+        self.engine.converter.early_stop_floor_stall = _opt(
+            args, "early_stop_floor_stall", DEFAULT_EARLY_STOP_FLOOR_STALL,
+        )
+        self.engine.converter.target_loss = float(_opt(args, "target_loss", DEFAULT_TARGET_LOSS))
+        self.calib_cache: dict[int, torch.Tensor] = {}
+        log.info(
+            "NVFP4 quantizer: %s on %s, %s iterations, scale mode %s, lr patience %s, "
+            "stop at loss %g, fp32 matmul",
+            "simple round-to-grid" if args.simple else f"learned rounding ({optimizer})",
+            device,
+            args.num_iter if not args.simple else 0,
+            args.scale_optimization,
+            _opt(args, "lr_patience", DEFAULT_LR_PATIENCE),
+            self.engine.converter.target_loss,
+        )
+        if not args.simple:
+            log.info("Attention projections stay NVFP4 and are fit across every singular direction.")
+
+    def _extract(self, weight: torch.Tensor, converted) -> tuple[np.ndarray, np.ndarray, float, np.ndarray]:
+        packed, _blocked, per_tensor, dequant, _extra = converted
+        rows, cols = weight.shape
+        if not torch.any(packed):
+            codes = np.zeros((rows, cols), dtype=np.uint8)
+            scales = np.zeros((rows, cols // 16), dtype=np.float32)
+            return codes, scales, 1.0, np.zeros((rows, cols), dtype=np.float32)
+
+        codes_t = self.unpack_uint4(packed.detach().contiguous())
+        codes = codes_t[:rows, :cols].to(dtype=torch.uint8).cpu().numpy()
+        total = self.engine.converter._captured_total_scale
+        if total is None:
+            raise RuntimeError("quantizer did not record a block scale")
+        pts = per_tensor.detach().to(device=total.device, dtype=total.dtype).reshape(-1)[0]
+        pts_value = float(pts.float().cpu())
+        if not np.isfinite(pts_value) or pts_value <= 0.0:
+            raise RuntimeError(f"invalid per-tensor scale {pts_value}")
+        block_scale = (total / pts)[:rows, : cols // 16].float().cpu().numpy()
+        dequant_np = dequant.detach().float().cpu().numpy()[:rows, :cols]
+        return codes, block_scale.astype(np.float32, copy=False), pts_value, dequant_np
+
+    def quantize_matrix(self, weight: np.ndarray, name: str = "") -> tuple[np.ndarray, np.ndarray, float, np.ndarray]:
+        tensor = torch.from_numpy(np.ascontiguousarray(weight, dtype=np.float32))
+        converter = self.engine.converter
+        saved = None
+        if _tight_attention(name):
+            # The stock objective keeps the top fifth of the singular spectrum.
+            # A complete basis makes the loss see every direction this matrix has.
+            saved = (converter.top_p, converter.max_k, converter.full_matrix)
+            converter.top_p = 1.0
+            converter.max_k = max(int(converter.max_k), int(min(weight.shape[-2:])), 1)
+            converter.full_matrix = True
+            log.info("    full-rank fit %s", name)
+        converted = None
+        try:
+            with torch.enable_grad():
+                converted = self.engine.convert(tensor)
+            return self._extract(tensor, converted)
+        finally:
+            if saved is not None:
+                converter.top_p, converter.max_k, converter.full_matrix = saved
+            del tensor, converted
+
+    def bake_global_scale(self, codes: np.ndarray, block_scale: np.ndarray, per_tensor: float) -> tuple[np.ndarray, np.ndarray]:
+        """Fold the per-tensor scale into UE4M3 block scales and keep the learned codes."""
+        product = block_scale * np.float32(per_tensor)
+        return codes, ue4m3_bits(product)
+
+    def runtime_dequant(self, raw: np.ndarray, per_tensor: float | None) -> np.ndarray:
+        if self.gguf is None:
+            raise RuntimeError("runtime dequant needs the gguf module")
+        values = self.gguf.quants.dequantize(raw, self.gguf.GGMLQuantizationType.NVFP4)
+        if per_tensor is not None:
+            values = values * np.float32(per_tensor)
+        return values
+
+    def bias_correction(self, original: np.ndarray, reconstructed: np.ndarray) -> np.ndarray:
+        rows, cols = original.shape
+        samples = self.args.calib_samples
+        cached = self.calib_cache.get(cols)
+        if cached is None or cached.shape[0] != samples:
+            generator = torch.Generator(device="cpu")
+            generator.manual_seed(self.args.seed + cols)
+            cached = torch.randn(samples, cols, generator=generator, dtype=torch.float32)
+            self.calib_cache[cols] = cached
+
+        error = torch.from_numpy(np.ascontiguousarray(original - reconstructed, dtype=np.float32))
+        error = error.to(self.device)
+        accumulator = torch.zeros(rows, dtype=torch.float32)
+        batch = 256
+        for start in range(0, samples, batch):
+            batch_x = cached[start:start + batch].to(self.device)
+            accumulator += (batch_x @ error.T).sum(dim=0).float().cpu()
+            del batch_x
+        del error
+        gc.collect()
+        return (accumulator / samples).numpy().astype(np.float32, copy=False)
+
+
+# Feed-forward matmuls in these graphs pass a null scale, so a sidecar
+# `.scale` tensor is loaded and then ignored. Gemma 4 and Gemma embedding 2
+# do apply it. See llama.cpp src/models/gemma3.cpp build_ffn.
+_FFN_SCALE_IGNORED = frozenset({
+    "gemma",
+    "gemma2",
+    "gemma3",
+    "gemma-embedding",
+})
+
+# gemma3n multiplies the NVFP4 scale only on the attention output and the LM head.
+_GEMMA3N_SCALE_USED = frozenset({"attn_output", "output"})
+
+
+def _lfm_ignores_nvfp4_scale(name: str) -> bool:
+    """LFM2 and LFM3 multiply a sidecar scale on Q, K, V, and the attention output.
+
+    The dense feed-forward, the per-head attention gate, and the short-conv
+    projections are called with a null scale. A short-conv sidecar is also
+    never loaded, so writing one makes the GGUF fail to open.
+    """
+    if _is_ffn_matmul(name):
+        return True
+    if ".shortconv." in name:
+        return True
+    return _weight_leaf(name) == "attn_gate"
+
+
+def _architecture_name(architecture: str | bytes | None) -> str:
+    if architecture is None:
+        return ""
+    if isinstance(architecture, bytes):
+        architecture = architecture.decode("utf-8", "replace")
+    return str(architecture).strip().lower()
+
+
+def _is_spark2_5(architecture: str | bytes | None) -> bool:
+    """True for the spark2_5 GGUF arch and the Spark2_5ForCausalLM class name.
+
+    Muse Spark and DeepSeek DSpark stay out of this profile.
+    """
+    name = _architecture_name(architecture).replace("-", "_").replace(".", "_")
+    head = name.split("for", 1)[0]
+    return head in {"spark2_5", "spark25"}
+
+
+class QuantProfile:
+    """Tensors an architecture leaves in the source float type unless --exclude is set."""
+
+    def __init__(self, arch: str, skip: str, label: str):
+        self.arch = arch
+        self.skip = skip
+        self.label = label
+
+
+# GGUF names are blk.N.attn_gate.weight. The Hugging Face checkpoint calls the
+# same matrix self_attn.g_proj.weight.
+_SPARK_PROFILE = QuantProfile(
+    "spark2_5",
+    r"(?:^|\.)(?:attn_gate|g_proj)\.weight$",
+    "attn_gate.weight",
+)
+
+
+def quant_profile(architecture: str | bytes | None) -> QuantProfile | None:
+    if _is_spark2_5(architecture):
+        return _SPARK_PROFILE
+    return None
+
+
+def _tight_attention(name: str) -> bool:
+    return bool(name) and name.endswith(_TIGHT_ATTENTION)
+
+
+def looks_like_spark(names) -> bool:
+    """True when a weight list has Spark's fused QKV and its attention gate."""
+    has_qkv = False
+    has_gate = False
+    for name in names:
+        if name.endswith("q_k_v_proj.weight"):
+            has_qkv = True
+        if name.endswith("g_proj.weight") or name.endswith("attn_gate.weight"):
+            has_gate = True
+        if has_qkv and has_gate:
+            return True
+    return False
+
+
+def bind_exclude(args, architecture: str | bytes | None) -> None:
+    """Compile args.exclude_re, using the architecture profile when --exclude is absent."""
+    if getattr(args, "_exclude_bound", False):
+        return
+    args._exclude_bound = True
+    profile = quant_profile(architecture)
+    user = getattr(args, "exclude", None)
+    pattern = user
+    if profile is not None and not user:
+        pattern = profile.skip
+        args.exclude = pattern
+        log.info("%s profile: leaving %s in the source float type.", profile.arch, profile.label)
+    elif profile is not None:
+        log.info(
+            "%s profile: --exclude replaces the default skip of %s.",
+            profile.arch,
+            profile.label,
+        )
+    args.exclude_re = re.compile(pattern) if pattern else None
+
+
+def quant_settings(args) -> dict:
+    """Flags that change the bytes in a checkpoint. A mismatch forces --fresh."""
+    return {
+        "version": QUANT_VERSION,
+        "simple": bool(getattr(args, "simple", False)),
+        "optimizer": args.optimizer,
+        "num_iter": args.num_iter,
+        "lr": args.lr,
+        "lr_schedule": args.lr_schedule,
+        "scale_optimization": args.scale_optimization,
+        "scale_refinement_rounds": args.scale_refinement_rounds,
+        "full_matrix": bool(getattr(args, "full_matrix", False)),
+        "top_p": args.top_p,
+        "min_k": args.min_k,
+        "max_k": args.max_k,
+        "early_stop_loss": args.early_stop_loss,
+        "early_stop_stall": _opt(args, "early_stop_stall", DEFAULT_EARLY_STOP_STALL),
+        "lr_threshold": _opt(args, "lr_threshold", DEFAULT_LR_THRESHOLD),
+        "target_loss": float(_opt(args, "target_loss", DEFAULT_TARGET_LOSS)),
+        "calib_samples": args.calib_samples,
+        "bias_correction": not getattr(args, "no_bias_correction", False),
+        "seed": args.seed,
+        "quantize_output": bool(getattr(args, "quantize_output", False)),
+        "exclude": getattr(args, "exclude", None),
+        "lr_patience": _opt(args, "lr_patience", DEFAULT_LR_PATIENCE),
+        "early_stop_lr": _opt(args, "early_stop_lr", DEFAULT_EARLY_STOP_LR),
+    }
+
+
+def require_quant_settings(directory: Path, args) -> None:
+    path = directory / "settings.json"
+    payload = quant_settings(args)
+    if path.is_file():
+        previous = json.loads(path.read_text(encoding="utf-8"))
+        if previous != payload:
+            raise SystemExit(
+                f"Checkpoint settings in {path} do not match this run. "
+                "Pass --fresh to recompute, or rerun with the same quantizer flags."
+            )
+        return
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+
+
+def _weight_leaf(name: str) -> str:
+    base = name[: -len(".weight")] if name.endswith(".weight") else name
+    return base.rsplit(".", 1)[-1]
+
+
+def _is_ffn_matmul(name: str) -> bool:
+    leaf = _weight_leaf(name)
+    if leaf.startswith("ffn_gate_inp"):
+        return False
+    return leaf.startswith(("ffn_up", "ffn_gate", "ffn_down"))
+
+
+def must_bake_nvfp4_scale(architecture: str | bytes | None, name: str, has_ffn_bias: bool = False) -> bool:
+    """Fold the per-tensor scale into the UE4M3 block scales.
+
+    Most llama.cpp graphs multiply ``name.scale`` after the NVFP4 matmul.
+    Gemma 3's feed-forward graph does not, which leaves those layers far too
+    small and the sampler stuck on one token. A dense FFN that also has a
+    bias cannot carry an external scale: llama.cpp adds the bias first and
+    aborts if both are present.
+    """
+    if has_ffn_bias and _is_ffn_matmul(name):
+        return True
+    arch = _architecture_name(architecture)
+    leaf = _weight_leaf(name)
+    if arch == "gemma4-assistant":
+        return True
+    if arch in {"lfm2", "lfm2moe", "lfm3"} and _lfm_ignores_nvfp4_scale(name):
+        return True
+    if arch == "gemma3n":
+        return leaf not in _GEMMA3N_SCALE_USED
+    if arch in _FFN_SCALE_IGNORED and _is_ffn_matmul(name):
+        return True
+    if arch in {"gemma4", "gemma-embedding2"} and (
+        leaf in {"inp_gate", "proj", "per_layer_model_proj"} or leaf.startswith("ffn_gate_inp")
+    ):
+        return True
+    # spark2-5.cpp passes null scales for the feed-forward, the attention
+    # gate, and the LM head. QKV and attn_output do multiply their sidecar.
+    if _is_spark2_5(arch) and (_is_ffn_matmul(name) or leaf in {"attn_gate", "output"}):
+        return True
+    return False
+
+
+def preserve_scale_tensor(architecture: str | bytes | None, weight_name: str) -> bool:
+    """Keep an existing ``.scale`` tensor that is not the NVFP4 weight scale.
+
+    Gemma 4 multiplies ``ffn_gate_inp.scale`` into the router activation
+    before the matmul. The NVFP4 weight scale is baked instead.
+    """
+    arch = _architecture_name(architecture)
+    if arch not in {"gemma4", "gemma-embedding2"}:
+        return False
+    return _weight_leaf(weight_name).startswith("ffn_gate_inp")
+
+
+def scale_bake_notice(architecture: str | bytes | None) -> str | None:
+    arch = _architecture_name(architecture)
+    if arch in _FFN_SCALE_IGNORED:
+        return (
+            f"{arch} feed-forward layers do not multiply an NVFP4 .scale tensor. "
+            "Those per-tensor scales are baked into the block scales."
+        )
+    if arch == "gemma3n":
+        return (
+            "gemma3n multiplies an NVFP4 .scale only on attn_output and output. "
+            "Every other matrix has its per-tensor scale baked into the block scales."
+        )
+    if arch in {"lfm2", "lfm2moe", "lfm3"}:
+        return (
+            f"{arch} does not multiply an NVFP4 .scale on the feed-forward, "
+            "the attention gate, or the short convolutions. "
+            "Those per-tensor scales are baked into the block scales."
+        )
+    if arch == "gemma4-assistant":
+        return (
+            "gemma4-assistant does not multiply NVFP4 .scale tensors. "
+            "Per-tensor scales are baked into the block scales."
+        )
+    if arch in {"gemma4", "gemma-embedding2"}:
+        return (
+            f"{arch} does not multiply an NVFP4 weight scale on the router, "
+            "the per-layer projections, or per_layer_model_proj. "
+            "Those scales are baked into the block scales."
+        )
+    if _is_spark2_5(arch):
+        return (
+            "spark2_5 does not multiply an NVFP4 .scale on the feed-forward, "
+            "the attention gate, or the output. Those per-tensor scales are baked "
+            "into the block scales."
+        )
+    return None
+
+
+def check_scale_policy() -> None:
+    """Guard the architecture table against the llama.cpp graphs it was read from."""
+    bake = must_bake_nvfp4_scale
+    assert bake("gemma3", "blk.0.ffn_up.weight")
+    assert bake("gemma3", "blk.0.ffn_gate.weight")
+    assert bake("gemma3", "blk.0.ffn_down.weight")
+    assert not bake("gemma3", "blk.0.attn_q.weight")
+    assert not bake("gemma3", "blk.0.attn_output.weight")
+    assert not bake("gemma3", "output.weight")
+    assert bake("gemma", "blk.0.ffn_up.weight")
+    assert bake("gemma2", "blk.0.ffn_down.weight")
+    assert bake(b"gemma3", "blk.0.ffn_up.weight")
+    assert bake("llama", "blk.0.ffn_up.weight", has_ffn_bias=True)
+    assert not bake("llama", "blk.0.ffn_up.weight")
+    assert not bake("llama", "blk.0.attn_q.weight")
+    assert bake("gemma3n", "blk.0.attn_q.weight")
+    assert bake("gemma3n", "blk.0.ffn_up.weight")
+    assert bake("gemma3n", "blk.0.laurel_l.weight")
+    assert not bake("gemma3n", "blk.0.attn_output.weight")
+    assert not bake("gemma3n", "output.weight")
+    assert not bake("gemma4", "blk.0.ffn_up.weight")
+    assert not bake("gemma4", "blk.0.attn_q.weight")
+    assert bake("gemma4", "blk.0.ffn_gate_inp.weight")
+    assert bake("gemma4", "blk.0.inp_gate.weight")
+    assert bake("gemma4", "blk.0.proj.weight")
+    assert bake("gemma4", "per_layer_model_proj.weight")
+    assert bake("gemma-embedding2", "per_layer_model_proj.weight")
+    assert not bake("llama", "per_layer_model_proj.weight")
+    assert preserve_scale_tensor("gemma4", "blk.0.ffn_gate_inp.weight")
+    assert not preserve_scale_tensor("gemma3", "blk.0.ffn_up.weight")
+    assert not preserve_scale_tensor("gemma4", "blk.0.ffn_up.weight")
+    assert bake("gemma4-assistant", "blk.0.attn_q.weight")
+    assert bake("gemma4-assistant", "output.weight")
+    assert bake("lfm2", "blk.0.ffn_up.weight")
+    assert bake("lfm2", "blk.0.ffn_gate.weight")
+    assert bake("lfm2", "blk.0.ffn_down.weight")
+    assert bake("lfm2", "blk.0.shortconv.in_proj.weight")
+    assert bake("lfm2", "blk.0.shortconv.out_proj.weight")
+    assert not bake("lfm2", "blk.0.attn_q.weight")
+    assert not bake("lfm2", "blk.0.attn_output.weight")
+    assert bake("lfm2moe", "blk.0.shortconv.in_proj.weight")
+    assert bake("lfm3", "blk.0.ffn_up.weight")
+    assert bake("lfm3", "blk.0.ffn_gate.weight")
+    assert bake("lfm3", "blk.0.ffn_down.weight")
+    assert bake("lfm3", "blk.4.attn_gate.weight")
+    assert bake("lfm3", "blk.0.shortconv.in_proj.weight")
+    assert bake("lfm3", "blk.0.shortconv.out_proj.weight")
+    assert bake("lfm3", "blk.0.shortconv.conv.weight")
+    assert not bake("lfm3", "blk.2.attn_q.weight")
+    assert not bake("lfm3", "blk.2.attn_k.weight")
+    assert not bake("lfm3", "blk.2.attn_v.weight")
+    assert not bake("lfm3", "blk.2.attn_output.weight")
+    assert not bake("lfm3", "output.weight")
+    assert scale_bake_notice("lfm3")
+    assert not bake("gemma-embedding2", "blk.0.ffn_up.weight")
+    assert bake("gemma-embedding2", "blk.0.inp_gate.weight")
+    assert scale_bake_notice("gemma3")
+    assert scale_bake_notice("llama") is None
+    assert bake("spark2_5", "blk.0.ffn_up.weight")
+    assert bake("spark2_5", "blk.0.ffn_gate.weight")
+    assert bake("spark2_5", "blk.0.ffn_down.weight")
+    assert bake("spark2_5", "blk.4.attn_gate.weight")
+    assert bake("spark2_5", "output.weight")
+    assert not bake("spark2_5", "blk.0.attn_qkv.weight")
+    assert not bake("spark2_5", "blk.0.attn_output.weight")
+    assert not bake("spark2_5", "output_norm.weight")
+    assert bake("Spark2_5ForCausalLM", "blk.3.ffn_up.weight")
+    assert bake("spark25", "blk.0.ffn_down.weight")
+    assert not bake("muse-spark", "blk.0.ffn_up.weight")
+    assert not bake("dspark", "output.weight")
+    assert scale_bake_notice("spark2_5")
+    spark = quant_profile("Spark2_5ForCausalLM")
+    assert spark is not None and spark.arch == "spark2_5"
+    spark_skip = re.compile(spark.skip)
+    assert spark_skip.search("blk.12.attn_gate.weight")
+    assert spark_skip.search("model.layers.0.self_attn.g_proj.weight")
+    assert not spark_skip.search("blk.0.attn_output.weight")
+    assert not spark_skip.search("blk.0.ffn_gate.weight")
+    assert not spark_skip.search("blk.0.attn_qkv.weight")
+    assert not spark_skip.search("model.layers.0.self_attn.attn_gate_proj.weight")
+    assert quant_profile("llama") is None
+    assert quant_profile("muse-spark") is None
+    assert quant_profile("dspark") is None
+    kept = argparse.Namespace(exclude=None)
+    bind_exclude(kept, "spark2_5")
+    assert kept.exclude_re.search("blk.1.attn_gate.weight")
+    replaced = argparse.Namespace(exclude=r"output\.weight")
+    bind_exclude(replaced, "spark2_5")
+    assert replaced.exclude == r"output\.weight"
+    assert not replaced.exclude_re.search("blk.1.attn_gate.weight")
+    assert looks_like_spark(["model.layers.0.self_attn.q_k_v_proj.weight", "model.layers.0.self_attn.g_proj.weight"])
+    assert not looks_like_spark(["blk.0.attn_qkv.weight", "blk.0.attn_gate.weight"])
+    assert _tight_attention("blk.0.attn_output.weight")
+    assert _tight_attention("blk.0.attn_qkv.weight")
+    assert _tight_attention("model.layers.0.self_attn.o_proj.weight")
+    assert _tight_attention("model.layers.0.self_attn.q_k_v_proj.weight")
+    assert not _tight_attention("blk.0.ffn_down.weight")
+    assert not _tight_attention("blk.0.attn_gate.weight")
+    assert not _tight_attention("")
+
+
+def quantize_and_pack(
+    quantizer: TensorQuantizer, weight: np.ndarray, bake: bool, name: str = "",
+) -> tuple[np.ndarray, np.ndarray | None]:
+    """Quantize one matrix, or a stack of expert matrices, into ggml NVFP4 bytes."""
+    if weight.ndim == 2:
+        codes, block_scale, per_tensor, _dequant = quantizer.quantize_matrix(weight, name)
+        return _pack_matrix(quantizer, codes, block_scale, per_tensor, bake)
+    raws = []
+    scales = []
+    for index in range(weight.shape[0]):
+        codes, block_scale, per_tensor, _dequant = quantizer.quantize_matrix(weight[index], name)
+        raw, scale = _pack_matrix(quantizer, codes, block_scale, per_tensor, bake)
+        raws.append(raw)
+        scales.append(1.0 if scale is None else float(scale[0]))
+    stacked = np.stack(raws, axis=0)
+    return stacked, None if bake else np.asarray(scales, dtype=np.float32)
+
+
+def _pack_matrix(quantizer: TensorQuantizer, codes, block_scale, per_tensor: float, bake: bool):
+    if bake:
+        baked_codes, bits = quantizer.bake_global_scale(codes, block_scale, per_tensor)
+        return pack_nvfp4(baked_codes, bits), None
+    raw = pack_nvfp4(codes, ue4m3_bits(block_scale))
+    return raw, np.asarray([per_tensor], dtype=np.float32)
+
+
+def report_reconstruction(quantizer: TensorQuantizer, weight: np.ndarray, raw: np.ndarray, scale: np.ndarray | None) -> None:
+    sample_w = weight[0] if weight.ndim == 3 else weight
+    sample_raw = raw[0] if raw.ndim == 3 else raw
+    per_tensor = None if scale is None else float(np.reshape(scale, -1)[0])
+    reconstructed = quantizer.runtime_dequant(sample_raw, per_tensor)
+    error = sample_w.astype(np.float32) - reconstructed
+    denom = float(np.max(np.abs(sample_w))) + 1e-8
+    rms = float(np.sqrt(np.mean(np.square(sample_w)))) + 1e-12
+    log.info(
+        "    rel-max %.3e  rmse %.3e  rmse/std %.4f  scale %s",
+        float(np.max(np.abs(error))) / denom,
+        float(np.sqrt(np.mean(np.square(error)))),
+        float(np.sqrt(np.mean(np.square(error)))) / rms,
+        "baked" if scale is None else f"{per_tensor:.4e}",
+    )
+
+
+class ShapeView:
+    def __init__(self, shape: tuple[int, ...]):
+        self.shape = shape
+        self.ndim = len(shape)
+
+
+def eligible_weight(name: str, shape: tuple[int, ...], args, gguf_module) -> bool:
+    """Llama.cpp weight that this project quantizes instead of leaving as BF16."""
+    if not name.endswith(".weight"):
+        return False
+    if len(shape) not in (2, 3):
+        return False
+    if len(shape) == 3 and "exps" not in name and "expert" not in name:
+        return False
+    length = int(shape[-1])
+    rows = int(shape[-2])
+    if length < QK_NVFP4 or length % QK_NVFP4 != 0 or rows < 1:
+        return False
+    lowered = name.lower()
+    if "norm" in lowered or "token_embd" in lowered:
+        return False
+    if name == "output.weight" and not getattr(args, "quantize_output", False):
+        return False
+    exclude = getattr(args, "exclude_re", None)
+    if exclude is not None and exclude.search(name):
+        return False
+    return True
+
+
+def add_quant_arguments(parser: argparse.ArgumentParser) -> None:
+    """Flags shared with convert_to_quant's learned NVFP4 defaults."""
+    parser.add_argument("--device", default="auto", help="auto, cpu, cuda, or cuda:N")
+    parser.add_argument("--simple", action="store_true", help="Round onto the E2M1 grid without learned rounding")
+    parser.add_argument("--optimizer", default="auto", choices=("auto", "prodigy", "adamw", "radam", "original"),
+                        help="auto uses prodigy when it is installed, otherwise AdamW")
+    parser.add_argument("--num-iter", type=int, default=CTQ_NUM_ITER,
+                        help="Learned-rounding iterations per matrix (convert_to_quant default: 4000)")
+    parser.add_argument("--lr", type=float, default=1.0, help="Initial learning rate")
+    parser.add_argument("--lr-schedule", default="plateau", choices=("plateau", "exponential", "adaptive"))
+    parser.add_argument(
+        "--scale-optimization", default=DEFAULT_SCALE_OPTIMIZATION, choices=("fixed", "iterative", "joint"),
+        help="fixed keeps the initial scale. iterative refreshes it on the original optimizer. "
+             "joint trains the block scales with the codes (default: joint)",
+    )
+    parser.add_argument(
+        "--scale-refinement-rounds", type=int, default=DEFAULT_SCALE_REFINEMENT_ROUNDS,
+        help="Scale refreshes when --scale-optimization is iterative (default: 2). Ignored for fixed and joint.",
+    )
+    parser.add_argument("--full-matrix", action="store_true", help="Use a full SVD instead of a low-rank SVD")
+    parser.add_argument("--top-p", type=float, default=0.2)
+    parser.add_argument("--min-k", type=int, default=128)
+    parser.add_argument("--max-k", type=int, default=1280)
+    parser.add_argument("--early-stop-loss", type=float, default=5e-9)
+    parser.add_argument("--early-stop-lr", type=float, default=1.01e-8)
+    parser.add_argument(
+        "--early-stop-stall", type=int, default=DEFAULT_EARLY_STOP_STALL,
+        help="Stop after this many steps without a new best, at any learning rate (default: 2000)",
+    )
+    parser.add_argument(
+        "--target-loss", type=float, default=DEFAULT_TARGET_LOSS,
+        help="Stop when the best loss reaches this value, at any learning rate (default: 1e-5). 0 disables it.",
+    )
+    parser.add_argument(
+        "--early-stop-floor-stall", type=int, default=DEFAULT_EARLY_STOP_FLOOR_STALL,
+        help="If the loss never reaches --target-loss, stop after this many steps without a new best once it has otherwise flattened. 0 disables it (default: 64)",
+    )
+    parser.add_argument(
+        "--lr-min", type=float, default=1e-8,
+        help="Lowest learning rate. The flat-loss check starts here (default: 1e-8).",
+    )
+    parser.add_argument("--use-speed", action="store_true", help="Enable Prodigy's use_speed")
+    parser.add_argument("--calib-samples", type=int, default=3072)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--no-bias-correction", action="store_true")
+    parser.add_argument("--quantize-output", action="store_true", help="Also quantize output.weight")
+    parser.add_argument("--exclude", default=None, help="Regex of tensor names to leave unquantized")
+    parser.add_argument("--convert-to-quant", type=Path, default=None,
+                        help="convert_to_quant checkout, if it is not installed")

requirements.txt

@@ -0,0 +1,19 @@
+# Dependencies for hf_to_nvfp4_gguf.py, ollama_nvfp4.py,
+# safetensors_to_nvfp4_gguf.py, and nvfp4gguf.
+#
+# llama.cpp (GGUF writer and Hugging Face conversion) and convert_to_quant
+# (LearnedNVFP4Converter) are local checkouts, not pip packages. Point the
+# scripts at them with --llama-cpp and --convert-to-quant.
+#
+# Install a CUDA build of torch that matches your GPU. A CPU wheel works
+# with --device cpu.
+
+numpy>=2.2.6
+torch>=2.6.0
+safetensors>=0.4.2
+tqdm>=4.27
+PyYAML>=5.1
+transformers>=4.46
+sentencepiece>=0.1.98,<0.3.0
+protobuf>=4.21.0,<5.0.0
+prodigy-plus-schedule-free

safetensors_to_nvfp4_gguf.py

@@ -0,0 +1,532 @@
+#!/usr/bin/env python3
+"""Quantize a safetensors file to NVFP4 GGUF without a Hugging Face model directory.
+
+The learned rounding is the same convert_to_quant pass the other scripts use,
+so a file you are already quantizing with convert_to_quant can be packed into
+GGUF and compared tensor by tensor. Tensor names are kept. There is no
+tokenizer and no architecture config; the GGUF is a weight container.
+
+By default, tensors that convert_to_quant's base exclusion list would skip
+(norms, biases, embeddings, lm_head, and the other names in that list) stay
+in their original dtype. Pass --all-weights to quantize every 2D matrix whose
+last dimension is a multiple of 64. A sibling bias is corrected the same way
+as hf_to_nvfp4_gguf.py. Repeat --mmproj to copy projector GGUFs beside the
+output and add a FROM line for each one.
+
+--no-quant skips that rounding and writes the safetensors into a GGUF as
+float. --outtype chooses f32, f16, bf16, or auto, which keeps each tensor's
+stored dtype. --outtype nvfp4 repacks a ComfyUI NVFP4 safetensors file
+(packed weight, weight_scale, and weight_scale_2) without fitting it again.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import logging
+import shutil
+from pathlib import Path
+
+import numpy as np
+import torch
+from safetensors import safe_open
+
+from nvfp4gguf import (
+    DEFAULT_LLAMA_CPP,
+    TensorQuantizer,
+    add_quant_arguments,
+    bind_exclude,
+    ctq_skips_key,
+    ensure_cuda_interpreter,
+    import_gguf,
+    load_learned_converter,
+    looks_like_spark,
+    quantize_and_pack,
+    report_reconstruction,
+    require_quant_settings,
+)
+from nvfp4gguf.bundle import (
+    outtype as _outtype,
+    refuse_existing_bundle,
+    require_context_length,
+    warn_missing_gguf_metadata,
+    write_simple_modelfile,
+)
+
+log = logging.getLogger("safetensors_to_nvfp4_gguf")
+
+
+def _checkpoint_file(directory: Path, name: str) -> Path:
+    digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:16]
+    safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in name)[:80]
+    return directory / f"{safe}.{digest}.npz"
+
+
+def _load_checkpoint(directory: Path, name: str, shape: tuple[int, ...], need_bias: bool):
+    path = _checkpoint_file(directory, name)
+    if not path.is_file():
+        return None
+    with np.load(path) as blob:
+        if tuple(int(v) for v in blob["shape"]) != tuple(shape):
+            return None
+        if need_bias and "bias" not in blob.files:
+            return None
+        raw = np.array(blob["raw"], copy=True)
+        scale = np.array(blob["scale"], copy=True)
+        bias = np.array(blob["bias"], copy=True) if "bias" in blob.files else np.zeros((0,), np.float32)
+    return raw, None if scale.size == 0 else scale, None if bias.size == 0 else bias
+
+
+def _save_checkpoint(directory: Path, name: str, shape: tuple[int, ...], raw, scale, bias) -> None:
+    path = _checkpoint_file(directory, name)
+    temporary = path.with_name(path.name + ".tmp")
+    with temporary.open("wb") as handle:
+        np.savez_compressed(
+            handle,
+            shape=np.asarray(shape, dtype=np.int64),
+            raw=raw,
+            scale=np.zeros((0,), np.float32) if scale is None else np.asarray(scale, dtype=np.float32),
+            bias=np.zeros((0,), np.float32) if bias is None else np.asarray(bias, dtype=np.float32),
+        )
+    temporary.replace(path)
+
+
+def _should_quantize(
+    key: str,
+    shape: tuple[int, ...],
+    all_weights: bool,
+    convert_to_quant_dir: Path | None,
+    exclude_re,
+) -> bool:
+    if len(shape) != 2 or not key.endswith(".weight"):
+        return False
+    if shape[-1] % 64 != 0 or shape[-1] < 64:
+        return False
+    if exclude_re is not None and exclude_re.search(key):
+        return False
+    if not all_weights and ctq_skips_key(key, convert_to_quant_dir):
+        return False
+    return True
+
+
+def _apply_bias(name: str, tensor: torch.Tensor, correction: np.ndarray | None) -> torch.Tensor:
+    if correction is None:
+        return tensor
+    values = tensor.detach().cpu().float()
+    flat = values.reshape(-1)
+    if tuple(flat.shape) != tuple(correction.shape):
+        log.warning("    bias %s shape %s does not match correction %s", name, tuple(flat.shape), tuple(correction.shape))
+        return tensor
+    flat = flat - torch.from_numpy(np.ascontiguousarray(correction, dtype=np.float32))
+    values = flat.reshape(values.shape).to(dtype=tensor.dtype)
+    log.info("    bias %s adjusted (mean %+.3e)", name, float(np.mean(correction)))
+    return values
+
+
+def _float_file_type(gguf, outtype: str):
+    return {
+        "f32": gguf.LlamaFileType.ALL_F32,
+        "f16": gguf.LlamaFileType.MOSTLY_F16,
+        "bf16": gguf.LlamaFileType.MOSTLY_BF16,
+    }[outtype]
+
+
+def _auto_file_type(gguf, dtypes: list[str]):
+    names = {dtype.upper() for dtype in dtypes}
+    if "BF16" in names:
+        return gguf.LlamaFileType.MOSTLY_BF16
+    if "F16" in names:
+        return gguf.LlamaFileType.MOSTLY_F16
+    if "F32" in names:
+        return gguf.LlamaFileType.ALL_F32
+    return gguf.LlamaFileType.GUESSED
+
+
+def _cast_float(tensor: torch.Tensor, outtype: str) -> torch.Tensor:
+    if outtype == "auto" or not tensor.is_floating_point():
+        return tensor
+    dtype = {"f32": torch.float32, "f16": torch.float16, "bf16": torch.bfloat16}[outtype]
+    if tensor.dtype == dtype:
+        return tensor
+    return tensor.to(dtype)
+
+
+def _default_output(source: Path, skip_quant: bool, outtype: str) -> Path:
+    if not skip_quant:
+        return source.with_name(source.stem + "-NVFP4.gguf")
+    if outtype == "auto":
+        return source.with_name(source.stem + ".gguf")
+    label = "NVFP4" if outtype == "nvfp4" else outtype.upper()
+    return source.with_name(f"{source.stem}-{label}.gguf")
+
+
+def _add_original(writer, gguf, name: str, tensor: torch.Tensor) -> None:
+    tensor = tensor.detach().cpu().contiguous()
+    if tensor.dtype == torch.bfloat16:
+        raw = tensor.view(torch.uint16).numpy().view(np.uint8)
+        writer.add_tensor(name, np.ascontiguousarray(raw), raw_dtype=gguf.GGMLQuantizationType.BF16)
+        return
+    if tensor.dtype == torch.float16:
+        writer.add_tensor(name, tensor.numpy())
+        return
+    if tensor.dtype == torch.float32:
+        writer.add_tensor(name, tensor.numpy())
+        return
+    writer.add_tensor(name, tensor.float().numpy())
+
+
+def _prepare_output(args) -> tuple[Path, Path, bool]:
+    source = Path(args.safetensors).resolve()
+    if not source.is_file():
+        raise SystemExit(f"Not a file: {source}")
+    skip_quant = bool(args.no_quant or args.outtype == "nvfp4")
+    output = Path(args.output).resolve() if args.output else _default_output(source, skip_quant, args.outtype)
+    return source, output, skip_quant
+
+
+def _comfy_formats(handle) -> dict[str, str]:
+    """Map a weight prefix to the format named in its .comfy_quant sidecar."""
+    found: dict[str, str] = {}
+    suffix = ".comfy_quant"
+    for key in handle.keys():
+        if not key.endswith(suffix):
+            continue
+        raw = handle.get_tensor(key).detach().cpu().numpy().tobytes()
+        try:
+            payload = json.loads(raw.decode("utf-8"))
+        except (UnicodeError, json.JSONDecodeError):
+            continue
+        fmt = payload.get("format") if isinstance(payload, dict) else None
+        if isinstance(fmt, str):
+            found[key[: -len(suffix)]] = fmt
+    return found
+
+
+def _pack_comfy_nvfp4(weight: torch.Tensor, scale: torch.Tensor) -> np.ndarray:
+    """Repack ComfyUI / ModelOpt NVFP4 into ggml super-blocks.
+
+    The code bytes are already nibble-packed the way ggml stores them. Each
+    group of four 16-value blocks becomes 4 UE4M3 scale bytes followed by
+    those 32 code bytes.
+    """
+    if scale.ndim != 2 or weight.ndim != 2:
+        raise ValueError(f"NVFP4 weight {tuple(weight.shape)} and scale {tuple(scale.shape)} must be matrices")
+    rows, n_blocks = scale.shape
+    if weight.shape != (rows, n_blocks * 8):
+        raise ValueError(f"NVFP4 weight {tuple(weight.shape)} does not match scale {tuple(scale.shape)}")
+    if n_blocks % 4 != 0:
+        raise ValueError(f"NVFP4 scale count {n_blocks} is not a multiple of 4")
+    n_super = n_blocks // 4
+    codes = np.ascontiguousarray(weight.detach().cpu().contiguous().view(torch.uint8).numpy()).reshape(rows, n_super, 32)
+    scales = np.ascontiguousarray(scale.detach().cpu().contiguous().view(torch.uint8).numpy()).reshape(rows, n_super, 4)
+    scales = scales & np.uint8(0x7F)
+    return np.ascontiguousarray(np.concatenate([scales, codes], axis=-1).reshape(rows, n_super * 36))
+
+
+def _scale_is_trivial(scale: torch.Tensor) -> bool:
+    return scale.numel() <= 1 and abs(float(scale.detach().float().sum()) - 1.0) < 1e-6
+
+
+def _float8_to_bf16(tensor: torch.Tensor, scale: torch.Tensor | None) -> torch.Tensor:
+    values = tensor.detach().float()
+    if scale is not None:
+        factor = scale.detach().float()
+        if factor.numel() == 1:
+            values = values * factor.reshape(())
+        elif tuple(factor.shape) == tuple(values.shape):
+            values = values * factor
+        else:
+            raise ValueError(f"float8 scale {tuple(factor.shape)} does not match weight {tuple(values.shape)}")
+    return values.to(torch.bfloat16)
+
+
+def convert_packed_nvfp4(args) -> None:
+    """Copy ComfyUI NVFP4 blocks into a GGUF. Float tensors are stored as stored."""
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s")
+    source, output, _skip = _prepare_output(args)
+    overwrite = bool(args.force or args.overwrite)
+    if output.exists() and not overwrite:
+        raise SystemExit(f"{output} already exists. Pass --overwrite to replace it.")
+    output.parent.mkdir(parents=True, exist_ok=True)
+    if args.mmproj:
+        modelfile = output.parent / "Modelfile"
+        if modelfile.exists() and not overwrite:
+            raise SystemExit(f"{modelfile} already exists. Pass --overwrite to replace it.")
+        refuse_existing_bundle(output, overwrite, args.mmproj, skip_gguf=True)
+
+    gguf = import_gguf(args.llama_cpp)
+    with safe_open(source, framework="pt", device="cpu") as handle:
+        keys = list(handle.keys())
+        formats = _comfy_formats(handle)
+    packed = [name for name, fmt in formats.items() if fmt == "nvfp4"]
+    if not packed:
+        raise SystemExit(
+            f"{source} has no ComfyUI NVFP4 weights to repack. "
+            "Use --no-quant with --outtype auto, f16, bf16, or f32 to write float weights without quantizing them."
+        )
+    present = set(keys)
+    for name in packed:
+        if name + ".weight" not in present or name + ".weight_scale" not in present:
+            raise SystemExit(f"{name} is marked nvfp4 but is missing weight or weight_scale")
+    log.info("Repacking %d NVFP4 tensors from %s. Learned rounding is skipped.", len(packed), source.name)
+    writer = gguf.GGUFWriter(output, "safetensors", use_temp_file=not args.no_temp_file)
+    writer.add_file_type(int(gguf.LlamaFileType.MOSTLY_NVFP4))
+    writer.add_string("general.name", source.stem)
+    writer.add_string("general.description", "NVFP4 weights repacked from safetensors without refitting")
+    writer.add_uint32("general.quantization_version", 2)
+    context_length = require_context_length(args.context_length)
+    if context_length is not None:
+        writer.add_context_length(context_length)
+        log.info("context length %d", context_length)
+
+    skip = {name + ".comfy_quant" for name in formats}
+    for name, fmt in formats.items():
+        if fmt in ("nvfp4", "float8_e4m3fn"):
+            skip.add(name + ".weight_scale")
+            skip.add(name + ".weight_scale_2")
+    promoted = 0
+    with safe_open(source, framework="pt", device="cpu") as handle:
+        for key in keys:
+            if key in skip:
+                continue
+            if key.endswith(".weight") and formats.get(key[: -len(".weight")]) == "nvfp4":
+                base = key[: -len(".weight")]
+                raw = _pack_comfy_nvfp4(handle.get_tensor(key), handle.get_tensor(base + ".weight_scale"))
+                writer.add_tensor(key, raw, raw_dtype=gguf.GGMLQuantizationType.NVFP4)
+                scale2_name = base + ".weight_scale_2"
+                if scale2_name in present:
+                    scale2 = handle.get_tensor(scale2_name)
+                    if not _scale_is_trivial(scale2):
+                        writer.add_tensor(base + ".scale", scale2.detach().float().numpy().reshape(-1).astype(np.float32))
+                continue
+            if key.endswith(".weight") and formats.get(key[: -len(".weight")]) == "float8_e4m3fn":
+                base = key[: -len(".weight")]
+                scale_name = base + ".weight_scale"
+                scale = handle.get_tensor(scale_name) if scale_name in present else None
+                _add_original(writer, gguf, key, _float8_to_bf16(handle.get_tensor(key), scale))
+                promoted += 1
+                log.info("Stored %s as BF16 (source is float8_e4m3fn)", key)
+                continue
+            _add_original(writer, gguf, key, handle.get_tensor(key))
+    if promoted:
+        log.info("%d float8 tensors were stored as BF16. GGUF has no float8 weight type.", promoted)
+    log.info("Writing %s", output)
+    warn_missing_gguf_metadata(writer)
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file(progress=True)
+    writer.close()
+    if args.mmproj:
+        write_simple_modelfile(output, overwrite, args.mmproj)
+    log.info("Wrote %s", output)
+
+
+def convert_float(args) -> None:
+    """Write the safetensors into a GGUF without learned rounding."""
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s")
+    if args.outtype == "nvfp4":
+        convert_packed_nvfp4(args)
+        return
+    source, output, _skip = _prepare_output(args)
+    overwrite = bool(args.force or args.overwrite)
+    if output.exists() and not overwrite:
+        raise SystemExit(f"{output} already exists. Pass --overwrite to replace it.")
+    output.parent.mkdir(parents=True, exist_ok=True)
+    if args.mmproj:
+        modelfile = output.parent / "Modelfile"
+        if modelfile.exists() and not overwrite:
+            raise SystemExit(f"{modelfile} already exists. Pass --overwrite to replace it.")
+        refuse_existing_bundle(output, overwrite, args.mmproj, skip_gguf=True)
+
+    gguf = import_gguf(args.llama_cpp)
+    with safe_open(source, framework="pt", device="cpu") as handle:
+        keys = list(handle.keys())
+        dtypes = [handle.get_slice(key).get_dtype() for key in keys]
+    file_type = _float_file_type(gguf, args.outtype) if args.outtype != "auto" else _auto_file_type(gguf, dtypes)
+    log.info("Writing float GGUF (%s). NVFP4 quantization is skipped.", args.outtype)
+    writer = gguf.GGUFWriter(output, "safetensors", use_temp_file=not args.no_temp_file)
+    writer.add_file_type(int(file_type))
+    writer.add_string("general.name", source.stem)
+    writer.add_string("general.description", "Weights converted from safetensors without quantization")
+    context_length = require_context_length(args.context_length)
+    if context_length is not None:
+        writer.add_context_length(context_length)
+        log.info("context length %d", context_length)
+    with safe_open(source, framework="pt", device="cpu") as handle:
+        for key in keys:
+            _add_original(writer, gguf, key, _cast_float(handle.get_tensor(key), args.outtype))
+    log.info("Writing %s", output)
+    warn_missing_gguf_metadata(writer)
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file(progress=True)
+    writer.close()
+    if args.mmproj:
+        write_simple_modelfile(output, overwrite, args.mmproj)
+    log.info("Wrote %s", output)
+
+
+def convert(args) -> None:
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s")
+    if args.no_quant or args.outtype == "nvfp4":
+        convert_float(args)
+        return
+    ensure_cuda_interpreter(args.device)
+    source, output, _skip = _prepare_output(args)
+    overwrite = bool(args.force or args.overwrite)
+    if output.exists() and not overwrite:
+        raise SystemExit(f"{output} already exists. Pass --overwrite to replace it.")
+    output.parent.mkdir(parents=True, exist_ok=True)
+    if args.mmproj:
+        modelfile = output.parent / "Modelfile"
+        if modelfile.exists() and not overwrite:
+            raise SystemExit(f"{modelfile} already exists. Pass --overwrite to replace it.")
+        refuse_existing_bundle(output, overwrite, args.mmproj, skip_gguf=True)
+
+    gguf = import_gguf(args.llama_cpp)
+    with safe_open(source, framework="pt", device="cpu") as handle:
+        keys = list(handle.keys())
+        shapes = {key: tuple(handle.get_slice(key).get_shape()) for key in keys}
+    bind_exclude(args, "spark2_5" if looks_like_spark(keys) else None)
+    chosen = [
+        key for key in keys
+        if _should_quantize(key, shapes[key], args.all_weights, args.convert_to_quant, args.exclude_re)
+    ]
+    log.info("%d of %d tensors queued for NVFP4", len(chosen), len(keys))
+    if not chosen:
+        raise SystemExit("No 2D .weight tensor has a last dimension divisible by 64.")
+
+    work = Path(args.checkpoint_dir) if args.checkpoint_dir else output.with_name(output.name + ".nvfp4-work")
+    if args.fresh and work.exists():
+        shutil.rmtree(work)
+    work.mkdir(parents=True, exist_ok=True)
+    require_quant_settings(work, args)
+    quantizer = TensorQuantizer(args, gguf, load_learned_converter(args.convert_to_quant)[1])
+    writer = gguf.GGUFWriter(output, "safetensors", use_temp_file=not args.no_temp_file)
+    writer.add_file_type(int(gguf.LlamaFileType.MOSTLY_NVFP4))
+    writer.add_string("general.name", source.stem)
+    writer.add_string("general.description", "NVFP4 weights converted from safetensors for comparison with convert_to_quant")
+    writer.add_uint32("general.quantization_version", 2)
+    context_length = require_context_length(args.context_length)
+    if context_length is not None:
+        writer.add_context_length(context_length)
+        log.info("context length %d", context_length)
+
+    done = 0
+    chosen_set = set(chosen)
+    pending_bias: dict[str, torch.Tensor] = {}
+    corrections: dict[str, np.ndarray] = {}
+
+    def _write_bias(key: str, tensor: torch.Tensor) -> None:
+        weight_name = key[: -len(".bias")] + ".weight"
+        correction = corrections.get(key) if weight_name in chosen_set and not args.no_bias_correction else None
+        _add_original(writer, gguf, key, _apply_bias(key, tensor, correction))
+
+    with safe_open(source, framework="pt", device="cpu") as handle:
+        for key in keys:
+            tensor = handle.get_tensor(key)
+            weight_name = key[: -len(".bias")] + ".weight" if key.endswith(".bias") else ""
+            if weight_name in chosen_set and not args.no_bias_correction and key not in corrections:
+                pending_bias[key] = tensor
+                continue
+            if key not in chosen_set:
+                if len(shapes[key]) == 2 and key.endswith(".weight") and shapes[key][-1] % 64 != 0:
+                    log.info("Keeping %s %s (last dimension is not a multiple of 64)", key, shapes[key])
+                if key.endswith(".bias") and weight_name in chosen_set:
+                    _write_bias(key, tensor)
+                else:
+                    _add_original(writer, gguf, key, tensor)
+                continue
+            done += 1
+            bias_name = key[: -len(".weight")] + ".bias"
+            need_bias = not args.no_bias_correction and bias_name in shapes and len(shapes[key]) == 2
+            cached = _load_checkpoint(work, key, shapes[key], need_bias)
+            log.info(
+                "NVFP4 [%d/%d] %s  %s%s",
+                done, len(chosen), key, shapes[key], "  (checkpoint)" if cached else "",
+            )
+            if cached is None:
+                weight = tensor.float().numpy()
+                raw, scale = quantize_and_pack(quantizer, weight, False, key)
+                report_reconstruction(quantizer, weight, raw, scale)
+                bias = None
+                if need_bias:
+                    per_tensor = None if scale is None else float(np.reshape(scale, -1)[0])
+                    runtime = quantizer.runtime_dequant(raw, per_tensor)
+                    bias = quantizer.bias_correction(weight, runtime)
+                _save_checkpoint(work, key, shapes[key], raw, scale, bias)
+                del weight
+            else:
+                raw, scale, bias = cached
+            if bias is not None:
+                corrections[bias_name] = bias
+                pending = pending_bias.pop(bias_name, None)
+                if pending is not None:
+                    _write_bias(bias_name, pending)
+            writer.add_tensor(key, raw, raw_dtype=gguf.GGMLQuantizationType.NVFP4)
+            if scale is not None:
+                writer.add_tensor(key[: -len(".weight")] + ".scale", scale)
+            del tensor, raw
+        for key, tensor in pending_bias.items():
+            _write_bias(key, tensor)
+
+    log.info("Writing %s", output)
+    warn_missing_gguf_metadata(writer)
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file(progress=True)
+    writer.close()
+    if not args.keep_checkpoints:
+        shutil.rmtree(work, ignore_errors=True)
+    if args.mmproj:
+        write_simple_modelfile(output, overwrite, args.mmproj)
+    log.info("Wrote %s", output)
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Quantize a safetensors file to an NVFP4 GGUF weight container.")
+    parser.add_argument("safetensors", nargs="?", help="Input .safetensors file")
+    parser.add_argument("-o", "--output", help="Output GGUF path. Default: <stem>-NVFP4.gguf, or <stem>.gguf with --no-quant and auto")
+    parser.add_argument("--no-quant", action="store_true",
+                        help="Skip learned rounding. Writes a float GGUF, or repacks ComfyUI NVFP4 weights with --outtype nvfp4")
+    parser.add_argument("--outtype", default="auto", type=_outtype, choices=("auto", "f32", "f16", "bf16", "nvfp4"),
+                        help="Type used when learned rounding is skipped. nvfp4 repacks ComfyUI NVFP4 weights. auto keeps each tensor's stored dtype (default: auto)")
+    parser.add_argument("--all-weights", action="store_true",
+                        help="Quantize every eligible 2D weight. By default, convert_to_quant's exclusion list is skipped too.")
+    parser.add_argument("--llama-cpp", type=Path, default=DEFAULT_LLAMA_CPP,
+                        help="llama.cpp checkout used for the GGUF writer (default: ~/llama.cpp)")
+    parser.add_argument("--overwrite", action="store_true", help="Replace an existing GGUF, Modelfile, or copied projector")
+    parser.add_argument("--force", action="store_true", help="Same as --overwrite")
+    parser.add_argument("--mmproj", action="append", type=Path, default=None,
+                        help="Vision projector GGUF. Copied next to the output under its own name and added as another FROM line. Repeat for more than one projector.")
+    parser.add_argument("--context-length", type=int, default=None,
+                        help="Max context stored in the GGUF. A safetensors file has no context length to copy.")
+    parser.add_argument("--checkpoint-dir", type=Path, default=None, help="Per-tensor resume directory")
+    parser.add_argument("--fresh", action="store_true")
+    parser.add_argument("--keep-checkpoints", action="store_true")
+    parser.add_argument("--no-temp-file", action="store_true")
+    parser.add_argument("--verbose", action="store_true")
+    parser.add_argument("--self-test", action="store_true", help="Check NVFP4 packing against llama.cpp's dequantizer and exit")
+    add_quant_arguments(parser)
+    args = parser.parse_args(argv)
+    if args.self_test:
+        if args.device == "auto":
+            args.device = "cpu"
+        return args
+    if not args.safetensors:
+        parser.error("safetensors file is required")
+    return args
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = parse_args(argv)
+    if args.self_test:
+        from hf_to_nvfp4_gguf import self_test
+        self_test(args)
+        return
+    convert(args)
+
+
+if __name__ == "__main__":
+    main()