From 416b0e630f9cbe5c4a8edf53447846fb6334e063 Mon Sep 17 00:00:00 2001 From: zefie Date: Wed, 4 Nov 2026 21:43:53 -0500 Subject: [PATCH] fix/improve hf to hf conversions --- Dockerfile | 8 +- README.md | 8 +- docker/README.txt | 22 +- docker/gguf_to_nvfp4_hf | 3 + docker/hf_to_nvfp4_hf | 3 + gguf_to_nvfp4_hf.py | 761 +++++++++++++++++++++++++++++++++ hf_to_nvfp4_hf.py | 620 +++++++++++++++++++++++++++ man/man1/gguf_to_nvfp4_hf.1 | 241 +++++++++++ man/man1/gguf_to_nvfp4_hf.py.1 | 1 + man/man1/hf_to_nvfp4_hf.1 | 231 ++++++++++ man/man1/hf_to_nvfp4_hf.py.1 | 1 + nvfp4_quant | 3 - 12 files changed, 1885 insertions(+), 17 deletions(-) create mode 100755 docker/gguf_to_nvfp4_hf create mode 100755 docker/hf_to_nvfp4_hf create mode 100755 gguf_to_nvfp4_hf.py create mode 100755 hf_to_nvfp4_hf.py create mode 100644 man/man1/gguf_to_nvfp4_hf.1 create mode 120000 man/man1/gguf_to_nvfp4_hf.py.1 create mode 100644 man/man1/hf_to_nvfp4_hf.1 create mode 120000 man/man1/hf_to_nvfp4_hf.py.1 diff --git a/Dockerfile b/Dockerfile index 4c1f7c8..24d5d4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,8 +72,8 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install torch RUN git clone --depth=1 https://github.com/silveroxides/convert_to_quant && \ cd convert_to_quant && pip3 install -r requirements.txt && pip3 install . -# Install heretic-llm -RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm +# Install heretic-llm and compressed-tensors +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip3 install heretic-llm compressed-tensors # Install zefie's nvfp4 quant requirements COPY --chmod=0644 requirements.txt /workspace/requirements-nvfp4.txt @@ -105,14 +105,14 @@ WORKDIR /opt/nvfp4 RUN mkdir nvfp4_gguf_quant COPY --exclude=__pycache__ nvfp4gguf ./nvfp4_gguf_quant/nvfp4gguf -COPY --chmod=0644 gguf_to_nvfp4_gguf.py hf_to_nvfp4_gguf.py hf_to_nvfp4_safetensors.py safetensors_to_nvfp4_gguf.py hf_info.py gguf_info.py ./nvfp4_gguf_quant/ +COPY --chmod=0644 --exclude=ollama_nvfp4.py *.py ./nvfp4_gguf_quant/ COPY --chmod=0644 docker/README.txt /workspace/README.txt COPY --chmod=0755 --exclude=README.txt docker/* /usr/local/bin/ # Install manpages for zefie's nvfp4 quant scripts, ctq and heretic RUN mkdir -p /usr/local/share/man/man1 -COPY --chmod=0644 man/man1/* /usr/local/share/man/man1/ +COPY --chmod=0644 --exclude=man/man1/ollama_nvfp4.1 --exclude=man/man1/ollama_nvfp4.py.1 man/man1/* /usr/local/share/man/man1/ # Copy llama.cpp compiled binaries, manpages and shared libraries from builder COPY --from=builder /install/bin/ /usr/local/bin/ diff --git a/README.md b/README.md index 71988d0..9d6d063 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,12 @@ These scripts require [llama.cpp](https://github.com/ggml-org/llama.cpp) and [Co `hf`, the Hugging Face CLI command is also suggested, but not required. For quantizations or conversions, try: -- `hf_to_nvfp4_quant` (huggingface structure to GGUF) -- `gguf_to_nvfp4_quant` (GGUF to GGUF) -- `safetensors_to_nvfp4_quant` (ComfyUI style safetensors to GGUF (untested)) +- `hf_to_nvfp4_gguf` (huggingface structure to GGUF) +- `gguf_to_nvfp4_gguf` (GGUF to GGUF) +- `safetensors_to_nvfp4_gguf` (ComfyUI style safetensors to GGUF (untested)) - `hf_to_nvfp4_safetensors` (huggingface structure to ComfyUI style safetensors) +- `hf_to_nvfp4_hf` (huggingface structure to NVFP4 Hugging Face structure) +- `gguf_to_nvfp4_hf` (GGUF to NVFP4 Hugging Face structure) For model info, try: - `hf_info` (huggingface structure) diff --git a/docker/README.txt b/docker/README.txt index 25b8ecd..c4ea682 100644 --- a/docker/README.txt +++ b/docker/README.txt @@ -1,11 +1,13 @@ Convert-to-Quant (ctq) is installed for normal quantizations. -For GGUF quantizations, try: +For quantizations and conversions, try: -hf_to_nvfp4_quant -gguf_to_nvfp4_quant -safetensors_to_nvfp4_quant -hf_to_nvfp4_safetensors +- hf_to_nvfp4_gguf (Hugging Face to GGUF) +- gguf_to_nvfp4_gguf (GGUF to GGUF) +- safetensors_to_nvfp4_gguf (ComfyUI safetensors to GGUF) +- hf_to_nvfp4_safetensors (Hugging Face to ComfyUI safetensors) +- hf_to_nvfp4_hf (Hugging Face to NVFP4 Hugging Face) +- gguf_to_nvfp4_hf (GGUF to NVFP4 Hugging Face) Depending on the source material. @@ -15,8 +17,14 @@ hf cli command is also installed, eg you can: hf download coder3101/gemma-4-12B-it-heretic --local-dir gemma-4-12B-it-heretic hf_to_nvfp4_gguf -o gemma-4-12B-it-heretic_nvfp4 gemma-4-12B-it-heretic -The same directory can be quantized to one ComfyUI NVFP4 safetensors file: +The same directory can be quantized to NVFP4 Hugging Face format: + +hf_to_nvfp4_hf -o gemma-4-12B-it-heretic-NVFP4-HF gemma-4-12B-it-heretic + +Or to one ComfyUI NVFP4 safetensors file: hf_to_nvfp4_safetensors -o gemma-4-12B-it-heretic-NVFP4.safetensors gemma-4-12B-it-heretic -This would create a NVFP4 GGUF for Ollama/llama.cpp from https://huggingface.co/coder3101/gemma-4-12B-it-heretic +A GGUF file can also be converted to NVFP4 Hugging Face format: + +gguf_to_nvfp4_hf -o gemma-4-12B-it-heretic-NVFP4-HF gemma-4-12B-it-heretic.gguf diff --git a/docker/gguf_to_nvfp4_hf b/docker/gguf_to_nvfp4_hf new file mode 100755 index 0000000..78ac6d8 --- /dev/null +++ b/docker/gguf_to_nvfp4_hf @@ -0,0 +1,3 @@ +#!/bin/bash +python /opt/nvfp4/nvfp4_gguf_quant/gguf_to_nvfp4_hf.py --llama-cpp /opt/nvfp4/llama.cpp --convert-to-quant /opt/nvfp4/convert_to_quant "${@}" + diff --git a/docker/hf_to_nvfp4_hf b/docker/hf_to_nvfp4_hf new file mode 100755 index 0000000..a143c97 --- /dev/null +++ b/docker/hf_to_nvfp4_hf @@ -0,0 +1,3 @@ +#!/bin/bash +python /opt/nvfp4/nvfp4_gguf_quant/hf_to_nvfp4_hf.py --llama-cpp /opt/nvfp4/llama.cpp --convert-to-quant /opt/nvfp4/convert_to_quant "${@}" + diff --git a/gguf_to_nvfp4_hf.py b/gguf_to_nvfp4_hf.py new file mode 100755 index 0000000..b196efe --- /dev/null +++ b/gguf_to_nvfp4_hf.py @@ -0,0 +1,761 @@ +"""Convert and quantize a GGUF model to NVFP4 Hugging Face format. + +Reads a GGUF file and writes a Hugging Face format model directory (config.json + +safetensors). By default (--format hf), writes standard Hugging Face compressed-tensors +format with .weight_packed, .weight_scale, and .weight_global_scale natively supported +by Hugging Face transformers, vLLM, SGLang, and llama.cpp. + +Optionally (--format modelopt / comfy), outputs raw weight tensors (.weight), +weight_scale, weight_scale_2, and .comfy_quant for ModelOpt and ComfyUI. + +If the source GGUF contains float or non-NVFP4 weights, weight matrices are quantized +to NVFP4 using convert_to_quant's learned rounding (or simple round-to-grid with --simple). +If the source GGUF is already NVFP4, weights are repacked into safetensors without loss +of precision. + +If --no-quant is specified, weights are dequantized and written to safetensors in the +target float format (--outtype f32, f16, bf16, or auto). +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import logging +import os +import re +import shutil +import sys +from pathlib import Path + +import numpy as np +import torch +from safetensors.torch import save_file + +from nvfp4gguf import ( + CTQ_NUM_ITER, + DEFAULT_CONVERT_TO_QUANT, + DEFAULT_LLAMA_CPP, + DEFAULT_EARLY_STOP_FLOOR_STALL, + DEFAULT_TARGET_LOSS, + DEFAULT_EARLY_STOP_STALL, + DEFAULT_LR_FLOOR, + DEFAULT_LR_PATIENCE, + DEFAULT_LR_THRESHOLD, + DEFAULT_SCALE_OPTIMIZATION, + DEFAULT_SCALE_REFINEMENT_ROUNDS, + QUANT_VERSION, + TensorQuantizer, + add_quant_arguments, + bind_exclude, + coordinate_descent_refine, + ctq_skips_key, + ensure_cuda_interpreter, + load_learned_converter, + quant_profile, + report_reconstruction, + resolve_optimizer, + sweep_per_tensor_scale, + ue4m3_bits, +) + +INDEX_NAME = "model.safetensors.index.json" +MAX_SHARD_BYTES = 5 * 1024 * 1024 * 1024 # 5 GB max shard size + +log = logging.getLogger("gguf_to_nvfp4_hf") + + +def _checkpoint_file(directory: Path, name: str) -> Path: + digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:16] + safe = re.sub(r"[^A-Za-z0-9._-]+", "_", name)[:80] + return directory / f"{safe}.{digest}.npz" + + +def _load_checkpoint(directory: Path, name: str, shape: tuple[int, ...], need_bias: bool): + path = _checkpoint_file(directory, name) + if not path.is_file(): + return None + with np.load(path) as blob: + if "version" in blob.files and int(blob["version"]) != QUANT_VERSION: + return None + if tuple(int(v) for v in blob["shape"]) != tuple(shape): + return None + if need_bias and "bias" not in blob.files: + return None + codes = np.array(blob["codes"], copy=True) + block_scale = np.array(blob["block_scale"], copy=True) + per_tensor = float(blob["per_tensor"]) + bias = np.array(blob["bias"], copy=True) if "bias" in blob.files else None + return codes, block_scale, per_tensor, bias + + +def _save_checkpoint(directory: Path, name: str, shape: tuple[int, ...], codes, block_scale, per_tensor, bias) -> None: + path = _checkpoint_file(directory, name) + temporary = path.with_name(path.name + ".tmp") + with temporary.open("wb") as handle: + save_dict = { + "version": np.int32(QUANT_VERSION), + "shape": np.asarray(shape, dtype=np.int64), + "codes": np.asarray(codes, dtype=np.uint8), + "block_scale": np.asarray(block_scale, dtype=np.float32), + "per_tensor": np.float32(per_tensor), + } + if bias is not None: + save_dict["bias"] = np.asarray(bias, dtype=np.float32) + np.savez_compressed(handle, **save_dict) + temporary.replace(path) + + +def _pythonize(value): + if isinstance(value, np.generic): + return value.item() + if isinstance(value, np.ndarray): + return value.tolist() + if isinstance(value, list): + return [_pythonize(item) for item in value] + return value + + +def _text(value) -> str: + value = _pythonize(value) + if isinstance(value, bytes): + return value.decode("utf-8", "replace") + if isinstance(value, list): + return "".join(_text(item) for item in value) + return str(value) + + +def _to_float(tensor, gguf_module) -> np.ndarray: + qtype = tensor.tensor_type + if qtype == gguf_module.GGMLQuantizationType.F32: + values = np.asarray(tensor.data, dtype=np.float32) + elif qtype == gguf_module.GGMLQuantizationType.F16: + values = np.asarray(tensor.data, dtype=np.float32) + elif qtype == gguf_module.GGMLQuantizationType.BF16: + values = np.asarray(tensor.data, dtype=np.float32) + else: + values = np.asarray(gguf_module.quants.dequantize(tensor.data, qtype), dtype=np.float32) + return np.array(values, dtype=np.float32, copy=True) + + +def _target_dtype(dtype_str: str) -> torch.dtype: + mapping = { + "float32": torch.float32, + "f32": torch.float32, + "float16": torch.float16, + "f16": torch.float16, + "bfloat16": torch.bfloat16, + "bf16": torch.bfloat16, + } + return mapping.get(dtype_str.lower(), torch.bfloat16) + + +def _assign_output(args) -> None: + source = Path(args.gguf) + stem = source.stem + passthrough = args.no_quant or args.outtype == "nvfp4" + if not args.output: + if passthrough and args.outtype == "auto": + directory = Path.cwd() / stem + elif passthrough: + label = "NVFP4" if args.outtype == "nvfp4" else args.outtype.upper() + directory = Path.cwd() / f"{stem}-{label}" + else: + directory = Path.cwd() / f"{stem}-NVFP4" + else: + directory = Path(args.output).resolve() + args.output = str(directory) + + +def pack_nvfp4_hf(codes: np.ndarray, block_scales: np.ndarray, per_tensor: float): + """Pack codes and scales into NVFP4 tensors.""" + packed = (codes[:, 0::2] & 0x0F) | ((codes[:, 1::2] & 0x0F) << 4) + packed_tensor = torch.from_numpy(np.ascontiguousarray(packed, dtype=np.uint8)) + + scale_bits = ue4m3_bits(block_scales) + scale_tensor = torch.from_numpy(np.ascontiguousarray(scale_bits, dtype=np.uint8)).view(torch.float8_e4m3fn) + scale_2_tensor = torch.tensor(float(per_tensor), dtype=torch.float32) + + return packed_tensor, scale_tensor, scale_2_tensor + + +def unpack_gguf_nvfp4_to_hf(raw: np.ndarray, per_tensor: float): + """Convert GGUF NVFP4 super-blocks (36 bytes / 64 values) into ModelOpt tensors.""" + rows = raw.shape[0] + n_super = raw.shape[1] // 36 + n_blocks = n_super * 4 + cols = n_blocks * 16 + + raw_t = torch.from_numpy(raw).reshape(rows, n_super, 36) + d_ue = raw_t[:, :, :4].reshape(rows, n_blocks) + qs = raw_t[:, :, 4:].reshape(rows, n_blocks, 8) + + vals = torch.empty((rows, n_blocks, 16), dtype=torch.uint8) + vals[:, :, :8] = qs & 0x0F + vals[:, :, 8:] = qs >> 4 + + w = ((vals[:, :, 0::2] & 0x0F) | ((vals[:, :, 1::2] & 0x0F) << 4)).reshape(rows, cols // 2) + + scale_tensor = d_ue.contiguous().view(torch.float8_e4m3fn) + scale_2_tensor = torch.tensor(float(per_tensor), dtype=torch.float32) + return w, scale_tensor, scale_2_tensor + + +def unpermute_rope(weights: np.ndarray, n_head: int, n_kv_head: int | None = None) -> np.ndarray: + """Invert RoPE permutation applied during HF to GGUF conversion.""" + head_count = n_kv_head if n_kv_head is not None else n_head + if head_count <= 0 or weights.shape[0] % head_count != 0: + return weights + head_dim = weights.shape[0] // head_count + if head_dim % 2 != 0: + return weights + w = weights.reshape(head_count, head_dim // 2, 2, *weights.shape[1:]) + w = np.swapaxes(w, 1, 2) + return w.reshape(weights.shape) + + +def build_tensor_name_map(arch_name: str, block_count: int, gguf_module) -> dict[str, str]: + """Build a mapping from GGUF tensor names to standard Hugging Face names.""" + arch_enum = getattr(gguf_module.MODEL_ARCH, arch_name.upper(), None) + mapping: dict[str, str] = {} + + if arch_enum is not None: + try: + tm = gguf_module.get_tensor_name_map(arch_enum, block_count) + rev: dict[str, list[str]] = {} + for hf_key, (_tensor_enum, gguf_name) in tm.mapping.items(): + if hf_key == gguf_name: + continue + if gguf_name not in rev: + rev[gguf_name] = [] + rev[gguf_name].append(hf_key) + + for g_name, hf_candidates in rev.items(): + preferred = [c for c in hf_candidates if c.startswith("model.") or c.startswith("transformer.")] + mapping[g_name] = preferred[0] if preferred else hf_candidates[0] + except Exception: + pass + + defaults = { + "token_embd": "model.embed_tokens", + "output_norm": "model.norm", + "output": "lm_head", + } + for g, h in defaults.items(): + mapping.setdefault(g, h) + + for bid in range(block_count): + bid_defaults = { + f"blk.{bid}.attn_q": f"model.layers.{bid}.self_attn.q_proj", + f"blk.{bid}.attn_k": f"model.layers.{bid}.self_attn.k_proj", + f"blk.{bid}.attn_v": f"model.layers.{bid}.self_attn.v_proj", + f"blk.{bid}.attn_output": f"model.layers.{bid}.self_attn.o_proj", + f"blk.{bid}.attn_norm": f"model.layers.{bid}.input_layernorm", + f"blk.{bid}.ffn_gate": f"model.layers.{bid}.mlp.gate_proj", + f"blk.{bid}.ffn_up": f"model.layers.{bid}.mlp.up_proj", + f"blk.{bid}.ffn_down": f"model.layers.{bid}.mlp.down_proj", + f"blk.{bid}.ffn_norm": f"model.layers.{bid}.post_attention_layernorm", + f"blk.{bid}.attn_q_norm": f"model.layers.{bid}.self_attn.q_norm", + f"blk.{bid}.attn_k_norm": f"model.layers.{bid}.self_attn.k_norm", + f"blk.{bid}.post_ffw_norm": f"model.layers.{bid}.post_feedforward_layernorm", + f"blk.{bid}.pre_ffw_norm": f"model.layers.{bid}.pre_feedforward_layernorm", + } + for g, h in bid_defaults.items(): + mapping.setdefault(g, h) + + return mapping + + +def _map_gguf_name(gguf_name: str, name_map: dict[str, str]) -> str: + base, _, suffix = gguf_name.rpartition(".") + if base in name_map: + return f"{name_map[base]}.{suffix}" + return gguf_name + + +def _synthesize_hf_configs(reader, output_dir: Path, arch: str, args, is_quantized: bool) -> None: + """Generate config.json and tokenizer files from GGUF metadata.""" + output_dir.mkdir(parents=True, exist_ok=True) + fields = reader.fields + + def get_field(key: str, default=None): + f = fields.get(key) + return _pythonize(f.contents()) if f is not None else default + + arch_map = { + "llama": ("LlamaForCausalLM", "llama"), + "qwen2": ("Qwen2ForCausalLM", "qwen2"), + "gemma": ("GemmaForCausalLM", "gemma"), + "gemma2": ("Gemma2ForCausalLM", "gemma2"), + "mistral": ("MistralForCausalLM", "mistral"), + "phi3": ("Phi3ForCausalLM", "phi3"), + "deepseek2": ("DeepseekV2ForCausalLM", "deepseek_v2"), + "gptneox": ("GPTNeoXForCausalLM", "gpt_neox"), + } + class_name, model_type = arch_map.get(arch.lower(), (f"{arch.capitalize()}ForCausalLM", arch.lower())) + + tokens = get_field("tokenizer.ggml.tokens", []) + vocab_size = len(tokens) if tokens else get_field(f"{arch}.vocab_size", 32000) + + config = { + "architectures": [class_name], + "model_type": model_type, + "vocab_size": vocab_size, + "hidden_size": get_field(f"{arch}.embedding_length", 4096), + "intermediate_size": get_field(f"{arch}.feed_forward_length", 11008), + "num_hidden_layers": get_field(f"{arch}.block_count", 32), + "num_attention_heads": get_field(f"{arch}.attention.head_count", 32), + "num_key_value_heads": get_field(f"{arch}.attention.head_count_kv", get_field(f"{arch}.attention.head_count", 32)), + "max_position_embeddings": getattr(args, "context_length", None) or get_field(f"{arch}.context_length", 4096), + "rms_norm_eps": get_field(f"{arch}.attention.layer_norm_rms_epsilon", 1e-5), + "rope_theta": get_field(f"{arch}.rope.freq_base", 10000.0), + "bos_token_id": get_field("tokenizer.ggml.bos_token_id", 1), + "eos_token_id": get_field("tokenizer.ggml.eos_token_id", 2), + "torch_dtype": getattr(args, "output_dtype", "bfloat16"), + } + + fmt = getattr(args, "format", "hf") + ignore = [] + if not getattr(args, "quantize_output", False): + ignore.append("lm_head") + if getattr(args, "exclude", None): + ignore.append(args.exclude) + + if is_quantized: + if fmt == "hf": + config["quantization_config"] = { + "quant_method": "compressed-tensors", + "format": "nvfp4-pack-quantized", + "quantization_status": "compressed", + "ignore": ignore, + "config_groups": { + "group_0": { + "weights": { + "num_bits": 4, + "type": "float", + "strategy": "tensor_group", + "group_size": 16, + "symmetric": True, + "dynamic": False, + "actorder": None, + }, + "input_activations": None, + "targets": ["Linear"], + "format": "nvfp4-pack-quantized", + } + }, + } + else: + config["quantization_config"] = { + "quant_method": "modelopt", + "quant_algo": "NVFP4", + "format": "nvfp4-pack-quantized", + "group_size": 16, + "bits": 4, + } + + (output_dir / "config.json").write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + + # Tokenizer configs + chat_template = get_field("tokenizer.chat_template") + tok_config = { + "bos_token_id": config["bos_token_id"], + "eos_token_id": config["eos_token_id"], + "model_max_length": config["max_position_embeddings"], + } + if chat_template: + tok_config["chat_template"] = _text(chat_template) + (output_dir / "tokenizer_config.json").write_text(json.dumps(tok_config, indent=2) + "\n", encoding="utf-8") + + special_tokens = {} + if tokens: + bos_id = config["bos_token_id"] + eos_id = config["eos_token_id"] + if 0 <= bos_id < len(tokens): + special_tokens["bos_token"] = _text(tokens[bos_id]) + if 0 <= eos_id < len(tokens): + special_tokens["eos_token"] = _text(tokens[eos_id]) + vocab = {_text(t): i for i, t in enumerate(tokens)} + (output_dir / "vocab.json").write_text(json.dumps(vocab, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + (output_dir / "special_tokens_map.json").write_text(json.dumps(special_tokens, indent=2) + "\n", encoding="utf-8") + + merges = get_field("tokenizer.ggml.merges") + if merges: + merge_lines = [_text(m) for m in merges] + (output_dir / "merges.txt").write_text("#version: 0.2\n" + "\n".join(merge_lines) + "\n", encoding="utf-8") + + +def _copy_template_files(template_dir: Path, output_dir: Path, args, is_quantized: bool) -> None: + output_dir.mkdir(parents=True, exist_ok=True) + fmt = getattr(args, "format", "hf") + + for path in template_dir.iterdir(): + if path.is_dir() or path.name.startswith("."): + continue + if any(path.name.endswith(ext) for ext in (".safetensors", ".bin", ".pt", ".pth", ".h5")): + continue + if path.name == INDEX_NAME: + continue + dest = output_dir / path.name + if path.name == "config.json": + try: + cfg = json.loads(path.read_text(encoding="utf-8")) + if is_quantized: + if fmt == "hf": + cfg["quantization_config"] = { + "quant_method": "compressed-tensors", + "format": "nvfp4-pack-quantized", + "quantization_status": "compressed", + "ignore": ignore, + "config_groups": { + "group_0": { + "weights": { + "num_bits": 4, + "type": "float", + "strategy": "tensor_group", + "group_size": 16, + "symmetric": True, + "dynamic": False, + "actorder": None, + }, + "input_activations": None, + "targets": ["Linear"], + "format": "nvfp4-pack-quantized", + } + }, + } + else: + cfg["quantization_config"] = { + "quant_method": "modelopt", + "quant_algo": "NVFP4", + "format": "nvfp4-pack-quantized", + "group_size": 16, + "bits": 4, + } + cfg["torch_dtype"] = getattr(args, "output_dtype", "bfloat16") + if getattr(args, "context_length", None): + cfg["max_position_embeddings"] = int(args.context_length) + dest.write_text(json.dumps(cfg, indent=2) + "\n", encoding="utf-8") + continue + except Exception: + pass + shutil.copy2(path, dest) + + +def export_gguf_to_hf(args) -> None: + gguf_path = Path(args.gguf).resolve() + if not gguf_path.is_file(): + raise FileNotFoundError(f"GGUF file does not exist: {gguf_path}") + + _assign_output(args) + output_dir = Path(args.output).resolve() + if output_dir.exists() and any(output_dir.iterdir()) and not args.overwrite: + raise SystemExit(f"Output directory {output_dir} already exists and is not empty. Use --overwrite to replace.") + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s") + log.info("Loading GGUF file: %s", gguf_path) + log.info("Target format: %s", "Hugging Face (compressed-tensors)" if args.format == "hf" else "ModelOpt / ComfyUI") + + import gguf + reader = gguf.GGUFReader(gguf_path) + arch = _text(reader.fields.get("general.architecture", "llama").contents()) + block_count = 32 + for k in (f"{arch}.block_count", "general.block_count"): + if k in reader.fields: + block_count = int(_pythonize(reader.fields[k].contents())) + break + + n_head = int(_pythonize(reader.fields.get(f"{arch}.attention.head_count", 32).contents())) + n_kv_head = int(_pythonize(reader.fields.get(f"{arch}.attention.head_count_kv", n_head).contents())) + should_unpermute = arch.lower() in ("llama", "mistral", "nanbeige") + + name_map = build_tensor_name_map(arch, block_count, gguf) + profile = quant_profile(arch) + if profile is not None: + if not args.exclude: + args.exclude = profile.skip + log.info("%s profile: leaving %s unquantized.", profile.arch, profile.label) + else: + log.info("%s profile: --exclude replaces default skip of %s.", profile.arch, profile.label) + args.exclude_re = re.compile(args.exclude) if args.exclude else None + + is_passthrough = args.no_quant or args.outtype == "nvfp4" + is_quantized = not is_passthrough + float_dtype = _target_dtype(args.output_dtype if not is_passthrough else (args.outtype if args.outtype != "auto" else "bfloat16")) + + # Config & Tokenizer generation / copying + if args.hf_template and Path(args.hf_template).is_dir(): + log.info("Copying configs and tokenizers from template: %s", args.hf_template) + _copy_template_files(Path(args.hf_template), output_dir, args, is_quantized=is_quantized) + else: + log.info("Synthesizing Hugging Face configs and tokenizer from GGUF metadata") + _synthesize_hf_configs(reader, output_dir, arch, args, is_quantized=is_quantized) + + # Scale tensors cache for NVFP4 repacking + scale_tensors: dict[str, float] = {} + for tensor in reader.tensors: + if tensor.name.endswith(".scale"): + base = tensor.name.removesuffix(".scale") + scale_val = float(np.asarray(tensor.data, dtype=np.float32).reshape(-1)[0]) + scale_tensors[base] = scale_val + + # Checkpoint dir + checkpoint_dir = Path(args.checkpoint_dir) if args.checkpoint_dir else output_dir / ".nvfp4-work" + if args.fresh and checkpoint_dir.exists(): + shutil.rmtree(checkpoint_dir) + checkpoint_dir.mkdir(parents=True, exist_ok=True) + + quantizer = None + if is_quantized: + ensure_cuda_interpreter(args.device) + args.optimizer = resolve_optimizer(args.optimizer) + quantizer = TensorQuantizer(args, gguf, load_learned_converter(args.convert_to_quant)[1]) + + quant_count = 0 + passthrough_count = 0 + bias_count = 0 + all_quant_meta: dict[str, dict] = {} + all_hf_tensors: dict[str, torch.Tensor] = {} + + log.info("Processing %d tensor(s) from GGUF", len(reader.tensors)) + + for tensor in reader.tensors: + g_name = tensor.name + if g_name.endswith(".scale") or g_name.endswith(".input_scale"): + continue + + hf_name = _map_gguf_name(g_name, name_map) + base_name = hf_name.removesuffix(".weight") + shape = tuple(int(d) for d in tensor.shape[::-1]) # GGUF shapes are reversed + + # If already NVFP4 in GGUF + if tensor.tensor_type == gguf.GGMLQuantizationType.NVFP4: + log.info(" Repacking NVFP4 [%s] %s -> %s", "x".join(str(d) for d in shape), g_name, hf_name) + raw = np.asarray(tensor.data, dtype=np.uint8).reshape(shape[0], -1) + per_tensor = scale_tensors.get(g_name, 1.0) + packed_w, scale_1, scale_2 = unpack_gguf_nvfp4_to_hf(raw, per_tensor) + + if args.format == "hf": + all_hf_tensors[f"{base_name}.weight_packed"] = packed_w + all_hf_tensors[f"{base_name}.weight_scale"] = scale_1 + all_hf_tensors[f"{base_name}.weight_global_scale"] = torch.tensor(1.0 / float(per_tensor), dtype=torch.float32) + else: + all_hf_tensors[hf_name] = packed_w + all_hf_tensors[f"{base_name}.weight_scale"] = scale_1 + all_hf_tensors[f"{base_name}.weight_scale_2"] = scale_2 + all_hf_tensors[f"{base_name}.comfy_quant"] = torch.tensor([1], dtype=torch.uint8) + + all_quant_meta[base_name] = {"quant_type": "nvfp4", "block_size": 16, "scale_type": "float8_e4m3fn"} + quant_count += 1 + continue + + # Convert tensor data to float + data_float = _to_float(tensor, gguf) + + # Unpermute RoPE for Q and K if needed + if should_unpermute: + if g_name.endswith("attn_q.weight"): + data_float = unpermute_rope(data_float, n_head, n_head) + elif g_name.endswith("attn_k.weight"): + data_float = unpermute_rope(data_float, n_head, n_kv_head) + + # Check eligibility for NVFP4 quantization + is_eligible = ( + is_quantized + and len(shape) == 2 + and hf_name.endswith(".weight") + and shape[-1] >= 64 + and shape[-1] % 64 == 0 + and not ("norm" in hf_name.lower() or "embed_tokens" in hf_name.lower()) + and (hf_name != "lm_head.weight" or getattr(args, "quantize_output", False)) + and not (args.exclude_re and args.exclude_re.search(hf_name)) + and not ctq_skips_key(hf_name, args.convert_to_quant) + ) + + if is_eligible and quantizer is not None: + log.info(" Quantizing [%s] %s -> %s", "x".join(str(d) for d in shape), g_name, hf_name) + has_bias = False + + cached = _load_checkpoint(checkpoint_dir, hf_name, shape, need_bias=has_bias) + if cached is not None: + codes, block_scale, per_tensor, bias_corr = cached + log.info(" Loaded from checkpoint: %s", hf_name) + else: + rounds = getattr(args, "scale_refinement_rounds", DEFAULT_SCALE_REFINEMENT_ROUNDS) + codes, block_scale, per_tensor, dequant = quantizer.quantize_matrix(data_float, hf_name) + codes, block_scale = coordinate_descent_refine( + data_float, codes, block_scale, per_tensor, bake=False, rounds=rounds, device=quantizer.device, + ) + codes, block_scale, per_tensor = sweep_per_tensor_scale( + data_float, codes, block_scale, per_tensor, bake=False, device=quantizer.device, + ) + _save_checkpoint(checkpoint_dir, hf_name, shape, codes, block_scale, per_tensor, None) + + packed_w, scale_1, scale_2 = pack_nvfp4_hf(codes, block_scale, per_tensor) + + if args.format == "hf": + all_hf_tensors[f"{base_name}.weight_packed"] = packed_w + all_hf_tensors[f"{base_name}.weight_scale"] = scale_1 + all_hf_tensors[f"{base_name}.weight_global_scale"] = torch.tensor(1.0 / float(per_tensor), dtype=torch.float32) + else: + all_hf_tensors[hf_name] = packed_w + all_hf_tensors[f"{base_name}.weight_scale"] = scale_1 + all_hf_tensors[f"{base_name}.weight_scale_2"] = scale_2 + all_hf_tensors[f"{base_name}.comfy_quant"] = torch.tensor([1], dtype=torch.uint8) + + all_quant_meta[base_name] = {"quant_type": "nvfp4", "block_size": 16, "scale_type": "float8_e4m3fn"} + quant_count += 1 + else: + # Float tensor passthrough + tensor_t = torch.from_numpy(data_float) + if tensor_t.ndim <= 1 and hf_name.endswith("norm.weight"): + tensor_t = tensor_t.float() + else: + tensor_t = tensor_t.to(dtype=float_dtype) + all_hf_tensors[hf_name] = tensor_t + passthrough_count += 1 + + # Shard writing logic + total_bytes = sum(t.numel() * t.element_size() for t in all_hf_tensors.values()) + metadata = {} + if args.format != "hf": + metadata["_quantization_metadata"] = json.dumps({"format_version": "1.0", "layers": all_quant_meta}, separators=(",", ":")) + + if total_bytes < MAX_SHARD_BYTES: + out_file = output_dir / "model.safetensors" + save_file(all_hf_tensors, out_file, metadata=metadata if metadata else None) + log.info("Saved single shard: %s", out_file) + else: + # Split into shards + shards_list: list[dict[str, torch.Tensor]] = [{}] + current_shard_size = 0 + for name, tensor in all_hf_tensors.items(): + t_size = tensor.numel() * tensor.element_size() + if current_shard_size + t_size > MAX_SHARD_BYTES and shards_list[-1]: + shards_list.append({}) + current_shard_size = 0 + shards_list[-1][name] = tensor + current_shard_size += t_size + + num_shards = len(shards_list) + weight_map: dict[str, str] = {} + for idx, shard_dict in enumerate(shards_list, start=1): + shard_filename = f"model-{idx:05d}-of-{num_shards:05d}.safetensors" + shard_path = output_dir / shard_filename + save_file(shard_dict, shard_path, metadata=metadata if metadata else None) + for k in shard_dict: + weight_map[k] = shard_filename + log.info("Saved shard %d/%d: %s", idx, num_shards, shard_path) + + index_payload = { + "metadata": {"total_size": total_bytes}, + "weight_map": weight_map, + } + (output_dir / INDEX_NAME).write_text(json.dumps(index_payload, indent=2) + "\n", encoding="utf-8") + + if not args.keep_checkpoints and quant_count: + shutil.rmtree(checkpoint_dir, ignore_errors=True) + + log.info( + "\nWrote %s (%d NVFP4 tensors, %d other tensors)", + output_dir, + quant_count, + passthrough_count, + ) + + +def self_test(args) -> None: + """Self-test NVFP4 unpacking and roundtripping.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + from nvfp4gguf import FP4_TABLE + import gguf + + log.info("Running gguf_to_nvfp4_hf self-test...") + _converter_cls, unpack_uint4 = load_learned_converter(args.convert_to_quant) + quantizer = TensorQuantizer(args, gguf, unpack_uint4) + + generator = np.random.default_rng(0) + for shape in ((64, 128), (128, 256)): + weight = generator.normal(size=shape).astype(np.float32) * 0.05 + codes, block_scale, per_tensor, dequant = quantizer.quantize_matrix(weight) + packed_w, scale_1, scale_2 = pack_nvfp4_hf(codes, block_scale, per_tensor) + + out_features = shape[0] + n_blocks = shape[1] // 16 + n_super = shape[1] // 64 + + w_b = packed_w.reshape(out_features, n_blocks, 8) + vals = torch.stack([w_b & 0x0F, w_b >> 4], dim=-1).reshape(out_features, n_blocks, 16) + d_ue = scale_1.view(torch.uint8).reshape(out_features, n_blocks) & 0x7F + qs = (vals[:, :, :8] | (vals[:, :, 8:] << 4)).to(torch.uint8) + d_grouped = d_ue.reshape(out_features, n_super, 4) + qs_grouped = qs.reshape(out_features, n_super, 32) + raw = torch.cat([d_grouped, qs_grouped], dim=-1).reshape(out_features, n_super * 36).numpy() + + w_rec, s1_rec, s2_rec = unpack_gguf_nvfp4_to_hf(raw, per_tensor) + assert torch.equal(packed_w, w_rec), "GGUF unpack weights mismatch!" + assert torch.equal(scale_1.view(torch.uint8) & 0x7F, s1_rec.view(torch.uint8) & 0x7F), "GGUF unpack scales mismatch!" + log.info("Shape %s GGUF-to-HF unpack roundtrip ok", shape) + + log.info("self-test passed successfully.") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Convert and quantize a GGUF model to NVFP4 Hugging Face format.", + ) + parser.add_argument("gguf", nargs="?", help="Input GGUF file") + parser.add_argument("-o", "--output", help="Output directory. Default: ./-NVFP4") + parser.add_argument("--format", default="hf", choices=("hf", "modelopt", "comfy"), + help="Safetensors tensor layout: 'hf' (default) uses Hugging Face compressed-tensors (.weight_packed, .weight_scale, .weight_global_scale). 'modelopt' (or 'comfy') uses raw .weight.") + parser.add_argument("--overwrite", action="store_true", help="Replace existing files in the output directory") + parser.add_argument("--no-quant", action="store_true", help="Skip learned rounding; write float HF model") + parser.add_argument("--outtype", default="auto", choices=("auto", "f32", "f16", "bf16", "nvfp4"), + help="Float type for tensors when --no-quant is used (default: auto)") + parser.add_argument("--output-dtype", default="bfloat16", choices=("bfloat16", "float16"), + help="Dtype for unquantized 2D weights and embeddings (default: bfloat16)") + parser.add_argument("--hf-template", type=Path, default=None, + help="Reference Hugging Face model directory to copy config and tokenizer files from") + parser.add_argument("--context-length", type=int, default=None, + help="Override context length stored in config.json") + parser.add_argument("--model-name", default=None, help="Override model name") + parser.add_argument("--checkpoint-dir", type=Path, default=None, help="Per-tensor resume directory") + parser.add_argument("--fresh", action="store_true", help="Delete checkpoint directory before starting") + parser.add_argument("--keep-checkpoints", action="store_true", help="Keep per-tensor checkpoints upon completion") + parser.add_argument("--eager", action="store_true", help="Eager loading") + parser.add_argument("--low-memory", action="store_true", help="Stream each tensor instead of loading all at once") + parser.add_argument("--heur", action="store_true", help="Skip layers heuristic rejects") + parser.add_argument("--verbose", action="store_true", help="Enable verbose logging") + parser.add_argument("--self-test", action="store_true", help="Run NVFP4 packing self-test and exit") + parser.add_argument("--llama-cpp", type=Path, default=DEFAULT_LLAMA_CPP, + help="llama.cpp checkout directory (default: ~/llama.cpp)") + + add_quant_arguments(parser) + args = parser.parse_args(argv) + + if args.self_test: + if args.device == "auto": + args.device = "cpu" + return args + + if not args.gguf: + parser.error("GGUF model file is required") + + if args.convert_to_quant is None and DEFAULT_CONVERT_TO_QUANT.is_dir(): + args.convert_to_quant = DEFAULT_CONVERT_TO_QUANT + + return args + + +def main(argv: list[str] | None = None) -> None: + args = parse_args(argv) + if args.self_test: + self_test(args) + return + export_gguf_to_hf(args) + + +if __name__ == "__main__": + main() diff --git a/hf_to_nvfp4_hf.py b/hf_to_nvfp4_hf.py new file mode 100755 index 0000000..837e12c --- /dev/null +++ b/hf_to_nvfp4_hf.py @@ -0,0 +1,620 @@ +"""Quantize a Hugging Face model directory to NVFP4 Hugging Face format. + +Reads a Hugging Face model directory (config.json + safetensors) and writes a +quantized Hugging Face model directory. By default (--format hf), writes standard +Hugging Face compressed-tensors format with .weight_packed, .weight_scale, and +.weight_global_scale natively supported by Hugging Face transformers, vLLM, SGLang, +and llama.cpp. + +Optionally (--format modelopt / comfy), outputs raw weight tensors (.weight), +weight_scale, weight_scale_2, and .comfy_quant for ModelOpt and ComfyUI. + +Weight matrices are quantized using convert_to_quant's learned rounding (or simple +round-to-grid with --simple). Biases are adjusted using calibration data unless +--no-bias-correction is specified. Embeddings, layer norms, and excluded layers +stay in their float format. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import logging +import os +import re +import shutil +import sys +from pathlib import Path + +import numpy as np +import torch +from safetensors import safe_open +from safetensors.torch import save_file + +from nvfp4gguf import ( + CTQ_NUM_ITER, + DEFAULT_CONVERT_TO_QUANT, + DEFAULT_LLAMA_CPP, + DEFAULT_EARLY_STOP_FLOOR_STALL, + DEFAULT_TARGET_LOSS, + DEFAULT_EARLY_STOP_STALL, + DEFAULT_LR_FLOOR, + DEFAULT_LR_PATIENCE, + DEFAULT_LR_THRESHOLD, + DEFAULT_SCALE_OPTIMIZATION, + DEFAULT_SCALE_REFINEMENT_ROUNDS, + QUANT_VERSION, + TensorQuantizer, + add_quant_arguments, + bind_exclude, + coordinate_descent_refine, + ctq_skips_key, + ensure_cuda_interpreter, + load_learned_converter, + quant_profile, + report_reconstruction, + resolve_optimizer, + sweep_per_tensor_scale, + ue4m3_bits, +) + +INDEX_NAME = "model.safetensors.index.json" +NON_WEIGHT_PATTERNS = { + ".safetensors", + ".bin", + ".pt", + ".pth", + ".h5", + ".msgpack", + ".ot", + ".tmp", +} + +log = logging.getLogger("hf_to_nvfp4_hf") + + +def _checkpoint_file(directory: Path, name: str) -> Path: + digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:16] + safe = re.sub(r"[^A-Za-z0-9._-]+", "_", name)[:80] + return directory / f"{safe}.{digest}.npz" + + +def _load_checkpoint(directory: Path, name: str, shape: tuple[int, ...], need_bias: bool): + path = _checkpoint_file(directory, name) + if not path.is_file(): + return None + with np.load(path) as blob: + if "version" in blob.files and int(blob["version"]) != QUANT_VERSION: + return None + if tuple(int(v) for v in blob["shape"]) != tuple(shape): + return None + if need_bias and "bias" not in blob.files: + return None + codes = np.array(blob["codes"], copy=True) + block_scale = np.array(blob["block_scale"], copy=True) + per_tensor = float(blob["per_tensor"]) + bias = np.array(blob["bias"], copy=True) if "bias" in blob.files else None + return codes, block_scale, per_tensor, bias + + +def _save_checkpoint(directory: Path, name: str, shape: tuple[int, ...], codes, block_scale, per_tensor, bias) -> None: + path = _checkpoint_file(directory, name) + temporary = path.with_name(path.name + ".tmp") + with temporary.open("wb") as handle: + save_dict = { + "version": np.int32(QUANT_VERSION), + "shape": np.asarray(shape, dtype=np.int64), + "codes": np.asarray(codes, dtype=np.uint8), + "block_scale": np.asarray(block_scale, dtype=np.float32), + "per_tensor": np.float32(per_tensor), + } + if bias is not None: + save_dict["bias"] = np.asarray(bias, dtype=np.float32) + np.savez_compressed(handle, **save_dict) + temporary.replace(path) + + +def discover_shards(model_dir: Path) -> list[str]: + """Return safetensors shard filenames in the checkpoint directory.""" + index_path = model_dir / INDEX_NAME + if index_path.is_file(): + try: + payload = json.loads(index_path.read_text(encoding="utf-8")) + weight_map = payload.get("weight_map") + if isinstance(weight_map, dict) and weight_map: + names = sorted(set(weight_map.values())) + missing = [name for name in names if not (model_dir / name).is_file()] + if not missing: + return names + except Exception: + pass + + single = model_dir / "model.safetensors" + if single.is_file(): + return ["model.safetensors"] + + shards = sorted(path.name for path in model_dir.glob("*.safetensors") if path.is_file()) + if not shards: + raise SystemExit(f"No safetensors shards found in {model_dir}") + return shards + + +def _assign_output(args) -> None: + source = Path(args.model).resolve() + stem = source.name + passthrough = args.no_quant or args.outtype == "nvfp4" + if not args.output: + if passthrough and args.outtype == "auto": + directory = Path.cwd() / stem + elif passthrough: + label = "NVFP4" if args.outtype == "nvfp4" else args.outtype.upper() + directory = Path.cwd() / f"{stem}-{label}" + else: + directory = Path.cwd() / f"{stem}-NVFP4" + else: + directory = Path(args.output).resolve() + args.output = str(directory) + + +def _apply_directory_profile(args, source: Path) -> None: + config_path = source / "config.json" + if not config_path.is_file(): + return + try: + config = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return + names: list[str] = [] + architectures = config.get("architectures") + if isinstance(architectures, list): + names.extend(str(item) for item in architectures) + elif isinstance(architectures, str): + names.append(architectures) + model_type = config.get("model_type") + if isinstance(model_type, str) and model_type: + names.append(model_type) + for name in names: + profile = quant_profile(name) + if profile is None: + continue + if not args.exclude: + args.exclude = profile.skip + log.info("%s profile: leaving %s unquantized.", profile.arch, profile.label) + else: + log.info("%s profile: --exclude replaces default skip of %s.", profile.arch, profile.label) + return + + +def _copy_non_weight_files(source_dir: Path, output_dir: Path) -> None: + """Copy configs, tokenizers, chat templates, and documentation into output.""" + output_dir.mkdir(parents=True, exist_ok=True) + for path in source_dir.iterdir(): + if path.is_dir() or path.name.startswith("."): + continue + if path.name == "config.json" or path.name == INDEX_NAME: + continue + if any(path.name.endswith(ext) for ext in NON_WEIGHT_PATTERNS): + continue + dest = output_dir / path.name + if not dest.exists() or dest.stat().st_mtime < path.stat().st_mtime: + shutil.copy2(path, dest) + + +def _write_config_json(source_dir: Path, output_dir: Path, args, is_quantized: bool) -> None: + config_path = source_dir / "config.json" + if not config_path.is_file(): + return + try: + config = json.loads(config_path.read_text(encoding="utf-8")) + except Exception as exc: + log.warning("Could not parse config.json: %s", exc) + return + + fmt = getattr(args, "format", "hf") + + def _build_ct_quant_config(): + ignore = [] + if not getattr(args, "quantize_output", False): + ignore.append("lm_head") + if getattr(args, "exclude", None): + ignore.append(args.exclude) + return { + "quant_method": "compressed-tensors", + "format": "nvfp4-pack-quantized", + "quantization_status": "compressed", + "ignore": ignore, + "config_groups": { + "group_0": { + "weights": { + "num_bits": 4, + "type": "float", + "strategy": "tensor_group", + "group_size": 16, + "symmetric": True, + "dynamic": False, + "actorder": None, + }, + "input_activations": None, + "targets": ["Linear"], + "format": "nvfp4-pack-quantized", + } + }, + } + + if is_quantized: + if fmt == "hf": + config["quantization_config"] = _build_ct_quant_config() + else: + config["quantization_config"] = { + "quant_method": "modelopt", + "quant_algo": "NVFP4", + "format": "nvfp4-pack-quantized", + "group_size": 16, + "bits": 4, + } + config["torch_dtype"] = getattr(args, "output_dtype", "bfloat16") + elif args.no_quant: + if args.outtype in ("f32", "f16", "bf16"): + config.pop("quantization_config", None) + config["torch_dtype"] = "bfloat16" if args.outtype == "bf16" else ("float16" if args.outtype == "f16" else "float32") + elif args.outtype == "nvfp4": + if fmt == "hf": + config["quantization_config"] = _build_ct_quant_config() + else: + config.setdefault("quantization_config", { + "quant_method": "modelopt", + "quant_algo": "NVFP4", + "format": "nvfp4-pack-quantized", + "group_size": 16, + "bits": 4, + }) + + if getattr(args, "context_length", None): + for key in ("max_position_embeddings", "seq_length", "max_sequence_length"): + if key in config: + config[key] = int(args.context_length) + + out_file = output_dir / "config.json" + out_file.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + + +def pack_nvfp4_hf(codes: np.ndarray, block_scales: np.ndarray, per_tensor: float): + """Pack codes and scales into NVFP4 tensors.""" + # Packed weights: low nibble is element 0, high nibble is element 1 + packed = (codes[:, 0::2] & 0x0F) | ((codes[:, 1::2] & 0x0F) << 4) + packed_tensor = torch.from_numpy(np.ascontiguousarray(packed, dtype=np.uint8)) + + scale_bits = ue4m3_bits(block_scales) + scale_tensor = torch.from_numpy(np.ascontiguousarray(scale_bits, dtype=np.uint8)).view(torch.float8_e4m3fn) + scale_2_tensor = torch.tensor(float(per_tensor), dtype=torch.float32) + + return packed_tensor, scale_tensor, scale_2_tensor + + +def _should_quantize( + key: str, + shape: tuple[int, ...], + args, + convert_to_quant_dir: Path | None, +) -> bool: + if len(shape) != 2: + return False + if not (key.endswith(".weight") or key.endswith(".weight_packed")): + return False + if shape[-1] % 64 != 0 or shape[-1] < 64: + return False + if key in ("lm_head.weight", "lm_head.weight_packed") and not getattr(args, "quantize_output", False): + return False + lowered = key.lower() + if "embed_tokens" in lowered or "wte" in lowered or "word_embeddings" in lowered: + return False + if "norm" in lowered or "ln" in lowered: + return False + if getattr(args, "exclude_re", None) and args.exclude_re.search(key): + return False + if ctq_skips_key(key, convert_to_quant_dir): + return False + return True + + +def _target_dtype(dtype_str: str) -> torch.dtype: + mapping = { + "float32": torch.float32, + "f32": torch.float32, + "float16": torch.float16, + "f16": torch.float16, + "bfloat16": torch.bfloat16, + "bf16": torch.bfloat16, + } + return mapping.get(dtype_str.lower(), torch.bfloat16) + + +def export_directory(args) -> None: + source_dir = Path(args.model).resolve() + if not source_dir.is_dir(): + raise FileNotFoundError(f"Model directory does not exist: {source_dir}") + + _assign_output(args) + output_dir = Path(args.output).resolve() + if output_dir == source_dir: + raise SystemExit("Output directory cannot be the same as the input directory.") + if output_dir.exists() and any(output_dir.iterdir()) and not args.overwrite: + raise SystemExit(f"Output directory {output_dir} already exists and is not empty. Use --overwrite to replace.") + + shards = discover_shards(source_dir) + _apply_directory_profile(args, source_dir) + args.exclude_re = re.compile(args.exclude) if args.exclude else None + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s") + log.info("Loading %s", source_dir) + log.info("Found %d safetensors shard(s)", len(shards)) + log.info("Target format: %s", "Hugging Face (compressed-tensors)" if args.format == "hf" else "ModelOpt / ComfyUI") + + output_dir.mkdir(parents=True, exist_ok=True) + _copy_non_weight_files(source_dir, output_dir) + + is_passthrough = args.no_quant or args.outtype == "nvfp4" + float_dtype = _target_dtype(args.output_dtype if not is_passthrough else (args.outtype if args.outtype != "auto" else "bfloat16")) + + if is_passthrough: + log.info("Quantization skipped. Converting tensors to %s", args.outtype) + weight_map: dict[str, str] = {} + total_size = 0 + for shard_name in shards: + log.info("Processing %s", shard_name) + src_shard = source_dir / shard_name + out_shard = output_dir / shard_name + out_tensors: dict[str, torch.Tensor] = {} + with safe_open(src_shard, framework="pt", device="cpu") as handle: + for key in handle.keys(): + t = handle.get_tensor(key) + if args.outtype in ("f32", "f16", "bf16") and t.is_floating_point(): + t = t.to(dtype=float_dtype) + out_tensors[key] = t + weight_map[key] = shard_name + total_size += t.numel() * t.element_size() + save_file(out_tensors, out_shard) + _write_config_json(source_dir, output_dir, args, is_quantized=False) + if len(shards) > 1: + (output_dir / INDEX_NAME).write_text( + json.dumps({"metadata": {"total_size": total_size}, "weight_map": weight_map}, indent=2) + "\n", + encoding="utf-8", + ) + log.info("Wrote %s", output_dir) + return + + # Learned NVFP4 Quantization + ensure_cuda_interpreter(args.device) + args.optimizer = resolve_optimizer(args.optimizer) + + # Prepare checkpoint directory for per-tensor resume + checkpoint_dir = Path(args.checkpoint_dir) if args.checkpoint_dir else output_dir / ".nvfp4-work" + if args.fresh and checkpoint_dir.exists(): + shutil.rmtree(checkpoint_dir) + checkpoint_dir.mkdir(parents=True, exist_ok=True) + + # Initialize quantizer + import gguf + quantizer = TensorQuantizer(args, gguf, load_learned_converter(args.convert_to_quant)[1]) + + # First pass: collect all tensor keys and shapes across shards + all_keys: dict[str, tuple[str, tuple[int, ...]]] = {} + for shard_name in shards: + with safe_open(source_dir / shard_name, framework="pt", device="cpu") as handle: + for key in handle.keys(): + slice_header = handle.get_slice(key) + all_keys[key] = (shard_name, tuple(int(d) for d in slice_header.get_shape())) + + quant_count = 0 + passthrough_count = 0 + bias_count = 0 + weight_map: dict[str, str] = {} + total_size = 0 + all_quant_meta: dict[str, dict] = {} + + for s_idx, shard_name in enumerate(shards, start=1): + log.info("\n[%d/%d] Processing shard: %s", s_idx, len(shards), shard_name) + src_shard = source_dir / shard_name + out_shard = output_dir / shard_name + shard_tensors: dict[str, torch.Tensor] = {} + + with safe_open(src_shard, framework="pt", device="cpu") as handle: + keys_in_shard = list(handle.keys()) + + for key in keys_in_shard: + tensor = handle.get_tensor(key) + shape = tuple(int(d) for d in tensor.shape) + base_key = key.removesuffix(".weight_packed").removesuffix(".weight") + + if _should_quantize(key, shape, args, args.convert_to_quant): + log.info(" Quantizing [%s] %s", "x".join(str(d) for d in shape), key) + weight_np = tensor.detach().float().cpu().numpy() + + # Check for sibling bias + bias_key = f"{base_key}.bias" + has_bias = bias_key in all_keys and not args.no_bias_correction + bias_shard = all_keys[bias_key][0] if has_bias else None + + # Check checkpoint cache + cached = _load_checkpoint(checkpoint_dir, key, shape, need_bias=has_bias) + if cached is not None: + codes, block_scale, per_tensor, bias_corr = cached + log.info(" Loaded from checkpoint: %s", key) + else: + rounds = getattr(args, "scale_refinement_rounds", DEFAULT_SCALE_REFINEMENT_ROUNDS) + codes, block_scale, per_tensor, dequant = quantizer.quantize_matrix(weight_np, key) + codes, block_scale = coordinate_descent_refine( + weight_np, codes, block_scale, per_tensor, bake=False, rounds=rounds, device=quantizer.device, + ) + codes, block_scale, per_tensor = sweep_per_tensor_scale( + weight_np, codes, block_scale, per_tensor, bake=False, device=quantizer.device, + ) + + bias_corr = None + if has_bias: + bias_corr = quantizer.bias_correction(weight_np, dequant) + + _save_checkpoint(checkpoint_dir, key, shape, codes, block_scale, per_tensor, bias_corr) + + packed_w, scale_1, scale_2 = pack_nvfp4_hf(codes, block_scale, per_tensor) + + if args.format == "hf": + # Standard Hugging Face compressed-tensors layout + shard_tensors[f"{base_key}.weight_packed"] = packed_w + shard_tensors[f"{base_key}.weight_scale"] = scale_1 + shard_tensors[f"{base_key}.weight_global_scale"] = torch.tensor(1.0 / float(per_tensor), dtype=torch.float32) + else: + # ModelOpt / ComfyUI layout + shard_tensors[f"{base_key}.weight"] = packed_w + shard_tensors[f"{base_key}.weight_scale"] = scale_1 + shard_tensors[f"{base_key}.weight_scale_2"] = scale_2 + shard_tensors[f"{base_key}.comfy_quant"] = torch.tensor([1], dtype=torch.uint8) + + quant_count += 1 + all_quant_meta[base_key] = { + "quant_type": "nvfp4", + "block_size": 16, + "scale_type": "float8_e4m3fn", + } + + # Bias correction application if bias is in the same shard + if has_bias and bias_shard == shard_name and bias_corr is not None: + bias_t = handle.get_tensor(bias_key).float() + corrected_bias = bias_t - torch.from_numpy(bias_corr).to(bias_t.device) + shard_tensors[bias_key] = corrected_bias.to(dtype=float_dtype) + bias_count += 1 + else: + # Non-quantized tensors (or bias already processed) + if key in shard_tensors: + continue + if tensor.is_floating_point(): + if tensor.ndim <= 1 and key.endswith("norm.weight"): + tensor = tensor.float() + else: + tensor = tensor.to(dtype=float_dtype) + shard_tensors[key] = tensor + passthrough_count += 1 + + shard_metadata = {} + if args.format != "hf": + shard_metadata["_quantization_metadata"] = json.dumps({"format_version": "1.0", "layers": all_quant_meta}, separators=(",", ":")) + + for k, t in shard_tensors.items(): + weight_map[k] = shard_name + total_size += t.numel() * t.element_size() + + save_file(shard_tensors, out_shard, metadata=shard_metadata if shard_metadata else None) + + _write_config_json(source_dir, output_dir, args, is_quantized=True) + + if len(shards) > 1: + index_payload = { + "metadata": {"total_size": total_size}, + "weight_map": weight_map, + } + (output_dir / INDEX_NAME).write_text(json.dumps(index_payload, indent=2) + "\n", encoding="utf-8") + + if not args.keep_checkpoints and quant_count: + shutil.rmtree(checkpoint_dir, ignore_errors=True) + + log.info( + "\nWrote %s (%d NVFP4 tensors, %d biases adjusted, %d other tensors)", + output_dir, + quant_count, + bias_count, + passthrough_count, + ) + + +def self_test(args) -> None: + """Check that packed blocks roundtrip accurately with dequantizer.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + from nvfp4gguf import FP4_TABLE + import gguf + + log.info("Running NVFP4 roundtrip self-test...") + _converter_cls, unpack_uint4 = load_learned_converter(args.convert_to_quant) + quantizer = TensorQuantizer(args, gguf, unpack_uint4) + + generator = np.random.default_rng(0) + for shape in ((64, 128), (128, 256)): + weight = generator.normal(size=shape).astype(np.float32) * 0.05 + codes, block_scale, per_tensor, dequant = quantizer.quantize_matrix(weight) + packed_w, scale_1, scale_2 = pack_nvfp4_hf(codes, block_scale, per_tensor) + + # Dequantize packed tensor + w_u = packed_w.numpy() + c0 = w_u & 0x0F + c1 = (w_u >> 4) & 0x0F + unpacked_codes = np.empty(shape, dtype=np.uint8) + unpacked_codes[:, 0::2] = c0 + unpacked_codes[:, 1::2] = c1 + assert np.array_equal(codes, unpacked_codes), "Nibble unpacking mismatch!" + + # Check values + fp4_vals = FP4_TABLE[unpacked_codes] + b_scales = block_scale.repeat(16, axis=1) + recon = fp4_vals * b_scales * per_tensor + max_diff = np.max(np.abs(recon - dequant)) + assert max_diff < 1e-4, f"Reconstruction difference {max_diff} exceeded tolerance!" + log.info("Shape %s roundtrip ok (max diff %.3e)", shape, max_diff) + + log.info("self-test passed successfully.") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Quantize a Hugging Face model directory to NVFP4 Hugging Face format.", + ) + parser.add_argument("model", nargs="?", help="Hugging Face model directory (config.json + safetensors)") + parser.add_argument("-o", "--output", help="Output directory. Default: ./-NVFP4") + parser.add_argument("--format", default="hf", choices=("hf", "modelopt", "comfy"), + help="Safetensors tensor layout: 'hf' (default) uses Hugging Face compressed-tensors (.weight_packed, .weight_scale, .weight_global_scale). 'modelopt' (or 'comfy') uses raw .weight.") + parser.add_argument("--overwrite", action="store_true", help="Replace existing files in the output directory") + parser.add_argument("--no-quant", action="store_true", help="Skip learned rounding. Rewrites or repacks tensors") + parser.add_argument("--outtype", default="auto", choices=("auto", "f32", "f16", "bf16", "nvfp4"), + help="Float type for tensors when --no-quant is used (default: auto)") + parser.add_argument("--output-dtype", default="bfloat16", choices=("bfloat16", "float16"), + help="Dtype for unquantized 2D weights and embeddings (default: bfloat16)") + parser.add_argument("--context-length", type=int, default=None, + help="Override context length stored in config.json") + parser.add_argument("--model-name", default=None, help="Override model name") + parser.add_argument("--checkpoint-dir", type=Path, default=None, help="Per-tensor resume directory") + parser.add_argument("--fresh", action="store_true", help="Delete checkpoint directory before starting") + parser.add_argument("--keep-checkpoints", action="store_true", help="Keep per-tensor checkpoints upon completion") + parser.add_argument("--eager", action="store_true", help="Eager loading") + parser.add_argument("--low-memory", action="store_true", help="Stream each shard instead of loading all at once") + parser.add_argument("--heur", action="store_true", help="Skip layers heuristic rejects") + parser.add_argument("--verbose", action="store_true", help="Enable verbose logging") + parser.add_argument("--self-test", action="store_true", help="Run NVFP4 packing self-test and exit") + parser.add_argument("--llama-cpp", type=Path, default=DEFAULT_LLAMA_CPP, + help="llama.cpp checkout directory (default: ~/llama.cpp)") + + add_quant_arguments(parser) + args = parser.parse_args(argv) + + if args.self_test: + if args.device == "auto": + args.device = "cpu" + return args + + if not args.model: + parser.error("model directory is required") + + if args.convert_to_quant is None and DEFAULT_CONVERT_TO_QUANT.is_dir(): + args.convert_to_quant = DEFAULT_CONVERT_TO_QUANT + + return args + + +def main(argv: list[str] | None = None) -> None: + args = parse_args(argv) + if args.self_test: + self_test(args) + return + export_directory(args) + + +if __name__ == "__main__": + main() diff --git a/man/man1/gguf_to_nvfp4_hf.1 b/man/man1/gguf_to_nvfp4_hf.1 new file mode 100644 index 0000000..2b211c6 --- /dev/null +++ b/man/man1/gguf_to_nvfp4_hf.1 @@ -0,0 +1,241 @@ +.TH GGUF_TO_NVFP4_HF 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +gguf_to_nvfp4_hf \- convert and quantize GGUF models to NVFP4 Hugging Face format +.SH SYNOPSIS +.B gguf_to_nvfp4_hf +[\fB\-o\fR \fIOUTPUT\fR] +[\fB\-\-overwrite\fR] +[\fB\-\-format\fR \fIFORMAT\fR] +[\fB\-\-no\-quant\fR] +[\fB\-\-outtype\fR \fITYPE\fR] +[\fB\-\-output\-dtype\fR \fIDTYPE\fR] +[\fB\-\-hf\-template\fR \fIDIR\fR] +[\fB\-\-context\-length\fR \fILEN\fR] +[\fB\-\-model\-name\fR \fINAME\fR] +[\fB\-\-checkpoint\-dir\fR \fIDIR\fR] +[\fB\-\-fresh\fR] +[\fB\-\-keep\-checkpoints\fR] +[\fB\-\-eager\fR] +[\fB\-\-low\-memory\fR] +[\fB\-\-heur\fR] +[\fB\-\-verbose\fR] +[\fB\-\-self\-test\fR] +[\fB\-\-llama\-cpp\fR \fIPATH\fR] +[\fB\-\-convert\-to\-quant\fR \fIPATH\fR] +[\fB\-\-device\fR \fIDEVICE\fR] +[\fB\-\-simple\fR] +[\fB\-\-optimizer\fR \fIOPT\fR] +[\fB\-\-num\-iter\fR \fIN\fR] +[\fB\-\-lr\fR \fILR\fR] +[\fB\-\-lr\-patience\fR \fIN\fR] +[\fB\-\-lr\-min\fR \fIMIN\fR] +[\fB\-\-lr\-schedule\fR \fISCHED\fR] +[\fB\-\-scale\-optimization\fR \fIMODE\fR] +[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR] +[\fB\-\-full\-matrix\fR] +[\fB\-\-top\-p\fR \fIP\fR] +[\fB\-\-min\-k\fR \fIK\fR] +[\fB\-\-max\-k\fR \fIK\fR] +[\fB\-\-early\-stop\-loss\fR \fILOSS\fR] +[\fB\-\-early\-stop\-stall\fR \fISTALL\fR] +[\fB\-\-target\-loss\fR \fITARGET\fR] +[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR] +[\fB\-\-calib\-samples\fR \fISAMPLES\fR] +[\fB\-\-seed\fR \fISEED\fR] +[\fB\-\-no\-bias\-correction\fR] +[\fB\-\-quantize\-output\fR] +[\fB\-\-exclude\fR \fIREGEX\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fIgguf\fR +.SH DESCRIPTION +.B gguf_to_nvfp4_hf +converts a GGUF model file into a standard Hugging Face model directory +(\fIconfig.json\fR + safetensors) with NVIDIA FP4 (NVFP4) quantized weights. +.PP +GGUF tensor names are reverse-mapped to canonical Hugging Face conventions +(e.g., \fIblk.0.attn_q.weight\fR maps to \fImodel.layers.0.self_attn.q_proj.weight\fR), +and RoPE permutations for Q and K are inverted. +.PP +If the source GGUF contains float or non-NVFP4 weights, 2D weight matrices are quantized +using convert_to_quant's learned rounding (or simple round-to-grid with \fB\-\-simple\fR). +If the source GGUF already contains NVFP4 super-blocks, they are unpacked and repacked +into ModelOpt-compatible safetensors without loss of precision. +.PP +If \fB\-\-no\-quant\fR is specified, the model is dequantized to standard float format +(\fI--outtype f32, f16, bf16, or auto\fR). +.PP +An optional reference Hugging Face model directory can be provided with \fB\-\-hf\-template\fR +to copy exact upstream tokenizer files, special token maps, and configs. If omitted, +synthesized configuration and tokenizer files are extracted directly from the GGUF metadata. +.SH POSITIONAL ARGUMENTS +.TP +.I gguf +Path to the input GGUF model file. +.SH OPTIONS +.SS Output & File Control +.TP +.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT +Output destination directory (default: \fI./-NVFP4\fR, or \fI./\fR with +\fB\-\-no\-quant\fR and \fBauto\fR). +.TP +.B \-\-overwrite +Overwrite existing files in the output directory without prompting. +.TP +.BI \-\-format " " {hf,modelopt,comfy} +Safetensors layout format. \fBhf\fR (default) produces standard Hugging Face +\fBcompressed-tensors\fR layout (\fI.weight_packed\fR, \fI.weight_scale\fR, +and \fI.weight_global_scale\fR) natively supported by Hugging Face transformers, +vLLM, SGLang, and llama.cpp. \fBmodelopt\fR (or \fBcomfy\fR) produces raw tensors +(\fI.weight\fR, \fI.weight_scale\fR, \fI.weight_scale_2\fR, and \fI.comfy_quant\fR). +.TP +.BI \-\-hf\-template " " DIR +Reference Hugging Face model directory to copy config, tokenizer, and template files from. +.TP +.BI \-\-context\-length " " LEN +Override the maximum sequence/context length recorded in \fIconfig.json\fR. +.TP +.BI \-\-model\-name " " NAME +Override the model name. +.SS Quantization Mode +.TP +.B \-\-no\-quant +Skip learned rounding. Rewrites tensors as float weights. +.TP +.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4} +Data type for tensors when \fB\-\-no\-quant\fR is used (default: \fBauto\fR). +.TP +.BI \-\-output\-dtype " " {bfloat16,float16} +Data type used for unquantized 2D weights, layer norms, and embeddings (default: \fBbfloat16\fR). +.TP +.B \-\-simple +Round weights directly onto the E2M1 grid without iterative learned rounding. +.SS Optimization & Learning Rate +.TP +.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original} +Optimizer for learned rounding. \fBauto\fR selects Prodigy if installed, otherwise AdamW (default: \fBauto\fR). +.TP +.BI \-\-num\-iter " " N +Maximum optimization iterations per weight matrix (default: 4000). +.TP +.BI \-\-lr " " LR +Initial learning rate (default: 1.0). +.TP +.BI \-\-lr\-patience " " N +Patience steps before decaying the learning rate (default: 5). +.TP +.BI \-\-lr\-min " " MIN +Minimum learning rate floor (default: 1e-8). +.TP +.BI \-\-lr\-schedule " " {plateau,exponential,adaptive} +Learning rate decay schedule (default: \fBplateau\fR). +.TP +.BI \-\-scale\-optimization " " {fixed,iterative,joint} +Scale factor optimization strategy. \fBjoint\fR optimizes block scales simultaneously with codes (default: \fBjoint\fR). +.TP +.BI \-\-scale\-refinement\-rounds " " N +Number of scale refinement passes when using iterative scale optimization (default: 2). +.SS SVD Low-Rank Approximation +.TP +.B \-\-full\-matrix +Perform full SVD instead of randomized low-rank SVD. +.TP +.BI \-\-top\-p " " P +Cumulative energy threshold for singular value truncation (default: 0.2). +.TP +.BI \-\-min\-k " " K +Minimum number of singular vectors retained (default: 128). +.TP +.BI \-\-max\-k " " K +Maximum number of singular vectors retained (default: 1280). +.SS Early Stopping +.TP +.BI \-\-target\-loss " " TARGET +Stop optimization once reconstruction loss reaches this threshold (default: 1e-5). Set 0 to disable. +.TP +.BI \-\-early\-stop\-stall " " STALL +Stop after this many steps without loss improvement (default: 2000). +.TP +.BI \-\-early\-stop\-floor\-stall " " FLOOR +Stop after this many steps without improvement once learning rate reaches its floor (default: 64). +.TP +.BI \-\-early\-stop\-loss " " LOSS +Early stopping absolute loss threshold (default: 5e-9). +.SS Calibration & Bias Correction +.TP +.BI \-\-calib\-samples " " SAMPLES +Number of random normal calibration samples used for bias error compensation (default: 3072). +.TP +.BI \-\-seed " " SEED +Random seed for calibration data generation (default: 42). +.TP +.B \-\-no\-bias\-correction +Leave bias tensors unaltered without applying calibration-based error correction. +.SS Layer Selection +.TP +.B \-\-quantize\-output +Also quantize the final language model head (\fIlm_head.weight\fR). By default, \fIlm_head\fR remains unquantized. +.TP +.BI \-\-exclude " " REGEX ", " \-\-exclude\-layers " " REGEX +Regular expression matching tensor names that should remain unquantized in float format. +.SS Checkpoints & Resuming +.TP +.BI \-\-checkpoint\-dir " " DIR +Directory to store and load intermediate per-tensor quantization checkpoints. +.TP +.B \-\-fresh +Delete existing per-tensor checkpoints before starting. +.TP +.B \-\-keep\-checkpoints +Retain per-tensor checkpoints upon successful completion. +.SS Execution & Environment +.TP +.BI \-\-device " " DEVICE +Compute device: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR). +.TP +.B \-\-eager +Load tensors eagerly into memory. +.TP +.B \-\-low\-memory +Stream tensors sequentially to minimize memory footprint. +.TP +.B \-\-heur +Skip layers that convert_to_quant's heuristic rejects. +.TP +.BI \-\-convert\-to\-quant " " PATH +Path to convert_to_quant source checkout if not installed in Python environment. +.TP +.BI \-\-llama\-cpp " " PATH +Path to llama.cpp source checkout (default: \fI~/llama.cpp\fR). +.SS Diagnostics +.TP +.B \-\-verbose +Enable detailed progress and debugging output. +.TP +.B \-\-self\-test +Run GGUF-to-HF unpack and roundtrip self-test and exit. +.TP +.B \-h\fR, \fB\-\-help +Show command-line help and exit. +.SH EXAMPLES +.PP +Convert and quantize a GGUF model to NVFP4 HF format: +.PP +.nf +.RS +gguf_to_nvfp4_hf -o ./meta-llama-3.1-8b-nvfp4 ./meta-llama-3.1-8b.gguf +.RE +.fi +.PP +Convert GGUF to HF using an upstream HF repo as template for tokenizers: +.PP +.nf +.RS +gguf_to_nvfp4_hf --hf-template ./upstream-hf-dir -o ./output-hf-dir ./model.gguf +.RE +.fi +.SH SEE ALSO +.BR hf_to_nvfp4_hf (1), +.BR hf_to_nvfp4_gguf (1), +.BR gguf_to_nvfp4_gguf (1), +.BR safetensors_to_nvfp4_gguf (1) + diff --git a/man/man1/gguf_to_nvfp4_hf.py.1 b/man/man1/gguf_to_nvfp4_hf.py.1 new file mode 120000 index 0000000..3ed54a7 --- /dev/null +++ b/man/man1/gguf_to_nvfp4_hf.py.1 @@ -0,0 +1 @@ +gguf_to_nvfp4_hf.1 \ No newline at end of file diff --git a/man/man1/hf_to_nvfp4_hf.1 b/man/man1/hf_to_nvfp4_hf.1 new file mode 100644 index 0000000..45c2c4d --- /dev/null +++ b/man/man1/hf_to_nvfp4_hf.1 @@ -0,0 +1,231 @@ +.TH HF_TO_NVFP4_HF 1 "November 2026" "nvfp4_gguf_quant" "User Commands" +.SH NAME +hf_to_nvfp4_hf \- quantize Hugging Face models to NVFP4 Hugging Face format +.SH SYNOPSIS +.B hf_to_nvfp4_hf +[\fB\-o\fR \fIOUTPUT\fR] +[\fB\-\-overwrite\fR] +[\fB\-\-format\fR \fIFORMAT\fR] +[\fB\-\-no\-quant\fR] +[\fB\-\-outtype\fR \fITYPE\fR] +[\fB\-\-output\-dtype\fR \fIDTYPE\fR] +[\fB\-\-context\-length\fR \fILEN\fR] +[\fB\-\-model\-name\fR \fINAME\fR] +[\fB\-\-checkpoint\-dir\fR \fIDIR\fR] +[\fB\-\-fresh\fR] +[\fB\-\-keep\-checkpoints\fR] +[\fB\-\-eager\fR] +[\fB\-\-low\-memory\fR] +[\fB\-\-heur\fR] +[\fB\-\-verbose\fR] +[\fB\-\-self\-test\fR] +[\fB\-\-llama\-cpp\fR \fIPATH\fR] +[\fB\-\-convert\-to\-quant\fR \fIPATH\fR] +[\fB\-\-device\fR \fIDEVICE\fR] +[\fB\-\-simple\fR] +[\fB\-\-optimizer\fR \fIOPT\fR] +[\fB\-\-num\-iter\fR \fIN\fR] +[\fB\-\-lr\fR \fILR\fR] +[\fB\-\-lr\-patience\fR \fIN\fR] +[\fB\-\-lr\-min\fR \fIMIN\fR] +[\fB\-\-lr\-schedule\fR \fISCHED\fR] +[\fB\-\-scale\-optimization\fR \fIMODE\fR] +[\fB\-\-scale\-refinement\-rounds\fR \fIN\fR] +[\fB\-\-full\-matrix\fR] +[\fB\-\-top\-p\fR \fIP\fR] +[\fB\-\-min\-k\fR \fIK\fR] +[\fB\-\-max\-k\fR \fIK\fR] +[\fB\-\-early\-stop\-loss\fR \fILOSS\fR] +[\fB\-\-early\-stop\-stall\fR \fISTALL\fR] +[\fB\-\-target\-loss\fR \fITARGET\fR] +[\fB\-\-early\-stop\-floor\-stall\fR \fIFLOOR\fR] +[\fB\-\-calib\-samples\fR \fISAMPLES\fR] +[\fB\-\-seed\fR \fISEED\fR] +[\fB\-\-no\-bias\-correction\fR] +[\fB\-\-quantize\-output\fR] +[\fB\-\-exclude\fR \fIREGEX\fR] +[\fB\-h\fR|\fB\-\-help\fR] +\fImodel\fR +.SH DESCRIPTION +.B hf_to_nvfp4_hf +quantizes a Hugging Face model directory (\fIconfig.json\fR + safetensors) to +NVIDIA FP4 (NVFP4) format packaged into a standard Hugging Face model directory. +.PP +The output directory preserves all non-weight metadata, configuration, tokenizer files, +vocabulary, chat templates, and generation configs. For each quantized weight matrix, +the script writes ModelOpt-compatible packed 4-bit weights alongside per-block E4M3 scale +factors (\fI.weight_scale\fR) and per-tensor scale factors (\fI.weight_scale_2\fR). +.PP +Weights are quantized using learned rounding (via \fBconvert_to_quant\fR) or simple nearest-grid +projection onto the E2M1 floating point grid with two-level scaling. Biases are adjusted +using simulated calibration data unless disabled. Checkpoint-based resumption is supported +per tensor so interrupted runs can resume without recomputing. +.SH POSITIONAL ARGUMENTS +.TP +.I model +Path to a Hugging Face model directory containing \fIconfig.json\fR and one or more \fI.safetensors\fR files. +.SH OPTIONS +.SS Output & File Control +.TP +.BI \-o " " OUTPUT ", " \-\-output " " OUTPUT +Output destination directory (default: \fI./-NVFP4\fR, or \fI./\fR with +\fB\-\-no\-quant\fR and \fBauto\fR). +.TP +.B \-\-overwrite +Overwrite existing files in the output directory without prompting. +.TP +.BI \-\-format " " {hf,modelopt,comfy} +Safetensors layout format. \fBhf\fR (default) produces standard Hugging Face +\fBcompressed-tensors\fR layout (\fI.weight_packed\fR, \fI.weight_scale\fR, +and \fI.weight_global_scale\fR) natively supported by Hugging Face transformers, +vLLM, SGLang, and llama.cpp. \fBmodelopt\fR (or \fBcomfy\fR) produces raw tensors +(\fI.weight\fR, \fI.weight_scale\fR, \fI.weight_scale_2\fR, and \fI.comfy_quant\fR). +.TP +.BI \-\-context\-length " " LEN +Override the maximum sequence/context length recorded in \fIconfig.json\fR. +.TP +.BI \-\-model\-name " " NAME +Override the model name. +.SS Quantization Mode +.TP +.B \-\-no\-quant +Skip learned rounding. Rewrites or repacks tensors without quantizing. +.TP +.BI \-\-outtype " " {auto,f32,f16,bf16,nvfp4} +Data type for non-quantized weights when learned rounding is skipped (default: \fBauto\fR). +.TP +.BI \-\-output\-dtype " " {bfloat16,float16} +Data type used for unquantized 2D weights, layer norms, and embeddings (default: \fBbfloat16\fR). +.TP +.B \-\-simple +Round weights directly onto the E2M1 grid without iterative learned rounding. +.SS Optimization & Learning Rate +.TP +.BI \-\-optimizer " " {auto,prodigy,adamw,radam,original} +Optimizer for learned rounding. \fBauto\fR selects Prodigy if installed, otherwise AdamW (default: \fBauto\fR). +.TP +.BI \-\-num\-iter " " N +Maximum optimization iterations per weight matrix (default: 4000). +.TP +.BI \-\-lr " " LR +Initial learning rate (default: 1.0). +.TP +.BI \-\-lr\-patience " " N +Patience steps before decaying the learning rate (default: 5). +.TP +.BI \-\-lr\-min " " MIN +Minimum learning rate floor (default: 1e-8). +.TP +.BI \-\-lr\-schedule " " {plateau,exponential,adaptive} +Learning rate decay schedule (default: \fBplateau\fR). +.TP +.BI \-\-scale\-optimization " " {fixed,iterative,joint} +Scale factor optimization strategy. \fBjoint\fR optimizes block scales simultaneously with codes (default: \fBjoint\fR). +.TP +.BI \-\-scale\-refinement\-rounds " " N +Number of scale refinement passes when using iterative scale optimization (default: 2). +.SS SVD Low-Rank Approximation +.TP +.B \-\-full\-matrix +Perform full SVD instead of randomized low-rank SVD. +.TP +.BI \-\-top\-p " " P +Cumulative energy threshold for singular value truncation (default: 0.2). +.TP +.BI \-\-min\-k " " K +Minimum number of singular vectors retained (default: 128). +.TP +.BI \-\-max\-k " " K +Maximum number of singular vectors retained (default: 1280). +.SS Early Stopping +.TP +.BI \-\-target\-loss " " TARGET +Stop optimization once reconstruction loss reaches this threshold (default: 1e-5). Set 0 to disable. +.TP +.BI \-\-early\-stop\-stall " " STALL +Stop after this many steps without loss improvement (default: 2000). +.TP +.BI \-\-early\-stop\-floor\-stall " " FLOOR +Stop after this many steps without improvement once learning rate reaches its floor (default: 64). +.TP +.BI \-\-early\-stop\-loss " " LOSS +Early stopping absolute loss threshold (default: 5e-9). +.SS Calibration & Bias Correction +.TP +.BI \-\-calib\-samples " " SAMPLES +Number of random normal calibration samples used for bias error compensation (default: 3072). +.TP +.BI \-\-seed " " SEED +Random seed for calibration data generation (default: 42). +.TP +.B \-\-no\-bias\-correction +Leave bias tensors unaltered without applying calibration-based error correction. +.SS Layer Selection +.TP +.B \-\-quantize\-output +Also quantize the final language model head (\fIlm_head.weight\fR). By default, \fIlm_head\fR remains unquantized. +.TP +.BI \-\-exclude " " REGEX ", " \-\-exclude\-layers " " REGEX +Regular expression matching tensor names that should remain unquantized in float format. +.SS Checkpoints & Resuming +.TP +.BI \-\-checkpoint\-dir " " DIR +Directory to store and load intermediate per-tensor quantization checkpoints. +.TP +.B \-\-fresh +Delete existing per-tensor checkpoints before starting. +.TP +.B \-\-keep\-checkpoints +Retain per-tensor checkpoints upon successful completion. +.SS Execution & Environment +.TP +.BI \-\-device " " DEVICE +Compute device: \fBauto\fR, \fBcpu\fR, \fBcuda\fR, or \fBcuda:N\fR (default: \fBauto\fR). +.TP +.B \-\-eager +Load tensors eagerly into memory. +.TP +.B \-\-low\-memory +Stream shards sequentially to minimize memory footprint. +.TP +.B \-\-heur +Skip layers that convert_to_quant's heuristic rejects. +.TP +.BI \-\-convert\-to\-quant " " PATH +Path to convert_to_quant source checkout if not installed in Python environment. +.TP +.BI \-\-llama\-cpp " " PATH +Path to llama.cpp source checkout if needed (default: \fI~/llama.cpp\fR). +.SS Diagnostics +.TP +.B \-\-verbose +Enable detailed progress and debugging output. +.TP +.B \-\-self\-test +Run NVFP4 packing and roundtrip self-test and exit. +.TP +.B \-h\fR, \fB\-\-help +Show command-line help and exit. +.SH EXAMPLES +.PP +Quantize a Hugging Face model directory to NVFP4 HF format: +.PP +.nf +.RS +hf_to_nvfp4_hf -o ./meta-llama-3.1-8b-nvfp4 ./meta-llama-3.1-8b +.RE +.fi +.PP +Quantize with fast simple grid rounding without learned iterations: +.PP +.nf +.RS +hf_to_nvfp4_hf --simple -o ./my-model-nvfp4 ./my-model +.RE +.fi +.SH SEE ALSO +.BR hf_to_nvfp4_gguf (1), +.BR gguf_to_nvfp4_hf (1), +.BR hf_to_nvfp4_safetensors (1), +.BR gguf_to_nvfp4_gguf (1) + diff --git a/man/man1/hf_to_nvfp4_hf.py.1 b/man/man1/hf_to_nvfp4_hf.py.1 new file mode 120000 index 0000000..698b098 --- /dev/null +++ b/man/man1/hf_to_nvfp4_hf.py.1 @@ -0,0 +1 @@ +hf_to_nvfp4_hf.1 \ No newline at end of file diff --git a/nvfp4_quant b/nvfp4_quant index 1763fd3..607b4fc 100755 --- a/nvfp4_quant +++ b/nvfp4_quant @@ -1,9 +1,6 @@ #!/bin/bash TAG="${TAG:-latest}" CMD="$(basename "${0}")" -if [ -z "${USER}" ]; then - USER="$(whoami)" -fi IMAGE="${IMAGE:-zefie/nvfp4_quant:${TAG}}" COMMON=(docker run --user 1000:1000 --rm --gpus all -it -v ./:/workspace "${IMAGE}")