initial commit
This commit is contained in:
zefie committed
commit
db7f28d087
31 files changed
+7462
No files matched your search
@@ -0,0 +1,667 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quantize a GGUF to llama.cpp NVFP4, or repack one that is already NVFP4.
|
||||
|
||||
The learned-rounding flags match hf_to_nvfp4_gguf.py. Vocabulary, tensor names,
|
||||
and architecture metadata are copied from the source GGUF. --outtype nvfp4
|
||||
copies an existing NVFP4 file without fitting it again. --no-quant with
|
||||
f32, f16, bf16, or auto rewrites the tensors as float, or leaves them as stored.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from nvfp4gguf import (
|
||||
CTQ_NUM_ITER,
|
||||
DEFAULT_CONVERT_TO_QUANT,
|
||||
DEFAULT_LLAMA_CPP,
|
||||
DEFAULT_EARLY_STOP_FLOOR_STALL,
|
||||
DEFAULT_TARGET_LOSS,
|
||||
DEFAULT_EARLY_STOP_STALL,
|
||||
DEFAULT_SCALE_OPTIMIZATION,
|
||||
DEFAULT_SCALE_REFINEMENT_ROUNDS,
|
||||
QUANT_VERSION,
|
||||
TensorQuantizer,
|
||||
bind_exclude,
|
||||
eligible_weight,
|
||||
ensure_cuda_interpreter,
|
||||
import_gguf,
|
||||
load_learned_converter,
|
||||
must_bake_nvfp4_scale,
|
||||
mtp_layer_tensor,
|
||||
bake_packed_nvfp4,
|
||||
preserve_scale_tensor,
|
||||
quantize_and_pack,
|
||||
report_reconstruction,
|
||||
require_quant_settings,
|
||||
resolve_optimizer,
|
||||
scale_bake_notice,
|
||||
)
|
||||
from nvfp4gguf.bundle import (
|
||||
flatten_chat_template,
|
||||
normalize_chat_template,
|
||||
is_model_context_key,
|
||||
outtype as _outtype,
|
||||
refuse_existing_bundle,
|
||||
require_context_length,
|
||||
scalar_context_length,
|
||||
warn_missing_gguf_metadata,
|
||||
write_simple_modelfile,
|
||||
)
|
||||
|
||||
log = logging.getLogger("gguf_to_nvfp4_gguf")
|
||||
|
||||
|
||||
def _checkpoint_path(directory: Path, name: str) -> Path:
|
||||
digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:16]
|
||||
safe = re.sub(r"[^A-Za-z0-9._-]+", "_", name)[:80]
|
||||
return directory / f"{safe}.{digest}.npz"
|
||||
|
||||
|
||||
def _pythonize(value):
|
||||
if isinstance(value, np.generic):
|
||||
return value.item()
|
||||
if isinstance(value, np.ndarray):
|
||||
return value.tolist()
|
||||
if isinstance(value, list):
|
||||
return [_pythonize(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _text(value) -> str:
|
||||
value = _pythonize(value)
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", "replace")
|
||||
if isinstance(value, list):
|
||||
return "".join(_text(item) for item in value)
|
||||
return str(value)
|
||||
|
||||
|
||||
def _logical_shape(tensor) -> tuple[int, ...]:
|
||||
return tuple(int(dim) for dim in tensor.shape[::-1])
|
||||
|
||||
|
||||
def _to_float(tensor, gguf_module) -> np.ndarray:
|
||||
qtype = tensor.tensor_type
|
||||
if qtype == gguf_module.GGMLQuantizationType.F32:
|
||||
values = np.asarray(tensor.data, dtype=np.float32)
|
||||
elif qtype == gguf_module.GGMLQuantizationType.F16:
|
||||
values = np.asarray(tensor.data, dtype=np.float32)
|
||||
else:
|
||||
values = np.asarray(gguf_module.quants.dequantize(tensor.data, qtype), dtype=np.float32)
|
||||
# The reader maps the file read-only. Torch refuses a non-writable view.
|
||||
return np.array(values, dtype=np.float32, copy=True)
|
||||
|
||||
|
||||
def _open_reader(gguf, path: Path):
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"GGUF file does not exist: {path}")
|
||||
reader = gguf.GGUFReader(path)
|
||||
if reader.byte_order != "I":
|
||||
raise SystemExit(f"{path} is not little-endian GGUF")
|
||||
return reader
|
||||
|
||||
|
||||
def _architecture(reader) -> str:
|
||||
field = reader.fields.get("general.architecture")
|
||||
if field is None:
|
||||
return "unknown"
|
||||
return _text(field.contents())
|
||||
|
||||
|
||||
def _file_type_name(reader, gguf_module) -> str:
|
||||
field = reader.fields.get("general.file_type")
|
||||
if field is None:
|
||||
return ""
|
||||
try:
|
||||
return gguf_module.LlamaFileType(int(field.contents())).name
|
||||
except (TypeError, ValueError):
|
||||
return str(field.contents())
|
||||
|
||||
|
||||
def _is_nvfp4(reader, gguf_module) -> bool:
|
||||
if _file_type_name(reader, gguf_module) == "MOSTLY_NVFP4":
|
||||
return True
|
||||
return any(tensor.tensor_type == gguf_module.GGMLQuantizationType.NVFP4 for tensor in reader.tensors)
|
||||
|
||||
|
||||
def _copy_metadata(reader, writer, gguf_module, file_type: int | None, model_name: str | None, context_length: int | None) -> None:
|
||||
requested = require_context_length(context_length)
|
||||
source_context = None
|
||||
wrote_context = False
|
||||
for field in reader.fields.values():
|
||||
if field.name.startswith("GGUF.") or field.name == "general.architecture" or not field.types:
|
||||
continue
|
||||
value = _pythonize(field.contents())
|
||||
if value is None:
|
||||
continue
|
||||
value_type = field.types[0]
|
||||
if value_type == gguf_module.GGUFValueType.ARRAY and not value:
|
||||
continue
|
||||
if field.name == "general.file_type" and file_type is not None:
|
||||
value = int(file_type)
|
||||
if field.name == "general.name" and model_name:
|
||||
value = model_name
|
||||
if field.name == "general.alignment":
|
||||
writer.data_alignment = int(value)
|
||||
subtype = field.types[-1] if value_type == gguf_module.GGUFValueType.ARRAY else None
|
||||
if field.name == "tokenizer.chat_template" or field.name.startswith("tokenizer.chat_template."):
|
||||
value = normalize_chat_template(value)
|
||||
if is_model_context_key(field.name):
|
||||
source_context = scalar_context_length(value)
|
||||
if requested is not None:
|
||||
value = requested
|
||||
value_type = gguf_module.GGUFValueType.UINT32
|
||||
subtype = None
|
||||
wrote_context = True
|
||||
writer.add_key_value(field.name, value, value_type, sub_type=subtype)
|
||||
if file_type is not None and "general.file_type" not in reader.fields:
|
||||
writer.add_file_type(int(file_type))
|
||||
if model_name and "general.name" not in reader.fields:
|
||||
writer.add_name(model_name)
|
||||
if requested is not None and not wrote_context:
|
||||
writer.add_context_length(requested)
|
||||
if requested is not None:
|
||||
log.info("context length %d", requested)
|
||||
elif source_context is not None:
|
||||
log.info("context length %d from the source GGUF", source_context)
|
||||
|
||||
|
||||
def _replaced_scale(name: str, quant_names: set[str], architecture: str) -> bool:
|
||||
if not (name.endswith(".scale") or name.endswith(".input_scale")):
|
||||
return False
|
||||
stem = name.rsplit(".", 1)[0]
|
||||
weight = stem + ".weight"
|
||||
if weight not in quant_names:
|
||||
return False
|
||||
return not (name.endswith(".scale") and preserve_scale_tensor(architecture, weight))
|
||||
|
||||
|
||||
def _keep_f32(name: str, rank: int) -> bool:
|
||||
return rank < 2 or "norm" in name.lower()
|
||||
|
||||
|
||||
def _add_float(writer, gguf_module, name: str, values: np.ndarray, outtype: str) -> None:
|
||||
values = np.ascontiguousarray(values, dtype=np.float32)
|
||||
if outtype == "f16":
|
||||
writer.add_tensor(name, values.astype(np.float16))
|
||||
return
|
||||
if outtype == "bf16":
|
||||
raw = gguf_module.quants.quantize(values, gguf_module.GGMLQuantizationType.BF16)
|
||||
writer.add_tensor(name, raw, raw_dtype=gguf_module.GGMLQuantizationType.BF16)
|
||||
return
|
||||
writer.add_tensor(name, values)
|
||||
|
||||
|
||||
def _copy_tensor(writer, tensor) -> None:
|
||||
writer.add_tensor(tensor.name, tensor.data, raw_dtype=tensor.tensor_type)
|
||||
|
||||
|
||||
def _load_checkpoint(directory: Path, name: str, shape: tuple[int, ...]):
|
||||
path = _checkpoint_path(directory, name)
|
||||
if not path.is_file():
|
||||
return None
|
||||
with np.load(path) as blob:
|
||||
if int(blob["version"]) != QUANT_VERSION:
|
||||
return None
|
||||
if tuple(int(v) for v in blob["shape"]) != tuple(shape):
|
||||
return None
|
||||
raw = np.array(blob["raw"], copy=True)
|
||||
scale = np.array(blob["scale"], copy=True)
|
||||
bias = np.array(blob["bias"], copy=True)
|
||||
return raw, None if scale.size == 0 else scale, None if bias.size == 0 else bias
|
||||
|
||||
|
||||
def _save_checkpoint(directory: Path, name: str, shape: tuple[int, ...], raw, scale, bias) -> None:
|
||||
path = _checkpoint_path(directory, name)
|
||||
temporary = path.with_name(path.name + ".tmp")
|
||||
with temporary.open("wb") as handle:
|
||||
np.savez_compressed(
|
||||
handle,
|
||||
version=np.int32(QUANT_VERSION),
|
||||
shape=np.asarray(shape, dtype=np.int64),
|
||||
raw=raw,
|
||||
scale=np.zeros((0,), np.float32) if scale is None else np.asarray(scale, dtype=np.float32),
|
||||
bias=np.zeros((0,), np.float32) if bias is None else np.asarray(bias, dtype=np.float32),
|
||||
)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def _assign_output(args) -> None:
|
||||
source = Path(args.gguf)
|
||||
stem = source.stem
|
||||
passthrough = args.no_quant or args.outtype == "nvfp4"
|
||||
if args.output is None:
|
||||
if passthrough and args.outtype == "auto":
|
||||
directory = Path.cwd() / stem
|
||||
gguf_name = f"{stem}.gguf"
|
||||
elif passthrough:
|
||||
label = "NVFP4" if args.outtype == "nvfp4" else args.outtype.upper()
|
||||
directory = Path.cwd() / f"{stem}-{label}"
|
||||
gguf_name = f"{directory.name}.gguf"
|
||||
else:
|
||||
directory = Path.cwd() / f"{stem}-NVFP4"
|
||||
gguf_name = f"{directory.name}.gguf"
|
||||
else:
|
||||
output = Path(args.output)
|
||||
if output.suffix == ".gguf":
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output = str(output.resolve())
|
||||
return
|
||||
directory = output
|
||||
gguf_name = f"{directory.name}.gguf"
|
||||
directory = directory.resolve()
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
args.output = str(directory / gguf_name)
|
||||
|
||||
|
||||
def _refuse_same_file(args) -> None:
|
||||
if Path(args.output).resolve() == Path(args.gguf).resolve():
|
||||
raise SystemExit("Output GGUF is the same file as the input. Choose a different --output.")
|
||||
|
||||
|
||||
def _target_file_type(gguf_module, outtype: str, source_name: str):
|
||||
if outtype == "nvfp4":
|
||||
return gguf_module.LlamaFileType.MOSTLY_NVFP4
|
||||
if outtype == "f32":
|
||||
return gguf_module.LlamaFileType.ALL_F32
|
||||
if outtype == "f16":
|
||||
return gguf_module.LlamaFileType.MOSTLY_F16
|
||||
if outtype == "bf16":
|
||||
return gguf_module.LlamaFileType.MOSTLY_BF16
|
||||
if source_name == "ALL_F32":
|
||||
return gguf_module.LlamaFileType.ALL_F32
|
||||
if source_name == "MOSTLY_F16":
|
||||
return gguf_module.LlamaFileType.MOSTLY_F16
|
||||
if source_name == "MOSTLY_BF16":
|
||||
return gguf_module.LlamaFileType.MOSTLY_BF16
|
||||
if source_name == "MOSTLY_NVFP4":
|
||||
return gguf_module.LlamaFileType.MOSTLY_NVFP4
|
||||
return None
|
||||
|
||||
|
||||
def _chat_template(reader) -> str:
|
||||
parts = []
|
||||
for name, field in reader.fields.items():
|
||||
if name == "tokenizer.chat_template" or name.startswith("tokenizer.chat_template."):
|
||||
parts.append(flatten_chat_template(field.contents()))
|
||||
return "\n".join(part for part in parts if part)
|
||||
|
||||
|
||||
def _kv_int(reader, key: str) -> int:
|
||||
field = reader.fields.get(key)
|
||||
if field is None:
|
||||
return 0
|
||||
try:
|
||||
return int(field.contents())
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _mtp_counts(reader, architecture: str) -> tuple[int, int]:
|
||||
return _kv_int(reader, f"{architecture}.block_count"), _kv_int(reader, f"{architecture}.nextn_predict_layers")
|
||||
|
||||
|
||||
def _folded_scales(reader, gguf_module, architecture: str, n_blocks: int, n_nextn: int) -> dict[str, float]:
|
||||
"""Per-tensor NVFP4 scales the llama.cpp graph never multiplies."""
|
||||
by_name = {tensor.name: tensor for tensor in reader.tensors}
|
||||
nvfp4 = gguf_module.GGMLQuantizationType.NVFP4
|
||||
scales: dict[str, float] = {}
|
||||
for tensor in reader.tensors:
|
||||
if not tensor.name.endswith(".scale"):
|
||||
continue
|
||||
weight_name = tensor.name[: -len(".scale")] + ".weight"
|
||||
weight = by_name.get(weight_name)
|
||||
if weight is None or weight.tensor_type != nvfp4:
|
||||
continue
|
||||
if preserve_scale_tensor(architecture, weight_name):
|
||||
continue
|
||||
bias_name = weight_name[: -len(".weight")] + ".bias"
|
||||
fold = mtp_layer_tensor(weight_name, n_blocks, n_nextn) or must_bake_nvfp4_scale(
|
||||
architecture, weight_name, bias_name in by_name
|
||||
)
|
||||
if not fold:
|
||||
continue
|
||||
values = np.asarray(tensor.data, dtype=np.float32).reshape(-1)
|
||||
if values.size != 1:
|
||||
log.warning(
|
||||
"Leaving %s in place. Baking expects one per-tensor scale, got %d values.",
|
||||
tensor.name,
|
||||
int(values.size),
|
||||
)
|
||||
continue
|
||||
scales[tensor.name] = float(values[0])
|
||||
return scales
|
||||
|
||||
|
||||
def export_copy(args) -> None:
|
||||
"""Rewrite the GGUF without learned rounding."""
|
||||
gguf = import_gguf(args.llama_cpp)
|
||||
source = Path(args.gguf).resolve()
|
||||
output = Path(args.output).resolve()
|
||||
reader = _open_reader(gguf, source)
|
||||
if args.outtype == "nvfp4" and not _is_nvfp4(reader, gguf):
|
||||
raise SystemExit(
|
||||
f"{source} is not an NVFP4 GGUF. "
|
||||
"Use --outtype f16, bf16, f32, or auto to write float weights, "
|
||||
"or omit --outtype nvfp4 to fit NVFP4 from these weights."
|
||||
)
|
||||
architecture = _architecture(reader)
|
||||
source_type = _file_type_name(reader, gguf)
|
||||
file_type = _target_file_type(gguf, args.outtype, source_type)
|
||||
if args.outtype == "nvfp4":
|
||||
log.info("Copying existing NVFP4 tensors. Learned rounding is skipped.")
|
||||
elif args.outtype == "auto":
|
||||
log.info("Copying tensors as stored (%s). Learned rounding is skipped.", source_type or "unknown file type")
|
||||
else:
|
||||
log.info("Writing float GGUF (%s). Learned rounding is skipped.", args.outtype)
|
||||
writer = gguf.GGUFWriter(output, architecture, use_temp_file=not args.no_temp_file)
|
||||
_copy_metadata(reader, writer, gguf, None if file_type is None else int(file_type), args.model_name, args.context_length)
|
||||
n_blocks, n_nextn = _mtp_counts(reader, architecture)
|
||||
folded = _folded_scales(reader, gguf, architecture, n_blocks, n_nextn) if args.outtype in ("auto", "nvfp4") else {}
|
||||
if folded:
|
||||
log.info(
|
||||
"Folding %d scale tensors into their NVFP4 blocks. "
|
||||
"llama.cpp does not claim those sidecars.",
|
||||
len(folded),
|
||||
)
|
||||
for tensor in reader.tensors:
|
||||
if tensor.name in folded:
|
||||
continue
|
||||
scale_name = tensor.name[: -len(".weight")] + ".scale" if tensor.name.endswith(".weight") else ""
|
||||
if scale_name in folded and tensor.tensor_type == gguf.GGMLQuantizationType.NVFP4:
|
||||
baked = bake_packed_nvfp4(np.array(tensor.data, copy=True), folded[scale_name])
|
||||
writer.add_tensor(tensor.name, baked, raw_dtype=gguf.GGMLQuantizationType.NVFP4)
|
||||
continue
|
||||
if args.outtype in ("auto", "nvfp4"):
|
||||
_copy_tensor(writer, tensor)
|
||||
continue
|
||||
if _keep_f32(tensor.name, len(_logical_shape(tensor))):
|
||||
if tensor.tensor_type == gguf.GGMLQuantizationType.F32:
|
||||
_copy_tensor(writer, tensor)
|
||||
else:
|
||||
_add_float(writer, gguf, tensor.name, _to_float(tensor, gguf), "f32")
|
||||
continue
|
||||
_add_float(writer, gguf, tensor.name, _to_float(tensor, gguf), args.outtype)
|
||||
log.info("Writing %s", output)
|
||||
warn_missing_gguf_metadata(writer)
|
||||
writer.write_header_to_file()
|
||||
writer.write_kv_data_to_file()
|
||||
writer.write_tensors_to_file(progress=True)
|
||||
writer.close()
|
||||
write_simple_modelfile(output, args.overwrite, args.mmproj, _chat_template(reader))
|
||||
log.info("Wrote %s", output)
|
||||
|
||||
|
||||
def _emit_bias(writer, gguf_module, tensor, correction: np.ndarray | None) -> bool:
|
||||
if correction is None:
|
||||
_copy_tensor(writer, tensor)
|
||||
return False
|
||||
values = _to_float(tensor, gguf_module)
|
||||
flat = values.reshape(-1)
|
||||
if flat.shape != correction.shape:
|
||||
log.warning(" bias %s shape %s does not match correction %s", tensor.name, values.shape, correction.shape)
|
||||
_copy_tensor(writer, tensor)
|
||||
return False
|
||||
values = (flat - correction).reshape(values.shape)
|
||||
log.info(" bias %s adjusted (mean %+.3e)", tensor.name, float(correction.mean()))
|
||||
if tensor.tensor_type == gguf_module.GGMLQuantizationType.F16:
|
||||
writer.add_tensor(tensor.name, np.ascontiguousarray(values, dtype=np.float32).astype(np.float16))
|
||||
return True
|
||||
if tensor.tensor_type == gguf_module.GGMLQuantizationType.BF16:
|
||||
raw = gguf_module.quants.quantize(
|
||||
np.ascontiguousarray(values, dtype=np.float32),
|
||||
gguf_module.GGMLQuantizationType.BF16,
|
||||
)
|
||||
writer.add_tensor(tensor.name, raw, raw_dtype=gguf_module.GGMLQuantizationType.BF16)
|
||||
return True
|
||||
writer.add_tensor(tensor.name, np.ascontiguousarray(values, dtype=np.float32))
|
||||
return True
|
||||
|
||||
|
||||
def export_quant(args) -> None:
|
||||
gguf = import_gguf(args.llama_cpp)
|
||||
source = Path(args.gguf).resolve()
|
||||
output = Path(args.output).resolve()
|
||||
ensure_cuda_interpreter(args.device)
|
||||
reader = _open_reader(gguf, source)
|
||||
architecture = _architecture(reader)
|
||||
bind_exclude(args, architecture)
|
||||
notice = scale_bake_notice(architecture)
|
||||
if notice:
|
||||
log.warning(notice)
|
||||
file_type_name = _file_type_name(reader, gguf)
|
||||
if file_type_name not in {"ALL_F32", "MOSTLY_F16", "MOSTLY_BF16"}:
|
||||
log.warning(
|
||||
"Source file type is %s. NVFP4 will be fit to the dequantized weights, "
|
||||
"so detail already discarded by the source quant stays discarded.",
|
||||
file_type_name or "quantized",
|
||||
)
|
||||
names = {tensor.name for tensor in reader.tensors}
|
||||
n_blocks, n_nextn = _mtp_counts(reader, architecture)
|
||||
quant_names = [
|
||||
tensor.name
|
||||
for tensor in reader.tensors
|
||||
if eligible_weight(tensor.name, _logical_shape(tensor), args, gguf)
|
||||
]
|
||||
quant_name_set = set(quant_names)
|
||||
log.info("%d matrices queued for NVFP4, %d tensors copied", len(quant_names), len(reader.tensors) - len(quant_names))
|
||||
if not quant_names:
|
||||
log.warning("No matrices are eligible for NVFP4. Tensors are copied and the file type stays unchanged.")
|
||||
|
||||
work = Path(args.checkpoint_dir) if args.checkpoint_dir else Path(str(output) + ".nvfp4-work")
|
||||
if args.fresh and work.exists():
|
||||
shutil.rmtree(work)
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
require_quant_settings(work, args)
|
||||
|
||||
_converter_cls, unpack_uint4 = load_learned_converter(args.convert_to_quant)
|
||||
del _converter_cls
|
||||
quantizer = TensorQuantizer(args, gguf, unpack_uint4)
|
||||
writer = gguf.GGUFWriter(output, architecture, use_temp_file=not args.no_temp_file)
|
||||
_copy_metadata(
|
||||
reader, writer, gguf,
|
||||
int(gguf.LlamaFileType.MOSTLY_NVFP4) if quant_names else None,
|
||||
args.model_name,
|
||||
args.context_length,
|
||||
)
|
||||
|
||||
pending: dict[str, object] = {}
|
||||
corrections: dict[str, np.ndarray] = {}
|
||||
done = 0
|
||||
baked = 0
|
||||
corrected = 0
|
||||
try:
|
||||
for tensor in reader.tensors:
|
||||
if _replaced_scale(tensor.name, quant_name_set, architecture):
|
||||
continue
|
||||
weight_name = tensor.name[: -len(".bias")] + ".weight" if tensor.name.endswith(".bias") else ""
|
||||
if weight_name in quant_name_set and not args.no_bias_correction:
|
||||
if tensor.name not in corrections:
|
||||
pending[tensor.name] = tensor
|
||||
continue
|
||||
if _emit_bias(writer, gguf, tensor, corrections[tensor.name]):
|
||||
corrected += 1
|
||||
continue
|
||||
if tensor.name not in quant_name_set:
|
||||
_copy_tensor(writer, tensor)
|
||||
continue
|
||||
done += 1
|
||||
logical = _logical_shape(tensor)
|
||||
bias_name = tensor.name[: -len(".weight")] + ".bias"
|
||||
bake = must_bake_nvfp4_scale(architecture, tensor.name, bias_name in names) or mtp_layer_tensor(
|
||||
tensor.name, n_blocks, n_nextn
|
||||
)
|
||||
cached = _load_checkpoint(work, tensor.name, logical)
|
||||
if cached is not None and (cached[1] is None) != bake:
|
||||
log.info(" checkpoint scale layout does not match %s; recomputing", tensor.name)
|
||||
cached = None
|
||||
log.info(
|
||||
"NVFP4 [%d/%d] %s %s from %s%s",
|
||||
done, len(quant_names), tensor.name, logical, tensor.tensor_type.name,
|
||||
" (checkpoint)" if cached else "",
|
||||
)
|
||||
if cached is None:
|
||||
weight = _to_float(tensor, gguf)
|
||||
raw, scale = quantize_and_pack(quantizer, weight, bake, tensor.name)
|
||||
report_reconstruction(quantizer, weight, raw, scale)
|
||||
bias = None
|
||||
if weight.ndim == 2 and not args.no_bias_correction and bias_name in names:
|
||||
per_tensor = None if scale is None else float(np.reshape(scale, -1)[0])
|
||||
runtime = quantizer.runtime_dequant(raw if raw.ndim == 2 else raw[0], per_tensor)
|
||||
sample = weight if weight.ndim == 2 else weight[0]
|
||||
bias = quantizer.bias_correction(sample, runtime)
|
||||
_save_checkpoint(work, tensor.name, logical, raw, scale, bias)
|
||||
del weight
|
||||
else:
|
||||
raw, scale, bias = cached
|
||||
if bias is not None:
|
||||
corrections[bias_name] = bias
|
||||
pending_tensor = pending.pop(bias_name, None)
|
||||
if pending_tensor is not None and _emit_bias(writer, gguf, pending_tensor, bias):
|
||||
corrected += 1
|
||||
if bake and scale is None:
|
||||
baked += 1
|
||||
writer.add_tensor(tensor.name, raw, raw_dtype=gguf.GGMLQuantizationType.NVFP4)
|
||||
if scale is not None:
|
||||
writer.add_tensor(tensor.name[: -len(".weight")] + ".scale", np.ascontiguousarray(scale, dtype=np.float32))
|
||||
del raw
|
||||
for name, tensor in list(pending.items()):
|
||||
if _emit_bias(writer, gguf, tensor, corrections.get(name)):
|
||||
corrected += 1
|
||||
pending.clear()
|
||||
except KeyboardInterrupt:
|
||||
log.warning("Interrupted. Checkpoints are in %s; rerun the same command to resume.", work)
|
||||
raise
|
||||
|
||||
log.info("Writing %s", output)
|
||||
warn_missing_gguf_metadata(writer)
|
||||
writer.write_header_to_file()
|
||||
writer.write_kv_data_to_file()
|
||||
writer.write_tensors_to_file(progress=True)
|
||||
writer.close()
|
||||
if not args.keep_checkpoints:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
write_simple_modelfile(output, args.overwrite, args.mmproj, _chat_template(reader))
|
||||
log.info(
|
||||
"Wrote %s (%d NVFP4 tensors, %d with scale folded into the blocks, %d biases adjusted)",
|
||||
output, done, baked, corrected,
|
||||
)
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Quantize a GGUF to NVFP4, or repack one that is already NVFP4, with the same options as hf_to_nvfp4_gguf.py.",
|
||||
)
|
||||
parser.add_argument("gguf", nargs="?", help="Input GGUF")
|
||||
parser.add_argument("-o", "--output", help="Output directory. Default: ./<name>-NVFP4, or ./<name> with --no-quant and auto. A path ending in .gguf is the GGUF file itself.")
|
||||
parser.add_argument("--modelfile-only", action="store_true", help="Write the Modelfile and skip quantization and GGUF creation")
|
||||
parser.add_argument("--overwrite", action="store_true", help="Replace an existing GGUF, Modelfile, or copied projector")
|
||||
parser.add_argument("--mmproj", action="append", type=Path, default=None,
|
||||
help="Vision projector GGUF. Copied into the output directory under its own name and added as another FROM line. Repeat for more than one projector.")
|
||||
parser.add_argument("--context-length", type=int, default=None,
|
||||
help="Max context stored in the GGUF. Replaces a source value, or sets one when the source has no single context length.")
|
||||
parser.add_argument("--no-quant", action="store_true",
|
||||
help="Skip learned rounding. Writes a float GGUF, or copies an existing NVFP4 GGUF with --outtype nvfp4")
|
||||
parser.add_argument("--outtype", default="auto", type=_outtype, choices=("auto", "f32", "f16", "bf16", "nvfp4"),
|
||||
help="Type used when learned rounding is skipped. nvfp4 copies a GGUF that is already NVFP4. auto leaves tensors as stored (default: auto)")
|
||||
parser.add_argument("--llama-cpp", type=Path, default=DEFAULT_LLAMA_CPP,
|
||||
help="llama.cpp checkout used for the GGUF reader and writer (default: ~/llama.cpp)")
|
||||
parser.add_argument("--convert-to-quant", type=Path, default=None,
|
||||
help="convert_to_quant checkout, if it is not installed in the environment")
|
||||
parser.add_argument("--model-name", default=None, help="Override the model name stored in the GGUF")
|
||||
parser.add_argument("--device", default="auto", help="auto, cpu, cuda, or cuda:N")
|
||||
parser.add_argument("--simple", action="store_true", help="Round each value onto the E2M1 grid without learned rounding")
|
||||
parser.add_argument("--optimizer", default="auto", choices=("auto", "prodigy", "adamw", "radam", "original"),
|
||||
help="Learned-rounding optimizer. auto uses prodigy when it is installed, otherwise AdamW")
|
||||
parser.add_argument("--num-iter", type=int, default=CTQ_NUM_ITER, help="Learned-rounding iterations per matrix (convert_to_quant default: 4000)")
|
||||
parser.add_argument("--lr", type=float, default=1.0, help="Initial learning rate")
|
||||
parser.add_argument("--lr-min", type=float, default=1e-8, help="Lowest learning rate. The flat-loss check starts here (default: 1e-8).")
|
||||
parser.add_argument("--lr-schedule", default="plateau", choices=("plateau", "exponential", "adaptive"))
|
||||
parser.add_argument(
|
||||
"--scale-optimization", default=DEFAULT_SCALE_OPTIMIZATION, choices=("fixed", "iterative", "joint"),
|
||||
help="fixed keeps the initial scale. iterative refreshes it on the original optimizer. "
|
||||
"joint trains the block scales with the codes (default: joint)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scale-refinement-rounds", type=int, default=DEFAULT_SCALE_REFINEMENT_ROUNDS,
|
||||
help="Scale refreshes when --scale-optimization is iterative (default: 2). Ignored for fixed and joint.",
|
||||
)
|
||||
parser.add_argument("--full-matrix", action="store_true", help="Use a full SVD instead of a low-rank SVD")
|
||||
parser.add_argument("--top-p", type=float, default=0.2)
|
||||
parser.add_argument("--min-k", type=int, default=128)
|
||||
parser.add_argument("--max-k", type=int, default=1280)
|
||||
parser.add_argument("--early-stop-loss", type=float, default=5e-9)
|
||||
parser.add_argument(
|
||||
"--early-stop-stall", type=int, default=DEFAULT_EARLY_STOP_STALL,
|
||||
help="Stop after this many steps without a new best, at any learning rate (default: 2000)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target-loss", type=float, default=DEFAULT_TARGET_LOSS,
|
||||
help="Stop when the best loss reaches this value, at any learning rate (default: 1e-5). 0 disables it.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--early-stop-floor-stall", type=int, default=DEFAULT_EARLY_STOP_FLOOR_STALL,
|
||||
help="If the loss never reaches --target-loss, stop after this many steps without a new best once it has otherwise flattened. 0 disables it (default: 64)",
|
||||
)
|
||||
parser.add_argument("--calib-samples", type=int, default=3072, help="Random inputs used to correct a sibling bias")
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--no-bias-correction", action="store_true")
|
||||
parser.add_argument("--quantize-output", action="store_true", help="Also quantize the output (lm_head) matrix")
|
||||
parser.add_argument("--exclude", default=None, help="Regex of GGUF tensor names to leave in their source type")
|
||||
parser.add_argument("--checkpoint-dir", type=Path, default=None, help="Per-tensor resume directory")
|
||||
parser.add_argument("--fresh", action="store_true", help="Delete the checkpoint directory before starting")
|
||||
parser.add_argument("--keep-checkpoints", action="store_true")
|
||||
parser.add_argument("--eager", action="store_true", help="Accepted for the same command line as hf_to_nvfp4_gguf. The GGUF reader maps the file either way.")
|
||||
parser.add_argument("--no-temp-file", action="store_true", help="Hold the finished GGUF tensors in RAM instead of a temp file")
|
||||
parser.add_argument("--verbose", action="store_true")
|
||||
parser.add_argument("--self-test", action="store_true", help="Check NVFP4 packing against llama.cpp's dequantizer and exit")
|
||||
args = parser.parse_args(argv)
|
||||
if args.self_test:
|
||||
if args.device == "auto":
|
||||
args.device = "cpu"
|
||||
return args
|
||||
if not args.gguf:
|
||||
parser.error("GGUF file is required")
|
||||
_assign_output(args)
|
||||
_refuse_same_file(args)
|
||||
if not args.modelfile_only:
|
||||
refuse_existing_bundle(Path(args.output), args.overwrite, args.mmproj)
|
||||
else:
|
||||
refuse_existing_bundle(Path(args.output), args.overwrite, args.mmproj, skip_gguf=True)
|
||||
args.exclude_re = re.compile(args.exclude) if args.exclude else None
|
||||
if args.convert_to_quant is None and DEFAULT_CONVERT_TO_QUANT.is_dir():
|
||||
args.convert_to_quant = DEFAULT_CONVERT_TO_QUANT
|
||||
return args
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
args = parse_args(argv)
|
||||
if args.self_test:
|
||||
from hf_to_nvfp4_gguf import self_test
|
||||
self_test(args)
|
||||
return
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format="%(message)s")
|
||||
if args.modelfile_only:
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
log.info("Skipping quantization and GGUF creation; writing Modelfile only")
|
||||
reader = _open_reader(import_gguf(args.llama_cpp), Path(args.gguf))
|
||||
write_simple_modelfile(output, args.overwrite, args.mmproj, _chat_template(reader))
|
||||
return
|
||||
if args.no_quant or args.outtype == "nvfp4":
|
||||
export_copy(args)
|
||||
return
|
||||
args.optimizer = resolve_optimizer(args.optimizer)
|
||||
export_quant(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user