initial commit
This commit is contained in:
zefie committed
commit
db7f28d087
31 files changed
+7462
No files matched your search
@@ -0,0 +1,340 @@
|
||||
"""Output bundle helpers shared by the GGUF conversion scripts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("nvfp4gguf.bundle")
|
||||
|
||||
CONTEXT_HPARAMS = (
|
||||
"max_position_embeddings",
|
||||
"n_ctx",
|
||||
"n_positions",
|
||||
"max_length",
|
||||
"max_sequence_length",
|
||||
"model_max_length",
|
||||
)
|
||||
|
||||
|
||||
def is_model_context_key(name: str) -> bool:
|
||||
"""The model's max context, not RoPE's original context length."""
|
||||
return name.endswith(".context_length") and ".rope." not in name
|
||||
|
||||
|
||||
def require_context_length(length: int | None) -> int | None:
|
||||
if length is None:
|
||||
return None
|
||||
if length <= 0:
|
||||
raise SystemExit("--context-length must be a positive token count")
|
||||
return int(length)
|
||||
|
||||
|
||||
def scalar_context_length(value) -> int | None:
|
||||
if isinstance(value, bool) or isinstance(value, (list, dict)):
|
||||
return None
|
||||
try:
|
||||
length = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if length <= 0:
|
||||
return None
|
||||
return length
|
||||
|
||||
|
||||
def _hf_context_key(hparams: dict) -> tuple[str | None, int | None]:
|
||||
"""Key llama.cpp's find_hparam will see, and its scalar length if it has one."""
|
||||
for key in CONTEXT_HPARAMS:
|
||||
if key in hparams:
|
||||
return key, scalar_context_length(hparams[key])
|
||||
for nested_name in ("text_config", "llm_config", "language_config"):
|
||||
nested = hparams.get(nested_name)
|
||||
if not isinstance(nested, dict):
|
||||
continue
|
||||
for key in CONTEXT_HPARAMS:
|
||||
if key in nested:
|
||||
return key, scalar_context_length(nested[key])
|
||||
return None, None
|
||||
|
||||
|
||||
def apply_hf_context_length(hparams: dict, length: int | None) -> int | None:
|
||||
"""Return the context llama.cpp will write, after an optional override.
|
||||
|
||||
llama.cpp reads the first usual context key on the top-level config. A
|
||||
list or a missing key is not a usable length. The override is written
|
||||
where that lookup will find it.
|
||||
"""
|
||||
requested = require_context_length(length)
|
||||
found_key, found_value = _hf_context_key(hparams)
|
||||
if requested is not None:
|
||||
hparams[found_key or "max_position_embeddings"] = requested
|
||||
log.info("context length %d", requested)
|
||||
return requested
|
||||
if found_value is None:
|
||||
return None
|
||||
if found_key not in hparams:
|
||||
hparams[found_key] = found_value
|
||||
log.info("context length %d from the nested checkpoint config", found_value)
|
||||
else:
|
||||
log.info("context length %d from the checkpoint", found_value)
|
||||
return found_value
|
||||
|
||||
OUTTYPES = ("auto", "f32", "f16", "bf16", "nvfp4")
|
||||
|
||||
|
||||
def outtype(value: str) -> str:
|
||||
return value.lower()
|
||||
|
||||
|
||||
def quote_modelfile_value(text: str) -> str:
|
||||
if text == "" or any(char.isspace() for char in text) or '"' in text:
|
||||
if '"""' in text:
|
||||
raise ValueError("Modelfile value contains a triple quote, which a Modelfile cannot hold")
|
||||
if "\n" in text or '"' in text:
|
||||
return '"""' + text + '"""'
|
||||
return '"' + text + '"'
|
||||
return text
|
||||
|
||||
|
||||
def mmproj_destinations(gguf_path: Path, sources: list[Path] | None) -> list[tuple[Path, Path]]:
|
||||
"""Pair each projector with the path it will have in the output directory."""
|
||||
dest_dir = gguf_path.parent
|
||||
planned: list[tuple[Path, Path]] = []
|
||||
seen: set[str] = set()
|
||||
for source in sources or []:
|
||||
source = Path(source)
|
||||
if not source.is_file():
|
||||
raise SystemExit(f"mmproj file does not exist: {source}")
|
||||
if source.name == gguf_path.name or source.name in seen:
|
||||
raise SystemExit(
|
||||
f"mmproj {source} would be copied as {source.name}, which collides with another file in {dest_dir}."
|
||||
)
|
||||
seen.add(source.name)
|
||||
planned.append((source, dest_dir / source.name))
|
||||
return planned
|
||||
|
||||
|
||||
def place_mmprojs(gguf_path: Path, sources: list[Path] | None, overwrite: bool) -> list[Path]:
|
||||
"""Copy projector GGUFs next to the language model, keeping their file names."""
|
||||
copied: list[Path] = []
|
||||
for source, dest in mmproj_destinations(gguf_path, sources):
|
||||
try:
|
||||
same = dest.exists() and dest.resolve() == source.resolve()
|
||||
except OSError:
|
||||
same = False
|
||||
if same:
|
||||
copied.append(dest)
|
||||
continue
|
||||
if dest.exists() and not overwrite:
|
||||
raise SystemExit(f"{dest} already exists. Pass --overwrite to replace it.")
|
||||
shutil.copy2(source, dest)
|
||||
log.info("Copied vision projector to %s", dest)
|
||||
copied.append(dest)
|
||||
return copied
|
||||
|
||||
|
||||
def refuse_existing_bundle(gguf_path: Path, overwrite: bool, mmprojs: list[Path] | None, *, skip_gguf: bool = False) -> None:
|
||||
"""Stop before a long run when the GGUF, Modelfile, or a copied projector is already there."""
|
||||
if not skip_gguf and gguf_path.exists() and not overwrite:
|
||||
raise SystemExit(f"{gguf_path} already exists. Pass --overwrite to replace it.")
|
||||
modelfile = gguf_path.parent / "Modelfile"
|
||||
if modelfile.exists() and not overwrite:
|
||||
raise SystemExit(f"{modelfile} already exists. Pass --overwrite to replace it.")
|
||||
for source, dest in mmproj_destinations(gguf_path, mmprojs):
|
||||
try:
|
||||
same = dest.exists() and dest.resolve() == source.resolve()
|
||||
except OSError:
|
||||
same = False
|
||||
if dest.exists() and not same and not overwrite:
|
||||
raise SystemExit(f"{dest} already exists. Pass --overwrite to replace it.")
|
||||
|
||||
|
||||
def _stored_value(kv: dict, name: str):
|
||||
item = kv.get(name)
|
||||
if item is None:
|
||||
return None
|
||||
return getattr(item, "value", item)
|
||||
|
||||
|
||||
def _nonempty(value) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
if isinstance(value, str):
|
||||
return bool(value.strip())
|
||||
if isinstance(value, (list, tuple)):
|
||||
return len(value) > 0
|
||||
return True
|
||||
|
||||
|
||||
def warn_missing_gguf_metadata(writer) -> None:
|
||||
"""Warn when the GGUF metadata cannot describe a usable chat model.
|
||||
|
||||
Context length, the chat template, and the tokenizer are the fields that
|
||||
still produce a file but change how it runs. Block count and embedding
|
||||
length are what llama.cpp needs to build the graph.
|
||||
"""
|
||||
kv = writer.kv_data[0] if writer.kv_data else {}
|
||||
arch = writer.arch or "unknown"
|
||||
missing: list[str] = []
|
||||
|
||||
context_keys = [name for name in kv if is_model_context_key(name)]
|
||||
if not context_keys:
|
||||
missing.append(f"context length ({arch}.context_length); pass --context-length to set it")
|
||||
elif all(scalar_context_length(_stored_value(kv, name)) is None for name in context_keys):
|
||||
missing.append("a single context length; pass --context-length to replace the stored value")
|
||||
|
||||
template_names = [
|
||||
name for name in kv
|
||||
if name == "tokenizer.chat_template" or name == "tokenizer.chat_templates" or name.startswith("tokenizer.chat_template.")
|
||||
]
|
||||
if not any(_nonempty(_stored_value(kv, name)) for name in template_names):
|
||||
missing.append("chat template (tokenizer.chat_template)")
|
||||
|
||||
if not _nonempty(_stored_value(kv, "tokenizer.ggml.model")):
|
||||
missing.append("tokenizer model (tokenizer.ggml.model)")
|
||||
if not _nonempty(_stored_value(kv, "tokenizer.ggml.tokens")):
|
||||
missing.append("tokenizer vocabulary (tokenizer.ggml.tokens)")
|
||||
if "tokenizer.ggml.eos_token_id" not in kv:
|
||||
missing.append("end-of-sequence token (tokenizer.ggml.eos_token_id)")
|
||||
if scalar_context_length(_stored_value(kv, f"{arch}.block_count")) is None:
|
||||
missing.append(f"block count ({arch}.block_count)")
|
||||
if scalar_context_length(_stored_value(kv, f"{arch}.embedding_length")) is None:
|
||||
missing.append(f"embedding length ({arch}.embedding_length)")
|
||||
|
||||
if missing:
|
||||
log.warning("GGUF is missing %s.", "; ".join(missing))
|
||||
|
||||
|
||||
# llama.cpp enables its DeepSeek-R1 tool parser only when the template source
|
||||
# contains this exact expression. The sglang DeepSeek-R1 template renders the
|
||||
# same calls but does not contain the expression, so the calls stay in the
|
||||
# visible reply.
|
||||
_DEEPSEEK_R1_TOOL_MARKER = (
|
||||
"{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>'"
|
||||
)
|
||||
# A brace was left between the two branches of that template. Jinja prints it
|
||||
# after every tool call that has no text.
|
||||
_DEEPSEEK_R1_STRAY_BRACE = "'<｜tool▁call▁end｜>'}}\n }\n {%- else %}"
|
||||
_DEEPSEEK_R1_STRAY_FIXED = "'<｜tool▁call▁end｜>'}}\n {%- else %}"
|
||||
|
||||
|
||||
def normalize_chat_template(value):
|
||||
"""Make a DeepSeek-R1 tool template recognizable to llama.cpp's parser."""
|
||||
if isinstance(value, str):
|
||||
return _normalize_one_chat_template(value)
|
||||
if isinstance(value, list):
|
||||
changed = []
|
||||
for item in value:
|
||||
if isinstance(item, str):
|
||||
changed.append(_normalize_one_chat_template(item))
|
||||
elif isinstance(item, dict) and isinstance(item.get("template"), str):
|
||||
item = dict(item)
|
||||
item["template"] = _normalize_one_chat_template(item["template"])
|
||||
changed.append(item)
|
||||
else:
|
||||
changed.append(item)
|
||||
return changed
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_one_chat_template(template: str) -> str:
|
||||
if "tool▁calls▁begin" not in template:
|
||||
return template
|
||||
if _DEEPSEEK_R1_STRAY_BRACE in template:
|
||||
template = template.replace(_DEEPSEEK_R1_STRAY_BRACE, _DEEPSEEK_R1_STRAY_FIXED, 1)
|
||||
if _DEEPSEEK_R1_TOOL_MARKER not in template:
|
||||
template = "{# " + _DEEPSEEK_R1_TOOL_MARKER + " #}\n" + template
|
||||
log.info("Adjusted the DeepSeek-R1 tool template so tool calls are parsed instead of printed.")
|
||||
return template
|
||||
|
||||
|
||||
def flatten_chat_template(value) -> str:
|
||||
"""Join a string or a list of named chat templates into one text blob."""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", "replace")
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, list):
|
||||
parts: list[str] = []
|
||||
for item in value:
|
||||
if isinstance(item, str):
|
||||
parts.append(item)
|
||||
elif isinstance(item, dict) and isinstance(item.get("template"), str):
|
||||
parts.append(item["template"])
|
||||
return "\n".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
# Ollama accepts a tools request only when the chat template both mentions tools
|
||||
# and can render a tool result. A bare "tools" string does not.
|
||||
_TOOL_RESULT_MARKERS = (
|
||||
"tool_response",
|
||||
"tool_results",
|
||||
"role'] == 'tool'",
|
||||
"role'] == \"tool\"",
|
||||
"role\"] == 'tool'",
|
||||
"role\"] == \"tool\"",
|
||||
"message.role == 'tool'",
|
||||
"message.role == \"tool\"",
|
||||
"ipython",
|
||||
)
|
||||
|
||||
|
||||
def chat_template_has_tools(template: str | None) -> bool:
|
||||
"""True when the template can render a tool call and the tool's result."""
|
||||
if not template:
|
||||
return False
|
||||
if "tools" not in template and "tool_call" not in template:
|
||||
return False
|
||||
if "tool_calls" not in template and "assistant_tool_call" not in template:
|
||||
return False
|
||||
return any(marker in template for marker in _TOOL_RESULT_MARKERS)
|
||||
|
||||
|
||||
def tool_modelfile_lines(template: str | None) -> list[str]:
|
||||
"""Declare tools when the stored chat template can round-trip them.
|
||||
|
||||
A template that cannot render a tool result leaves the capability off.
|
||||
Ollama then rejects a request that includes tools instead of dropping the
|
||||
call on the floor.
|
||||
"""
|
||||
if chat_template_has_tools(template):
|
||||
return [
|
||||
"CAPABILITY tools",
|
||||
"# Tool calls use the chat template stored in the GGUF.",
|
||||
]
|
||||
return ["# This chat template has no tool round-trip, so tools stay off."]
|
||||
|
||||
|
||||
def write_simple_modelfile(
|
||||
gguf_path: Path,
|
||||
overwrite: bool,
|
||||
mmprojs: list[Path] | None,
|
||||
chat_template: str | None = None,
|
||||
) -> Path:
|
||||
"""Write a Modelfile that points at a finished GGUF and any projector copies.
|
||||
|
||||
Context is not set here. A PARAMETER num_ctx would override OLLAMA_CONTEXT_LENGTH.
|
||||
"""
|
||||
modelfile = gguf_path.parent / "Modelfile"
|
||||
if modelfile.exists() and not overwrite:
|
||||
raise SystemExit(f"{modelfile} already exists. Pass --overwrite to replace it.")
|
||||
projectors = place_mmprojs(gguf_path, mmprojs, overwrite)
|
||||
lines = [f"FROM ./{gguf_path.name}"]
|
||||
for projector in projectors:
|
||||
lines.append(f"FROM ./{projector.name}")
|
||||
lines.append("")
|
||||
lines.append("# The chat template is the one stored in the GGUF.")
|
||||
lines.extend(tool_modelfile_lines(chat_template))
|
||||
if chat_template_has_tools(chat_template):
|
||||
log.info("Chat template supports tools.")
|
||||
elif chat_template:
|
||||
log.info("Chat template has no tool round-trip. Tool requests stay rejected.")
|
||||
lines.append("# Context length comes from OLLAMA_CONTEXT_LENGTH.")
|
||||
lines.append("")
|
||||
modelfile.write_text("\n".join(lines), encoding="utf-8")
|
||||
log.info("Wrote %s", modelfile)
|
||||
return modelfile
|
||||
Reference in new issue
Block a user