115 lines
5.4 KiB
Python
115 lines
5.4 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import torch
|
|
|
|
from .base import MmprojModel, ModelBase, gguf
|
|
from .qwen import Qwen2Model
|
|
|
|
|
|
@ModelBase.example("sensenova/SenseNova-U1.5-8B-MoT")
|
|
@ModelBase.register("NEOChatModel")
|
|
class SenseNovaU1Model(Qwen2Model):
|
|
model_arch = gguf.MODEL_ARCH.SENSENOVA_U1
|
|
|
|
@classmethod
|
|
def filter_tensors(cls, item):
|
|
name, _ = item
|
|
if not name.startswith("language_model.") or "_mot_gen" in name:
|
|
return None
|
|
return super().filter_tensors(item)
|
|
|
|
def set_gguf_parameters(self):
|
|
if int(self.hparams.get("num_experts") or 0) > 1:
|
|
raise ValueError("SenseNova U1 conversion currently supports the dense understanding branch")
|
|
if self.rope_parameters.get("rope_type", "default") not in ("default", None):
|
|
raise ValueError("SenseNova U1 scaled RoPE is not supported")
|
|
if self.hparams.get("attention_bias", False):
|
|
raise ValueError("SenseNova U1 attention bias is not supported")
|
|
if self.hparams.get("use_sliding_window", False):
|
|
raise ValueError("SenseNova U1 sliding attention is not supported")
|
|
super().set_gguf_parameters()
|
|
self.gguf_writer.add_rope_freq_base_spatial(self.hparams["rope_theta_hw"])
|
|
|
|
def get_tensors(self):
|
|
for name, gen in self.model_tensors.items():
|
|
if name.endswith((".q_norm_hw.weight", ".k_norm_hw.weight")):
|
|
continue
|
|
data = gen()
|
|
if name.endswith((".q_norm.weight", ".k_norm.weight")):
|
|
spatial = self.model_tensors[name.replace("_norm.weight", "_norm_hw.weight")]()
|
|
# Keep one weight vector; the graph normalizes its two halves independently.
|
|
data = torch.cat((data, spatial), dim=0)
|
|
yield name, data
|
|
|
|
|
|
@ModelBase.register("NEOVisionModel")
|
|
@ModelBase.example("sensenova/SenseNova-U1.5-8B-MoT")
|
|
class SenseNovaU1VisionModel(MmprojModel):
|
|
def __init__(self, dir_model: Path, *args, **kwargs):
|
|
hparams = kwargs.pop("hparams", None)
|
|
if hparams is None:
|
|
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
|
else:
|
|
hparams = dict(hparams)
|
|
|
|
vision_config = dict(hparams["vision_config"])
|
|
# NEOVisionModel has no transformer blocks
|
|
# these defaults satisfy the generic mmproj metadata contract without replacing checkpoint values
|
|
vision_config.setdefault("image_size", 0)
|
|
vision_config.setdefault("intermediate_size", 0)
|
|
vision_config.setdefault("num_hidden_layers", 0)
|
|
vision_config.setdefault("num_attention_heads", 1)
|
|
hparams["vision_config"] = vision_config
|
|
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
|
|
|
|
self.preprocessor_config.setdefault("image_mean", [0.485, 0.456, 0.406])
|
|
self.preprocessor_config.setdefault("image_std", [0.229, 0.224, 0.225])
|
|
|
|
@classmethod
|
|
def filter_tensors(cls, item):
|
|
name, gen = item
|
|
if not name.startswith("vision_model."):
|
|
return None
|
|
return super().filter_tensors((name, gen))
|
|
|
|
def set_gguf_parameters(self):
|
|
super().set_gguf_parameters()
|
|
self.gguf_writer.add_clip_projector_type("sensenova_u1")
|
|
vision_config = self.hparams["vision_config"]
|
|
# These defaults match the known U1.5 checkpoint when older configs omit fields.
|
|
hidden_size = int(vision_config.get("hidden_size", 1024))
|
|
min_pixels = int(vision_config.get("min_pixels", 65536))
|
|
max_pixels = int(vision_config.get("max_pixels", 16777216))
|
|
downsample_ratio = float(vision_config.get("downsample_ratio", 0.5))
|
|
if downsample_ratio <= 0:
|
|
raise ValueError(f"SenseNova U1 downsample_ratio must be positive, got {downsample_ratio}")
|
|
merge_size = round(1.0 / downsample_ratio)
|
|
if abs((1.0 / downsample_ratio) - merge_size) > 1e-6:
|
|
raise ValueError(f"SenseNova U1 downsample_ratio must have an integer reciprocal, got {downsample_ratio}")
|
|
norm_eps = float(vision_config.get("layer_norm_eps", vision_config.get("rms_norm_eps", 1e-6)))
|
|
self.gguf_writer.add_vision_head_dim(hidden_size)
|
|
self.gguf_writer.add_vision_attention_layernorm_eps(norm_eps)
|
|
self.gguf_writer.add_vision_min_pixels(min_pixels)
|
|
self.gguf_writer.add_vision_max_pixels(max_pixels)
|
|
self.gguf_writer.add_vision_spatial_merge_size(merge_size)
|
|
|
|
def modify_tensors(self, data_torch, name, bid):
|
|
tensor_names = {
|
|
"vision_model.embeddings.patch_embedding.weight": "v.patch_embd.weight",
|
|
"vision_model.embeddings.patch_embedding.bias": "v.patch_embd.bias",
|
|
"vision_model.embeddings.dense_embedding.weight": self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0),
|
|
"vision_model.embeddings.dense_embedding.bias": self.format_tensor_name(
|
|
gguf.MODEL_TENSOR.V_MMPROJ, 0, suffix=".bias"
|
|
),
|
|
}
|
|
if name not in tensor_names:
|
|
raise ValueError(f"Unexpected SenseNova U1 vision tensor: {name!r}")
|
|
return [(tensor_names[name], data_torch)]
|
|
|
|
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
|
if new_name == self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, 0):
|
|
return gguf.GGMLQuantizationType.F16 if self.ftype == gguf.LlamaFileType.MOSTLY_F16 else gguf.GGMLQuantizationType.F32
|
|
return super().tensor_force_quant(name, new_name, bid, n_dims)
|