include llama.cpp since there were some edits
This commit is contained in:
zefie committed
1 parent
db7f28d087
commit
8c66990853
3918 files changed
+1640170
-1
No files matched your search
@@ -0,0 +1,178 @@
|
||||
# mtmd
|
||||
|
||||
set(MTMD_VIDEO_HELP "enable video support in mtmd (requires ffmpeg binary in PATH)")
|
||||
|
||||
set(MTMD_VIDEO ON CACHE BOOL "${MTMD_VIDEO_HELP}")
|
||||
# TODO: add MTMD_VIDEO_METHOD in the future to select between ffmpeg and other backends
|
||||
|
||||
if (MTMD_VIDEO AND NOT LLAMA_SUBPROCESS)
|
||||
message(STATUS "Disabling MTMD_VIDEO because LLAMA_SUBPROCESS is OFF")
|
||||
set(MTMD_VIDEO OFF CACHE BOOL "${MTMD_VIDEO_HELP}" FORCE)
|
||||
endif()
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
add_library(mtmd
|
||||
mtmd.cpp
|
||||
mtmd-audio.cpp
|
||||
mtmd-image.cpp
|
||||
mtmd.h
|
||||
mtmd-internal.h
|
||||
mtmd-helper.cpp
|
||||
mtmd-helper-gen.cpp
|
||||
mtmd-helper-common.h
|
||||
mtmd-helper.h
|
||||
clip.cpp
|
||||
clip.h
|
||||
clip-impl.h
|
||||
clip-model.h
|
||||
clip-graph.h
|
||||
models/models.h
|
||||
models/cogvlm.cpp
|
||||
models/conformer.cpp
|
||||
models/deepseek4v.cpp
|
||||
models/dots3note.cpp
|
||||
models/dotsocr.cpp
|
||||
models/exaone4_5.cpp
|
||||
models/gemma4a.cpp
|
||||
models/gemma4v.cpp
|
||||
models/gemma4ua.cpp
|
||||
models/gemma4uv.cpp
|
||||
models/glm4v.cpp
|
||||
models/granite-speech.cpp
|
||||
models/granite4-vision.cpp
|
||||
models/hunyuanvl.cpp
|
||||
models/internvl.cpp
|
||||
models/kimivl.cpp
|
||||
models/kimik25.cpp
|
||||
models/nemotron-v2-vl.cpp
|
||||
models/nemotron-omni.cpp
|
||||
models/muse-glimmer.cpp
|
||||
models/muse-spark-audio.cpp
|
||||
models/llama4.cpp
|
||||
models/llava.cpp
|
||||
models/minicpmv.cpp
|
||||
models/paddleocr.cpp
|
||||
models/pixtral.cpp
|
||||
models/qwen2vl.cpp
|
||||
models/sensenova-u1.cpp
|
||||
models/minimax-m3.cpp
|
||||
models/qwen3vl.cpp
|
||||
models/lfm3vl.cpp
|
||||
models/ling3vl.cpp
|
||||
models/mimovl.cpp
|
||||
models/qwen3a.cpp
|
||||
models/mimo-audio.cpp
|
||||
models/qwen3tts-spkenc.cpp
|
||||
models/qwen3tts-gen.cpp
|
||||
models/pockettts-seanet.cpp
|
||||
models/pockettts-spkenc.cpp
|
||||
models/pockettts-gen.cpp
|
||||
models/soprano.cpp
|
||||
models/step3vl.cpp
|
||||
models/siglip.cpp
|
||||
models/whisper-enc.cpp
|
||||
models/voxtral-rt.cpp
|
||||
models/deepseekocr.cpp
|
||||
models/deepseekocr2.cpp
|
||||
models/mobilenetv5.cpp
|
||||
models/youtuvl.cpp
|
||||
models/yasa2.cpp
|
||||
models/parakeet.cpp
|
||||
)
|
||||
|
||||
set_target_properties(mtmd PROPERTIES
|
||||
VERSION ${LLAMA_VERSION_BASE}
|
||||
SOVERSION ${LLAMA_VERSION_MAJOR}
|
||||
MACHO_CURRENT_VERSION 0 # keep macOS linker from seeing oversized version number
|
||||
)
|
||||
|
||||
target_link_libraries (mtmd PUBLIC ggml llama)
|
||||
target_link_libraries (mtmd PRIVATE Threads::Threads vendor::hash vendor::miniaudio vendor::stb vendor::sheredom)
|
||||
target_include_directories(mtmd PUBLIC .)
|
||||
target_compile_features (mtmd PRIVATE cxx_std_17)
|
||||
|
||||
if (MTMD_VIDEO)
|
||||
target_compile_definitions(mtmd PRIVATE MTMD_VIDEO)
|
||||
endif()
|
||||
|
||||
if (BUILD_SHARED_LIBS)
|
||||
set_target_properties (mtmd PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||
target_compile_definitions(mtmd PRIVATE LLAMA_BUILD)
|
||||
target_compile_definitions(mtmd PUBLIC LLAMA_SHARED)
|
||||
|
||||
# export all symbols so that internal components can be tested by test-mtmd-impl
|
||||
set_target_properties (mtmd PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
|
||||
endif()
|
||||
|
||||
set(MTMD_PUBLIC_HEADERS
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/mtmd.h
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/mtmd-helper.h
|
||||
)
|
||||
|
||||
set_target_properties(mtmd
|
||||
PROPERTIES
|
||||
PUBLIC_HEADER "${MTMD_PUBLIC_HEADERS}")
|
||||
|
||||
set_target_properties(mtmd
|
||||
PROPERTIES
|
||||
PRIVATE_HEADER debug/mtmd-debug.h)
|
||||
|
||||
install(TARGETS mtmd LIBRARY PUBLIC_HEADER)
|
||||
|
||||
if (NOT MSVC)
|
||||
# for stb_image.h and miniaudio.h
|
||||
target_compile_options(mtmd PRIVATE -Wno-cast-qual)
|
||||
endif()
|
||||
|
||||
if (ANDROID)
|
||||
# miniaudio.h defines ma_android_sdk_version() without a prior prototype
|
||||
target_compile_options(mtmd PRIVATE -Wno-missing-prototypes)
|
||||
endif()
|
||||
|
||||
# MSVC defines _M_X64 for ARM64EC, so miniaudio.h takes its x64 path.
|
||||
# ARM64EC does not support AVX types.
|
||||
string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" MTMD_GENERATOR_PLATFORM_LWR)
|
||||
if (MSVC AND MTMD_GENERATOR_PLATFORM_LWR STREQUAL "arm64ec")
|
||||
target_compile_definitions(mtmd PRIVATE MA_NO_AVX2)
|
||||
endif()
|
||||
|
||||
if (TARGET BUILD_INFO)
|
||||
add_dependencies(mtmd BUILD_INFO)
|
||||
add_dependencies(mtmd-helper BUILD_INFO)
|
||||
endif()
|
||||
|
||||
# if mtmd is linked against llama-common, we throw an error
|
||||
if (TARGET mtmd)
|
||||
get_target_property(libs mtmd LINK_LIBRARIES)
|
||||
if (libs AND "llama-common" IN_LIST libs)
|
||||
message(FATAL_ERROR "mtmd is designed to be a public library.\n"
|
||||
"It must not link against llama-common")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Gate CLI binaries on LLAMA_BUILD_TOOLS so that standalone library-only
|
||||
# builds (LLAMA_BUILD_MTMD=ON with LLAMA_BUILD_TOOLS=OFF — e.g. Apple
|
||||
# XCFramework packaging) skip the executables entirely. LLAMA_BUILD_COMMON
|
||||
# defaults to ON in standalone builds, so we cannot rely on it for gating.
|
||||
if (LLAMA_BUILD_TOOLS)
|
||||
add_executable(llama-llava-cli deprecation-warning.cpp)
|
||||
add_executable(llama-gemma3-cli deprecation-warning.cpp)
|
||||
add_executable(llama-minicpmv-cli deprecation-warning.cpp)
|
||||
add_executable(llama-qwen2vl-cli deprecation-warning.cpp)
|
||||
|
||||
set(TARGET llama-mtmd-cli)
|
||||
add_executable (${TARGET} mtmd-cli.cpp)
|
||||
set_target_properties (${TARGET} PROPERTIES OUTPUT_NAME llama-mtmd-cli)
|
||||
if(LLAMA_TOOLS_INSTALL)
|
||||
install(TARGETS ${TARGET} RUNTIME)
|
||||
endif()
|
||||
target_link_libraries (${TARGET} PRIVATE llama-common mtmd Threads::Threads)
|
||||
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
||||
|
||||
# mtmd-debug tool
|
||||
add_executable(llama-mtmd-debug debug/mtmd-debug.cpp)
|
||||
set_target_properties(llama-mtmd-debug PROPERTIES OUTPUT_NAME llama-mtmd-debug)
|
||||
target_link_libraries(llama-mtmd-debug PRIVATE llama-common mtmd Threads::Threads)
|
||||
target_compile_features(llama-mtmd-debug PRIVATE cxx_std_17)
|
||||
endif()
|
||||
@@ -0,0 +1,93 @@
|
||||
# libmtmd dev guide
|
||||
|
||||
## History
|
||||
|
||||
Please refer to [multimodal.md](../../docs/multimodal.md) for a broader context.
|
||||
|
||||
In short:
|
||||
- `libmtmd` started as a wrapper around `libllava` / `clip.cpp`
|
||||
- Various components that used to be in `clip.cpp` are moved progressively to mtmd. For example, preprocessor is now part of mtmd
|
||||
|
||||
## Terminologies
|
||||
|
||||
- mtmd: **M**ul**T**i**M**o**D**al
|
||||
- bitmap: representing a raw input data, for example: RGB image, PCM audio
|
||||
- tiles / slices: for llava-uhd-style models, the preprocessor breaks a large input into smaller square images called tiles or slices
|
||||
- chunk: a mtmd_input_chunk represents a preprocessed input that can then be passed through `mtmd_encode()`
|
||||
|
||||
## Pipeline
|
||||
|
||||
A typical pipeline of the core libmtmd is as follows:
|
||||
- A bitmap (RGB image or PCM audio) is created
|
||||
- Bitmap and the text prompt is provided to `mtmd_tokenize()` that breaks the input into chunks
|
||||
- Alternatively, `mtmd_tokenize_from_parts()` takes a list of pre-split text/media parts instead of a marker-based prompt
|
||||
- The tokenizer function first expands a "lazy" bitmap if it finds one. Typically, this is used by video, so that one media token corresponds to one input bitmap
|
||||
- For models that support "fused" temporal frames like Qwen-VL, the tokenizer tries to merge pair of consecutive frames into one batch. Only bitmaps marked by `mtmd_bitmap_set_mergeable()` are merged
|
||||
- The preprocessor will then be called, which produces a list of chunks
|
||||
- Depending on the model itself, special tokens will be injected to separate image chunks (i.e. llava-uhd-style models)
|
||||
- Multiple bitmaps may be batched together to form a larger `mtmd_batch()`
|
||||
- Single image or batch is encoded, via `mtmd_encode()` or `mtmd_batch_encode()`
|
||||
- Get the output embeddings
|
||||
|
||||
## Helper
|
||||
|
||||
We provide a set of helper functions via `mtmd_helper` to make using libmtmd easier. The helper provides:
|
||||
- Image, audio and video file decoding (for example, decode raw JPEG into RGB bitmap)
|
||||
- Manage `llama_batch` and calls to `llama_decode`
|
||||
|
||||
## Audio generation support
|
||||
|
||||
Audio generation is added to mtmd in PR [#26254](https://github.com/ggml-org/llama.cpp/pull/26254)
|
||||
|
||||
Currently, we support the 3-stage pipeline below which should cover most TTS models:
|
||||
- Stage 1: Backbone / Semantic Stage: Backbone model accepts text prompt and reference voice as input
|
||||
- Stage 2: Acoustic Detail Generator: A model takes the hidden state from backbone and generate audio details (usually as audio codes or mel-spectrogram)
|
||||
- Stage 3: Waveform Reconstruction: Convert the semantic and acoustic data from previous stages to the final waveform
|
||||
|
||||
For example, Qwen3-TTS:
|
||||
- Reference voice is encoded using ECAPA-TDNN speaker encoder (`speaker_encoder`)
|
||||
- Text prompt and reference voice are processed via a backbone (`talker.model`)
|
||||
- A model converts sampled semantic token and hidden state from stage 2 into a list of 15 acoustic codes (`talker.code_predictor`)
|
||||
- 16 generated codes are converted into waveform (`code2wav`)
|
||||
|
||||
### API design constraints
|
||||
|
||||
Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system is designed to be flexible and reusable by new models.
|
||||
|
||||
`mtmd_gen_audio` is split into 2 main API:
|
||||
- Core API `mtmd.h`: handles main inference. Important: `mtmd_gen_state` holds only the per-sequence data that the mmproj itself writes and reads back (RNG, decoder KV / conv state, output buffers). Data that goes to or comes from the backbone (hidden states, codes, PCM) must stay explicit in `mtmd_gen_inp` / `mtmd_gen_out`; the caller drives the backbone and handles audio frame accumulation.
|
||||
- Helper API `mtmd-helper.h`: provides a model-agnostic stateful API. Usage example can be found in the `tools/tts` directory.
|
||||
|
||||
### Checklist for porting new audio generation models to mtmd
|
||||
|
||||
1. Make sure to consult merged PRs about adding new TTS models, especially reviewer comments
|
||||
- Example: https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr+mtmd+tts+is%3Amerged
|
||||
2. Establish a list of reusable and missing components from the current mtmd implementation.
|
||||
3. For GGUF conversion:
|
||||
- Backbone model should be converted to a normal text model (loadable via `libllama`)
|
||||
- If model used hard-coded embedding row ID, append them to token embeddings and assign token name for them (see `qwen3tts.py`)
|
||||
- If model have a specific output logits head for audio codes (usually semantic code), keep the head as-is and pad the logits at inference time (see `src/models/qwen3vl.cpp`)
|
||||
- Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `clip_context` if necessary)
|
||||
- Note: it should use `ggml_build_forward_select` to select graphs if multiple graphs living in the same context
|
||||
- Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams to `clip.cpp` as they won't likely to change.
|
||||
- For tensor naming:
|
||||
- Prefixed with `a.*` for tensors used by speaker encoder pipeline
|
||||
- Prefixed with `a.gen.*` for generation stages (code / mel-spectrogram / PCM generation)
|
||||
- For GGUF metadata:
|
||||
- Reuse as many existing keys as possible
|
||||
- In most cases, you can hard-code model configs in the model graph class, or in `clip_hparams`
|
||||
- If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary
|
||||
- Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them
|
||||
4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this:
|
||||
- 10-20% changes is to add new backbone (text) model and conversion
|
||||
- 60% changes inside `mtmd-helper-gen.cpp`
|
||||
- 10% changes inside `libmtmd` and `clip.cpp` systems
|
||||
- The rest downstream code (CLI, server) should have no changes at all
|
||||
5. Update usage documentation in `tools/tts/README.md`
|
||||
|
||||
IMPORTANT: If your model needs changes that don't fit the existing infrastructure, **open an issue first for discussion**.
|
||||
|
||||
No-go checklist (these will get the PR rejected and require discussion before proceeding):
|
||||
- Violating the API design constraints stated above
|
||||
- Adding a new model-specific binary: the API and binary surface must stay model-agnostic
|
||||
- Adding new flags to conversion script: this script must stay model-agnostic
|
||||
@@ -0,0 +1,67 @@
|
||||
# Multimodal Support in llama.cpp
|
||||
|
||||
This directory provides multimodal capabilities for `llama.cpp`. Initially intended as a showcase for running LLaVA models, its scope has expanded significantly over time to include various other vision-capable models. As a result, LLaVA is no longer the only multimodal architecture supported.
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
> Multimodal support can be viewed as a sub-project within `llama.cpp`. It is under **very heavy development**, and **breaking changes are expected**.
|
||||
|
||||
The naming and structure related to multimodal support have evolved, which might cause some confusion. Here's a brief timeline to clarify:
|
||||
|
||||
- [#3436](https://github.com/ggml-org/llama.cpp/pull/3436): Initial support for LLaVA 1.5 was added, introducing `llava.cpp` and `clip.cpp`. The `llava-cli` binary was created for model interaction.
|
||||
- [#4954](https://github.com/ggml-org/llama.cpp/pull/4954): Support for MobileVLM was added, becoming the second vision model supported. This built upon the existing `llava.cpp`, `clip.cpp`, and `llava-cli` infrastructure.
|
||||
- **Expansion & Fragmentation:** Many new models were subsequently added (e.g., [#7599](https://github.com/ggml-org/llama.cpp/pull/7599), [#10361](https://github.com/ggml-org/llama.cpp/pull/10361), [#12344](https://github.com/ggml-org/llama.cpp/pull/12344), and others). However, `llava-cli` lacked support for the increasingly complex chat templates required by these models. This led to the creation of model-specific binaries like `qwen2vl-cli`, `minicpmv-cli`, and `gemma3-cli`. While functional, this proliferation of command-line tools became confusing for users.
|
||||
- [#12849](https://github.com/ggml-org/llama.cpp/pull/12849): `libmtmd` was introduced as a replacement for `llava.cpp`. Its goals include providing a single, unified command-line interface, improving the user/developer experience (UX/DX), and supporting both audio and image inputs.
|
||||
- [#13012](https://github.com/ggml-org/llama.cpp/pull/13012): `mtmd-cli` was added, consolidating the various model-specific CLIs into a single tool powered by `libmtmd`.
|
||||
|
||||
## Pre-quantized models
|
||||
|
||||
See the list of pre-quantized model [here](../../docs/multimodal.md)
|
||||
|
||||
## How it works and what is `mmproj`?
|
||||
|
||||
Multimodal support in `llama.cpp` works by encoding images into embeddings using a separate model component, and then feeding these embeddings into the language model.
|
||||
|
||||
This approach keeps the multimodal components distinct from the core `libllama` library. Separating these allows for faster, independent development cycles. While many modern vision models are based on Vision Transformers (ViTs), their specific pre-processing and projection steps can vary significantly. Integrating this diverse complexity directly into `libllama` is currently challenging.
|
||||
|
||||
Consequently, running a multimodal model typically requires two GGUF files:
|
||||
1. The standard language model file.
|
||||
2. A corresponding **multimodal projector (`mmproj`)** file, which handles the image encoding and projection.
|
||||
|
||||
## What is `libmtmd`?
|
||||
|
||||
As outlined in the history, `libmtmd` is the modern library designed to replace the original `llava.cpp` implementation for handling multimodal inputs.
|
||||
|
||||
Built upon `clip.cpp` (similar to `llava.cpp`), `libmtmd` offers several advantages:
|
||||
- **Unified Interface:** Aims to consolidate interaction for various multimodal models.
|
||||
- **Improved UX/DX:** Features a more intuitive API, inspired by the `Processor` class in the Hugging Face `transformers` library.
|
||||
- **Flexibility:** Designed to support multiple input types (text, audio, images) while respecting the wide variety of chat templates used by different models.
|
||||
|
||||
## How to obtain `mmproj`
|
||||
|
||||
Multimodal projector (`mmproj`) files are specific to each model architecture.
|
||||
|
||||
For the following models, you can use `convert_hf_to_gguf.py` with `--mmproj` flag to get the `mmproj` file:
|
||||
- [Gemma 3](https://huggingface.co/collections/google/gemma-3-release-67c6c6f89c4f76621268bb6d) ; See the guide [here](../../docs/multimodal/gemma3.md) - Note: 1B variant does not have vision support
|
||||
- SmolVLM (from [HuggingFaceTB](https://huggingface.co/HuggingFaceTB))
|
||||
- SmolVLM2 (from [HuggingFaceTB](https://huggingface.co/HuggingFaceTB))
|
||||
- [Pixtral 12B](https://huggingface.co/mistral-community/pixtral-12b) - only works with `transformers`-compatible checkpoint
|
||||
- Qwen 2 VL and Qwen 2.5 VL (from [Qwen](https://huggingface.co/Qwen))
|
||||
- [Mistral Small 3.1 24B](https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503)
|
||||
- InternVL 2.5 and InternVL 3 from [OpenGVLab](https://huggingface.co/OpenGVLab) (note: we don't support conversion of `InternVL3-*-hf` model, only non-HF version is supported ; `InternLM2Model` **text** model is not supported)
|
||||
- [MiniCPM-V 4.6](https://huggingface.co/openbmb/MiniCPM-V-4_6) ; See the guide [here](../../docs/multimodal/minicpmv4.6.md) - requires the standard `transformers` v5.7.0+ checkpoint
|
||||
|
||||
For older models, please refer to the relevant guide for instructions on how to obtain or create them:
|
||||
|
||||
NOTE: conversion scripts are located under `tools/mtmd/legacy-models`
|
||||
|
||||
- [LLaVA](../../docs/multimodal/llava.md)
|
||||
- [MobileVLM](../../docs/multimodal/MobileVLM.md)
|
||||
- [GLM-Edge](../../docs/multimodal/glmedge.md)
|
||||
- [MiniCPM-V 2.5](../../docs/multimodal/minicpmv2.5.md)
|
||||
- [MiniCPM-V 2.6](../../docs/multimodal/minicpmv2.6.md)
|
||||
- [MiniCPM-o 2.6](../../docs/multimodal/minicpmo2.6.md)
|
||||
- [MiniCPM-V 4.0](../../docs/multimodal/minicpmv4.0.md)
|
||||
- [MiniCPM-o 4.0](../../docs/multimodal/minicpmo4.0.md)
|
||||
- [MiniCPM-V 4.5](../../docs/multimodal/minicpmv4.5.md)
|
||||
- [IBM Granite Vision](../../docs/multimodal/granitevision.md)
|
||||
@@ -0,0 +1,176 @@
|
||||
#pragma once
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpp.h"
|
||||
#include "clip.h"
|
||||
#include "clip-impl.h"
|
||||
#include "clip-model.h"
|
||||
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
|
||||
#define DEFAULT_INTERPOLATION_MODE (GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS)
|
||||
|
||||
struct build_vit_opts {
|
||||
ggml_tensor * attn_mask = nullptr;
|
||||
// TODO @ngxson : merge attn_mask and attn_mask_layers into one call
|
||||
std::vector<ggml_tensor *> attn_mask_layers; // one per layer
|
||||
|
||||
// hook at layer output embeddings
|
||||
std::function<void(ggml_tensor * cur, int il)> callback_layer_out = nullptr;
|
||||
|
||||
// whether to skip the automatic post-layernorm (model.post_ln_w) applied at the end
|
||||
bool skip_post_ln = false;
|
||||
};
|
||||
|
||||
struct clip_graph {
|
||||
const clip_model & model;
|
||||
const clip_hparams & hparams;
|
||||
projector_type proj_type;
|
||||
|
||||
const clip_image_f32 & img; // for backward compat
|
||||
const clip_image_f32_batch * img_batch = nullptr;
|
||||
|
||||
const int patch_size;
|
||||
const int n_patches_x;
|
||||
const int n_patches_y;
|
||||
const int n_patches;
|
||||
const int n_embd;
|
||||
const int n_head;
|
||||
const int n_head_kv;
|
||||
const int d_head;
|
||||
const int n_layer;
|
||||
const int n_mmproj_embd;
|
||||
const float eps;
|
||||
float kq_scale; // TODO: maybe move this to hparams
|
||||
const clip_flash_attn_type flash_attn_type;
|
||||
|
||||
// TODO [QWEN_VIDEO]: improve this in the future
|
||||
int n_batch = 1;
|
||||
|
||||
ggml_context_ptr ctx0_ptr;
|
||||
ggml_context * ctx0;
|
||||
ggml_cgraph * gf;
|
||||
|
||||
clip_graph(clip_ctx * ctx, const clip_image_f32 & img);
|
||||
|
||||
// build sub-graph, reuse buf from parent
|
||||
clip_graph(const clip_graph & parent);
|
||||
|
||||
virtual ~clip_graph() = default;
|
||||
virtual ggml_cgraph * build() = 0;
|
||||
|
||||
// wrapper around ggml_mul_mat, allow hooking (e.g. LoRA, clamping) depending on the model
|
||||
// tensor w should be the weight matrix, and tensor x should be the input
|
||||
virtual ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const;
|
||||
// TODO: build_mm(w, b, x) to support bias
|
||||
|
||||
virtual bool support_batch() const {
|
||||
return false;
|
||||
}
|
||||
|
||||
//
|
||||
// utility functions
|
||||
//
|
||||
void cb(ggml_tensor * cur0, const char * name, int il) const;
|
||||
|
||||
const clip_image_f32 & get_img(size_t idx) const {
|
||||
GGML_ASSERT(img_batch);
|
||||
GGML_ASSERT(idx < img_batch->entries.size());
|
||||
return img_batch->entries[idx];
|
||||
}
|
||||
|
||||
// siglip2 naflex
|
||||
ggml_tensor * resize_position_embeddings(uint32_t interpolation_mode = DEFAULT_INTERPOLATION_MODE);
|
||||
|
||||
// build vision transformer (ViT) cgraph
|
||||
// this function should cover most of the models
|
||||
// if your model has specific features, you should probably duplicate this function
|
||||
ggml_tensor * build_vit(
|
||||
ggml_tensor * inp,
|
||||
int64_t n_pos,
|
||||
norm_type norm_t,
|
||||
ffn_op_type ffn_t,
|
||||
ggml_tensor * learned_pos_embd,
|
||||
std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos,
|
||||
const build_vit_opts & opts = {});
|
||||
|
||||
// build the input after conv2d (inp_raw --> patches)
|
||||
// returns tensor with shape [n_embd, n_patches]
|
||||
ggml_tensor * build_inp();
|
||||
|
||||
ggml_tensor * build_inp_raw(int channels = 3);
|
||||
|
||||
// qwen-vl style temporal merge, handles n_batch == 1 (image) or n_batch == 2 (2 video frames)
|
||||
// requires patch_embeddings_0 and patch_embeddings_1; returns tensor with shape [w, h, n_embd, 1]
|
||||
ggml_tensor * build_inp_temporal_merge();
|
||||
|
||||
// f16 if flash attn is enabled, set it with set_input_attn_mask()
|
||||
// idx is only needed when the graph has more than one mask
|
||||
ggml_tensor * build_inp_attn_mask(int64_t n_kv, int64_t n_q, int idx = 0);
|
||||
|
||||
ggml_tensor * build_norm(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * mw,
|
||||
ggml_tensor * mb,
|
||||
norm_type type,
|
||||
float norm_eps,
|
||||
int il) const;
|
||||
|
||||
ggml_tensor * build_ffn(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * up,
|
||||
ggml_tensor * up_b,
|
||||
ggml_tensor * gate,
|
||||
ggml_tensor * gate_b,
|
||||
ggml_tensor * down,
|
||||
ggml_tensor * down_b,
|
||||
ffn_op_type type_op,
|
||||
int il) const;
|
||||
|
||||
ggml_tensor * build_moe_ffn(
|
||||
ggml_tensor * cur,
|
||||
const clip_layer & layer,
|
||||
ffn_op_type type_op,
|
||||
int il) const;
|
||||
|
||||
ggml_tensor * build_attn(
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
ggml_tensor * q_cur,
|
||||
ggml_tensor * k_cur,
|
||||
ggml_tensor * v_cur,
|
||||
ggml_tensor * kq_mask,
|
||||
float kq_scale,
|
||||
int il,
|
||||
ggml_tensor * sinks = nullptr) const;
|
||||
|
||||
// implementation of the 2D RoPE using two ggml_rope_ext calls
|
||||
//
|
||||
// unlike GGML_ROPE_TYPE_VISION which forces NEOX ordering, this rotates adjacent pairs (normal ordering)
|
||||
//
|
||||
// example:
|
||||
// given a single head with size = 8 --> [00000000]
|
||||
// dims [0, 4) rotate with pos_a, dims [4, 8) rotate with pos_b --> [aaaabbbb]
|
||||
// interleave_freq = false --> both halves use the same inv_freq set (like GGML_ROPE_TYPE_VISION)
|
||||
// interleave_freq = true --> first half uses even inv_freq, second half uses odd inv_freq (used by pixtral)
|
||||
ggml_tensor * build_rope_2d(
|
||||
ggml_context * ctx0,
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * pos_a, // first half
|
||||
ggml_tensor * pos_b, // second half
|
||||
const float freq_base,
|
||||
const bool interleave_freq
|
||||
);
|
||||
|
||||
// aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL)
|
||||
// support dynamic resolution
|
||||
ggml_tensor * build_patch_merge_permute(ggml_tensor * cur, int scale_factor);
|
||||
|
||||
// Generic function to stack frames for audio processing
|
||||
// Abstracts out the StackAudioFrames logic used by ultravox
|
||||
ggml_tensor * build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed);
|
||||
|
||||
// append the separators of img.suffix_type after the image tokens
|
||||
ggml_tensor * build_suffix(ggml_tensor * cur);
|
||||
};
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,965 @@
|
||||
#pragma once
|
||||
|
||||
#include "ggml.h"
|
||||
#include "clip.h"
|
||||
#include "clip-impl.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <vector>
|
||||
#include <unordered_set>
|
||||
#include <cstdint>
|
||||
#include <cmath>
|
||||
|
||||
enum ffn_op_type {
|
||||
FFN_GELU,
|
||||
FFN_GELU_ERF,
|
||||
FFN_SILU,
|
||||
FFN_GELU_QUICK,
|
||||
FFN_RELU_SQR,
|
||||
};
|
||||
|
||||
enum norm_type {
|
||||
NORM_TYPE_NORMAL,
|
||||
NORM_TYPE_RMS,
|
||||
};
|
||||
|
||||
enum patch_merge_type {
|
||||
PATCH_MERGE_FLAT,
|
||||
PATCH_MERGE_SPATIAL_UNPAD,
|
||||
};
|
||||
|
||||
// the low bits select the algo, Pillow-compatible (matching PIL.Image.resize output)
|
||||
// RESIZE_ALGO_FLAG_TORCH matches torchvision's antialiased resize of uint8 images on the CPU (int16 weights), bilinear and bicubic only
|
||||
enum resize_algo {
|
||||
RESIZE_ALGO_BILINEAR,
|
||||
RESIZE_ALGO_BICUBIC,
|
||||
RESIZE_ALGO_LANCZOS,
|
||||
|
||||
RESIZE_ALGO_FLAG_TORCH = 1 << 8,
|
||||
};
|
||||
|
||||
// Padding style for img_tool::resize
|
||||
// PAD_NONE - no padding; direct resize to target dimensions
|
||||
// PAD_CEIL - aspect-preserving pad (default)
|
||||
// PAD_NEAREST - aspect-preserving pad with nearest-integer rounding (Pillow byte-parity)
|
||||
enum pad_style {
|
||||
PAD_NONE,
|
||||
PAD_CEIL,
|
||||
PAD_NEAREST,
|
||||
};
|
||||
|
||||
struct clip_hparams {
|
||||
int32_t image_size = 0;
|
||||
int32_t patch_size = 0;
|
||||
int32_t n_embd = 0;
|
||||
int32_t n_ff = 0;
|
||||
int32_t projection_dim = 0;
|
||||
int32_t n_head = 0;
|
||||
int32_t n_head_kv = 0;
|
||||
// 0 = derive from n_embd; set when qkv width != n_embd
|
||||
int32_t n_embd_head = 0;
|
||||
int32_t n_layer = 0;
|
||||
int32_t n_merge = 1; // number of patch merges **per-side**
|
||||
|
||||
// for preprocessor
|
||||
int32_t image_longest_edge = 0;
|
||||
int32_t image_min_pixels = -1;
|
||||
int32_t image_max_pixels = -1;
|
||||
resize_algo image_resize_algo = RESIZE_ALGO_BICUBIC;
|
||||
pad_style image_resize_pad = PAD_CEIL; // padding style when resizing
|
||||
bool image_round_half_even = false; // smart_resize rounds .5 to even, like Python round() in the HF processor
|
||||
bool image_floor_before_align = false;
|
||||
std::array<uint8_t, 3> image_pad_color = {0, 0, 0};
|
||||
|
||||
// (preprocessor) for llava-uhd style models
|
||||
std::vector<clip_image_size> image_res_candidates;
|
||||
int32_t preproc_min_tiles = 0;
|
||||
int32_t preproc_max_tiles = 0;
|
||||
int32_t max_slice_nums = 9; // llava-uhd slice cap; per-model, carried in the GGUF
|
||||
int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr, lfm2)
|
||||
float preproc_max_pixels_tolerance = 1.0f;
|
||||
resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC;
|
||||
resize_algo image_resize_algo_ov = RESIZE_ALGO_BICUBIC;
|
||||
pad_style image_pad_rf = PAD_CEIL; // padding style for the refined image (e.g. llava-1.6)
|
||||
pad_style image_pad_ov = PAD_NONE; // padding style for the overview image (e.g. llava-1.6)
|
||||
std::array<uint8_t, 3> image_pad_color_rf = {0, 0, 0}; // padding color for refined image
|
||||
std::array<uint8_t, 3> image_pad_color_ov = {0, 0, 0}; // padding color for overview image
|
||||
|
||||
float image_mean[3];
|
||||
float image_std[3];
|
||||
|
||||
// for models using dynamic image size, we need to have a smaller image size to warmup
|
||||
// otherwise, user will get OOM every time they load the model
|
||||
int32_t warmup_image_size = 0;
|
||||
int32_t warmup_audio_size = 3000;
|
||||
|
||||
ffn_op_type ffn_op = FFN_GELU;
|
||||
|
||||
patch_merge_type mm_patch_merge_type = PATCH_MERGE_FLAT;
|
||||
|
||||
float eps = 1e-6;
|
||||
float rope_theta = 0.0;
|
||||
|
||||
float swiglu_clamp = 0.0f;
|
||||
int32_t n_expert_used = 0;
|
||||
std::vector<int32_t> feature_layers;
|
||||
int32_t attn_window_size = 0;
|
||||
int32_t n_wa_pattern = 0;
|
||||
std::unordered_set<int32_t> wa_layer_indexes; // explicit layer indexes that use full attention (for irregular patterns like YoutuVL)
|
||||
std::vector<int32_t> wa_pattern_mode; // mimovl: per-layer window-attention mode
|
||||
|
||||
// deepseek4v: resize solver caps the LLM token count of the aligner grid
|
||||
int32_t dsv4_max_n_token = 0;
|
||||
int32_t dsv4_max_wh_ratio = 0;
|
||||
|
||||
// deepseek-ocr (sam)
|
||||
int32_t sam_n_layer = 0;
|
||||
int32_t sam_n_head = 0;
|
||||
int32_t sam_n_embd = 0;
|
||||
|
||||
// Granite4 Vision
|
||||
std::vector<int32_t> proj_spatial_offsets;
|
||||
int32_t downsample_query_side;
|
||||
int32_t downsample_window_side;
|
||||
|
||||
// Muse Glimmer vision (per-block sparse-window pattern, learned pos-emb, patch-temporal)
|
||||
// NOTE: these perhaps shouldn't have the architecture prefix
|
||||
int32_t muse_glimmer_patch_temporal = 0;
|
||||
int32_t muse_glimmer_sparse_factor = 0;
|
||||
|
||||
// audio
|
||||
int32_t n_mel_bins = 0; // whisper preprocessor
|
||||
int32_t proj_stack_factor = 0; // ultravox
|
||||
int32_t subsampling_factor = 0; // parakeet
|
||||
|
||||
int32_t audio_chunk_size = 0;
|
||||
int32_t audio_conv_kernel_size = 0;
|
||||
int32_t audio_max_pos_emb = 0;
|
||||
int32_t audio_proj_window_size = 0;
|
||||
int32_t audio_proj_downsample_rate = 0;
|
||||
int32_t audio_proj_head_count = 0;
|
||||
|
||||
// audio pyramid (muse-spark): widths, folded samples and blocks per stage
|
||||
std::vector<int32_t> audio_hidden_sizes;
|
||||
std::vector<int32_t> audio_patch_sizes;
|
||||
std::vector<int32_t> audio_depths;
|
||||
|
||||
int32_t audio_samples_per_token() const {
|
||||
int64_t n = 1;
|
||||
for (int32_t p : audio_patch_sizes) {
|
||||
n *= p;
|
||||
GGML_ASSERT(n >= 1 && n <= (1 << 20));
|
||||
}
|
||||
return (int32_t) n;
|
||||
}
|
||||
|
||||
// audio-to-mel preprocessor params
|
||||
int32_t audio_chunk_len = -1; // in seconds
|
||||
int32_t audio_sample_rate = -1;
|
||||
int32_t audio_n_fft = -1;
|
||||
int32_t audio_window_len = -1;
|
||||
int32_t audio_hop_len = -1;
|
||||
|
||||
// parakeet
|
||||
std::vector<float> mel_filters;
|
||||
std::vector<float> window;
|
||||
|
||||
// mimo-audio-tokenizer: residual vector quantizer
|
||||
int32_t rvq_num_quantizers = 0;
|
||||
std::vector<int32_t> rvq_codebook_size; // per-quantizer bin count (ragged, e.g. 1024/1024/256/128x17)
|
||||
|
||||
// threshold for the "out_eos_score" graph output
|
||||
float gen_eos_threshold = 0.0f;
|
||||
|
||||
// name of the weight variant, some pipelines tune themselves on it
|
||||
std::string gen_model_variant;
|
||||
|
||||
// pocket-tts
|
||||
static constexpr int32_t pockettts_max_spk_seconds = 30;
|
||||
int32_t seanet_n_stage = 0;
|
||||
std::vector<int32_t> seanet_ratios; // encoder order (reversed compared to the config)
|
||||
int32_t mimi_downsample = 0; // encoder frame rate / model frame rate
|
||||
int32_t mimi_tfm_context = 0; // attention window of the mimi transformers, in frames
|
||||
int32_t flow_n_step = 1; // lsd_decode steps
|
||||
|
||||
// qwen3tts code2wav
|
||||
int32_t wav_tfm_n_layer = 0;
|
||||
int32_t wav_tfm_n_embd = 0;
|
||||
int32_t wav_tfm_n_ff = 0;
|
||||
int32_t wav_tfm_n_head = 0;
|
||||
int32_t wav_tfm_n_head_kv = 0;
|
||||
float wav_tfm_eps = 1e-5f;
|
||||
float wav_tfm_rope_theta = 10000.0f;
|
||||
int32_t wav_upsample_n_block = 0;
|
||||
int32_t wav_dac_n_block = 0;
|
||||
int32_t wav_dac_n_res = 0;
|
||||
int32_t wav_tfm_swa = 0; // pre_transformer's KV cache size, in frames
|
||||
|
||||
// mimo-v2.5: LLM-side connector (input_local_transformer)
|
||||
int32_t audio_local_n_layer = 0;
|
||||
int32_t audio_local_group_size = 0;
|
||||
|
||||
// legacy
|
||||
bool has_llava_projector = false;
|
||||
int minicpmv_version = 0;
|
||||
int32_t minicpmv_query_num = 0; // MiniCPM-V query number
|
||||
int32_t insert_layer_id = 0; // MiniCPM-V 4.6 ViT merger insertion layer
|
||||
|
||||
// custom value provided by user, can be undefined if not set
|
||||
int32_t custom_image_min_tokens = -1;
|
||||
int32_t custom_image_max_tokens = -1;
|
||||
|
||||
void set_image_resize_algo(resize_algo algo, bool torch = false) {
|
||||
GGML_ASSERT(!(torch && algo == RESIZE_ALGO_LANCZOS) && "torchvision has no lanczos");
|
||||
image_resize_algo = static_cast<resize_algo>(torch ? algo | RESIZE_ALGO_FLAG_TORCH : algo);
|
||||
}
|
||||
|
||||
void set_limit_image_tokens(int n_tokens_min, int n_tokens_max) {
|
||||
const int patch_area = patch_size * patch_size * n_merge * n_merge;
|
||||
image_min_pixels = (custom_image_min_tokens > 0 ? custom_image_min_tokens : n_tokens_min) * patch_area;
|
||||
image_max_pixels = (custom_image_max_tokens > 0 ? custom_image_max_tokens : n_tokens_max) * patch_area;
|
||||
warmup_image_size = static_cast<int>(std::sqrt(image_max_pixels));
|
||||
}
|
||||
|
||||
// used by longest_edge preprocessor (no model-specific value for min/max tokens)
|
||||
void set_limit_image_tokens() {
|
||||
const int patch_area = patch_size * patch_size * n_merge * n_merge;
|
||||
if (custom_image_min_tokens > 0) {
|
||||
image_min_pixels = custom_image_min_tokens * patch_area;
|
||||
}
|
||||
if (custom_image_max_tokens > 0) {
|
||||
image_max_pixels = custom_image_max_tokens * patch_area;
|
||||
}
|
||||
}
|
||||
|
||||
void set_warmup_n_tokens(int n_tokens) {
|
||||
int n_tok_per_side = static_cast<int>(std::sqrt(n_tokens));
|
||||
GGML_ASSERT(n_tok_per_side * n_tok_per_side == n_tokens && "n_tokens must be n*n");
|
||||
// do not warmup with more tokens than the max allowed
|
||||
if (custom_image_max_tokens > 0 && n_tokens > custom_image_max_tokens) {
|
||||
n_tok_per_side = std::max(1, static_cast<int>(std::sqrt(custom_image_max_tokens)));
|
||||
}
|
||||
warmup_image_size = n_tok_per_side * patch_size * n_merge;
|
||||
}
|
||||
// sam vit deepseek-ocr
|
||||
std::vector<int32_t> global_attn_indices() const {
|
||||
return { 2, 5, 8, 11 };
|
||||
}
|
||||
bool is_global_attn(int32_t layer) const {
|
||||
const auto indices = global_attn_indices();
|
||||
|
||||
for (const auto & idx : indices) {
|
||||
if (layer == idx) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool is_feature_layer(int32_t layer) const {
|
||||
return std::find(feature_layers.begin(), feature_layers.end(), layer) != feature_layers.end();
|
||||
}
|
||||
};
|
||||
|
||||
struct clip_layer {
|
||||
// layernorm 1 (or layer input norm, or pre-attention norm)
|
||||
ggml_tensor * ln_1_w = nullptr;
|
||||
ggml_tensor * ln_1_b = nullptr;
|
||||
|
||||
// attention
|
||||
ggml_tensor * k_w = nullptr;
|
||||
ggml_tensor * k_b = nullptr;
|
||||
ggml_tensor * q_w = nullptr;
|
||||
ggml_tensor * q_b = nullptr;
|
||||
ggml_tensor * v_w = nullptr;
|
||||
ggml_tensor * v_b = nullptr;
|
||||
ggml_tensor * qkv_w = nullptr;
|
||||
ggml_tensor * qkv_b = nullptr;
|
||||
|
||||
ggml_tensor * o_w = nullptr;
|
||||
ggml_tensor * o_b = nullptr;
|
||||
|
||||
ggml_tensor * attn_sinks = nullptr;
|
||||
|
||||
ggml_tensor * k_norm = nullptr;
|
||||
ggml_tensor * q_norm = nullptr;
|
||||
|
||||
ggml_tensor * attn_post_norm_w = nullptr;
|
||||
|
||||
ggml_tensor * ff_up_w = nullptr;
|
||||
ggml_tensor * ff_up_b = nullptr;
|
||||
ggml_tensor * ff_gate_w = nullptr;
|
||||
ggml_tensor * ff_gate_b = nullptr;
|
||||
ggml_tensor * ff_down_w = nullptr;
|
||||
ggml_tensor * ff_down_b = nullptr;
|
||||
|
||||
// MoE FFN (dots3note vision pyramid blocks)
|
||||
ggml_tensor * ff_gate_inp_w = nullptr;
|
||||
ggml_tensor * ff_gate_exps_w = nullptr;
|
||||
ggml_tensor * ff_up_exps_w = nullptr;
|
||||
ggml_tensor * ff_down_exps_w = nullptr;
|
||||
ggml_tensor * ff_exp_probs_b = nullptr;
|
||||
|
||||
// layernorm 2 (or pre-FFN norm)
|
||||
ggml_tensor * ln_2_w = nullptr;
|
||||
ggml_tensor * ln_2_b = nullptr;
|
||||
|
||||
ggml_tensor * ff_post_norm_w = nullptr;
|
||||
|
||||
// depthwise causal conv block (muse-spark audio), conv_norm_w is declared below
|
||||
ggml_tensor * conv_w = nullptr;
|
||||
ggml_tensor * conv_b = nullptr;
|
||||
|
||||
// layer scale (no bias)
|
||||
ggml_tensor * ls_1_w = nullptr;
|
||||
ggml_tensor * ls_2_w = nullptr;
|
||||
ggml_tensor * ls_out_w = nullptr; // gemma4
|
||||
|
||||
// qwen3vl deepstack merger
|
||||
ggml_tensor * deepstack_norm_w = nullptr;
|
||||
ggml_tensor * deepstack_norm_b = nullptr;
|
||||
ggml_tensor * deepstack_fc1_w = nullptr;
|
||||
ggml_tensor * deepstack_fc1_b = nullptr;
|
||||
ggml_tensor * deepstack_fc2_w = nullptr;
|
||||
ggml_tensor * deepstack_fc2_b = nullptr;
|
||||
|
||||
// lfm3vl conv layers
|
||||
ggml_tensor * conv2d_in_proj_w = nullptr;
|
||||
ggml_tensor * conv2d_spatial_w = nullptr; // may be nullptr
|
||||
ggml_tensor * conv2d_out_proj_w = nullptr;
|
||||
|
||||
// sam rel_pos
|
||||
ggml_tensor * rel_pos_w = nullptr;
|
||||
ggml_tensor * rel_pos_h = nullptr;
|
||||
// lfm2
|
||||
ggml_tensor * ff_norm_w = nullptr;
|
||||
ggml_tensor * ff_norm_b = nullptr;
|
||||
ggml_tensor * ff_norm_1_w = nullptr;
|
||||
ggml_tensor * ff_norm_1_b = nullptr;
|
||||
ggml_tensor * ff_up_1_w = nullptr;
|
||||
ggml_tensor * ff_up_1_b = nullptr;
|
||||
ggml_tensor * ff_down_1_w = nullptr;
|
||||
ggml_tensor * ff_down_1_b = nullptr;
|
||||
ggml_tensor * pos_bias_u = nullptr;
|
||||
ggml_tensor * pos_bias_v = nullptr;
|
||||
ggml_tensor * norm_conv_w = nullptr;
|
||||
ggml_tensor * norm_conv_b = nullptr;
|
||||
ggml_tensor * linear_pos_w = nullptr;
|
||||
|
||||
ggml_tensor * conv_norm_w = nullptr;
|
||||
ggml_tensor * conv_norm_b = nullptr;
|
||||
ggml_tensor * conv_norm_mean = nullptr; // parakeet
|
||||
ggml_tensor * conv_norm_var = nullptr; // parakeet
|
||||
ggml_tensor * conv_dw_w = nullptr;
|
||||
ggml_tensor * conv_dw_b = nullptr;
|
||||
ggml_tensor * conv_pw1_w = nullptr;
|
||||
ggml_tensor * conv_pw1_b = nullptr;
|
||||
ggml_tensor * conv_pw2_w = nullptr;
|
||||
ggml_tensor * conv_pw2_b = nullptr;
|
||||
|
||||
// gemma4 audio conformer per-layer
|
||||
ggml_tensor * attn_pre_norm_w = nullptr;
|
||||
ggml_tensor * attn_k_rel_w = nullptr;
|
||||
ggml_tensor * per_dim_scale_w = nullptr;
|
||||
ggml_tensor * per_dim_k_scale_w = nullptr;
|
||||
ggml_tensor * ff_post_norm_1_w = nullptr;
|
||||
|
||||
// granite_speech conformer per-layer
|
||||
ggml_tensor * attn_rel_pos_emb = nullptr;
|
||||
|
||||
// granite_speech qformer cross-attention
|
||||
ggml_tensor * cross_attn_q_w = nullptr;
|
||||
ggml_tensor * cross_attn_q_b = nullptr;
|
||||
ggml_tensor * cross_attn_k_w = nullptr;
|
||||
ggml_tensor * cross_attn_k_b = nullptr;
|
||||
ggml_tensor * cross_attn_v_w = nullptr;
|
||||
ggml_tensor * cross_attn_v_b = nullptr;
|
||||
ggml_tensor * cross_attn_o_w = nullptr;
|
||||
ggml_tensor * cross_attn_o_b = nullptr;
|
||||
ggml_tensor * cross_attn_norm_w = nullptr;
|
||||
ggml_tensor * cross_attn_norm_b = nullptr;
|
||||
|
||||
// qwen3tts speaker encoder: SE-Res2Net block, tdnn1/tdnn2 reuse conv_pw1_w/b and conv_pw2_w/b above
|
||||
ggml_tensor * se_conv1_w = nullptr;
|
||||
ggml_tensor * se_conv1_b = nullptr;
|
||||
ggml_tensor * se_conv2_w = nullptr;
|
||||
ggml_tensor * se_conv2_b = nullptr;
|
||||
std::vector<ggml_tensor *> res2_conv_w; // Res2Net hierarchical branches
|
||||
std::vector<ggml_tensor *> res2_conv_b;
|
||||
|
||||
bool has_deepstack() const {
|
||||
return deepstack_fc1_w != nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
// Expanded MobileNetV5 block structure for Gemma3n vision encoder
|
||||
struct mobilenetv5_block {
|
||||
// Stage 0 (Edge Residual)
|
||||
ggml_tensor * s0_conv_exp_w = nullptr;
|
||||
ggml_tensor * s0_bn1_w = nullptr;
|
||||
ggml_tensor * s0_conv_pwl_w = nullptr;
|
||||
ggml_tensor * s0_bn2_w = nullptr;
|
||||
|
||||
// Stage 1+ (Universal Inverted Residual)
|
||||
ggml_tensor * dw_start_w = nullptr;
|
||||
ggml_tensor * dw_start_bn_w = nullptr;
|
||||
|
||||
ggml_tensor * pw_exp_w = nullptr;
|
||||
ggml_tensor * pw_exp_bn_w = nullptr;
|
||||
|
||||
ggml_tensor * dw_mid_w = nullptr;
|
||||
ggml_tensor * dw_mid_bn_w = nullptr;
|
||||
|
||||
ggml_tensor * pw_proj_w = nullptr;
|
||||
ggml_tensor * pw_proj_bn_w = nullptr;
|
||||
|
||||
ggml_tensor * layer_scale_w = nullptr;
|
||||
|
||||
// Attention (MQA) components
|
||||
ggml_tensor * attn_q_w = nullptr;
|
||||
ggml_tensor * attn_k_w = nullptr;
|
||||
ggml_tensor * attn_v_w = nullptr;
|
||||
ggml_tensor * attn_o_w = nullptr;
|
||||
|
||||
// Optional downsampling/norm in attention
|
||||
ggml_tensor * attn_k_dw_w = nullptr;
|
||||
ggml_tensor * attn_k_norm_w = nullptr;
|
||||
ggml_tensor * attn_v_dw_w = nullptr;
|
||||
ggml_tensor * attn_v_norm_w = nullptr;
|
||||
|
||||
// Block norm (often present in attention blocks)
|
||||
ggml_tensor * attn_norm_w = nullptr;
|
||||
};
|
||||
|
||||
struct yasa2_block {
|
||||
ggml_tensor * dw_w = nullptr;
|
||||
ggml_tensor * dw_b = nullptr;
|
||||
ggml_tensor * ln_w = nullptr;
|
||||
ggml_tensor * ln_b = nullptr;
|
||||
ggml_tensor * pw1_w = nullptr;
|
||||
ggml_tensor * pw1_b = nullptr;
|
||||
ggml_tensor * grn_w = nullptr;
|
||||
ggml_tensor * grn_b = nullptr;
|
||||
ggml_tensor * pw2_w = nullptr;
|
||||
ggml_tensor * pw2_b = nullptr;
|
||||
};
|
||||
|
||||
struct yasa2_stage {
|
||||
ggml_tensor * down_ln_w = nullptr;
|
||||
ggml_tensor * down_ln_b = nullptr;
|
||||
ggml_tensor * down_conv_w = nullptr;
|
||||
ggml_tensor * down_conv_b = nullptr;
|
||||
std::vector<yasa2_block> blocks;
|
||||
};
|
||||
|
||||
// QFormer projector block for models with 1 (or more) QFormer projectors
|
||||
// Granite Speech, Granite4 Vision
|
||||
struct qf_block {
|
||||
ggml_tensor * qf_proj_query = nullptr;
|
||||
ggml_tensor * qf_proj_norm_w = nullptr;
|
||||
ggml_tensor * qf_proj_norm_b = nullptr;
|
||||
ggml_tensor * qf_proj_linear_w = nullptr;
|
||||
ggml_tensor * qf_proj_linear_b = nullptr;
|
||||
ggml_tensor * qf_proj_post_norm_w = nullptr;
|
||||
ggml_tensor * qf_proj_post_norm_b = nullptr;
|
||||
ggml_tensor * qf_proj_img_pos = nullptr; // Vision only
|
||||
std::vector<clip_layer> qf_proj_layers;
|
||||
};
|
||||
|
||||
// pocket-tts SEANet stack, used in both directions:
|
||||
// encoder = conv_in -> per stage (residual unit, strided conv) -> conv_out
|
||||
// decoder = conv_in -> per stage (strided convtr, residual unit) -> conv_out
|
||||
struct clip_seanet {
|
||||
// one residual unit: ELU -> dilated conv -> ELU -> pointwise conv, added to the input
|
||||
struct stage {
|
||||
ggml_tensor * res_conv1_w = nullptr;
|
||||
ggml_tensor * res_conv1_b = nullptr;
|
||||
ggml_tensor * res_conv2_w = nullptr;
|
||||
ggml_tensor * res_conv2_b = nullptr;
|
||||
ggml_tensor * scale_conv_w = nullptr; // strided conv (encoder) or convtr (decoder)
|
||||
ggml_tensor * scale_conv_b = nullptr;
|
||||
};
|
||||
|
||||
ggml_tensor * conv_in_w = nullptr;
|
||||
ggml_tensor * conv_in_b = nullptr;
|
||||
ggml_tensor * conv_out_w = nullptr;
|
||||
ggml_tensor * conv_out_b = nullptr;
|
||||
std::vector<stage> stages;
|
||||
};
|
||||
|
||||
// pocket-tts flow-matching decoder (SimpleMLPAdaLN)
|
||||
struct clip_flow_net {
|
||||
// AdaLN res block: in_ln -> modulate -> Linear -> SiLU -> Linear, gated residual
|
||||
struct block {
|
||||
ggml_tensor * norm_w = nullptr;
|
||||
ggml_tensor * norm_b = nullptr;
|
||||
ggml_tensor * up_w = nullptr;
|
||||
ggml_tensor * up_b = nullptr;
|
||||
ggml_tensor * down_w = nullptr;
|
||||
ggml_tensor * down_b = nullptr;
|
||||
ggml_tensor * ada_w = nullptr; // -> shift, scale, gate
|
||||
ggml_tensor * ada_b = nullptr;
|
||||
};
|
||||
|
||||
// timestep embedder: cos/sin(t * freqs) -> Linear -> SiLU -> Linear -> RMSNorm
|
||||
struct time_embd {
|
||||
ggml_tensor * freqs = nullptr;
|
||||
ggml_tensor * up_w = nullptr;
|
||||
ggml_tensor * up_b = nullptr;
|
||||
ggml_tensor * down_w = nullptr;
|
||||
ggml_tensor * down_b = nullptr;
|
||||
ggml_tensor * norm = nullptr; // RMSNorm alpha
|
||||
};
|
||||
|
||||
ggml_tensor * input_proj_w = nullptr;
|
||||
ggml_tensor * input_proj_b = nullptr;
|
||||
ggml_tensor * cond_embd_w = nullptr;
|
||||
ggml_tensor * cond_embd_b = nullptr;
|
||||
ggml_tensor * final_ada_w = nullptr; // -> shift, scale
|
||||
ggml_tensor * final_ada_b = nullptr;
|
||||
ggml_tensor * final_proj_w = nullptr;
|
||||
ggml_tensor * final_proj_b = nullptr;
|
||||
std::vector<time_embd> time;
|
||||
std::vector<block> blocks;
|
||||
};
|
||||
|
||||
// qwen3tts code2wav: RVQ codes -> raw PCM
|
||||
struct clip_code2wav {
|
||||
// "upsample" stage: one ConvNeXt block plus the causal ConvTranspose1d before it
|
||||
struct upsample_block {
|
||||
ggml_tensor * conv_w = nullptr; // causal ConvTranspose1d, 2x
|
||||
ggml_tensor * conv_b = nullptr;
|
||||
ggml_tensor * dwconv_w = nullptr; // depthwise causal conv, k=7
|
||||
ggml_tensor * dwconv_b = nullptr;
|
||||
ggml_tensor * norm_w = nullptr; // LayerNorm
|
||||
ggml_tensor * norm_b = nullptr;
|
||||
ggml_tensor * pw1_w = nullptr; // pointwise expand
|
||||
ggml_tensor * pw1_b = nullptr;
|
||||
ggml_tensor * pw2_w = nullptr; // pointwise project
|
||||
ggml_tensor * pw2_b = nullptr;
|
||||
ggml_tensor * gamma = nullptr; // layer scale
|
||||
};
|
||||
|
||||
// one DAC residual unit: SnakeBeta -> dilated causal conv -> SnakeBeta -> pointwise causal conv
|
||||
struct dac_res {
|
||||
ggml_tensor * act1_alpha = nullptr;
|
||||
ggml_tensor * act1_beta = nullptr;
|
||||
ggml_tensor * conv1_w = nullptr;
|
||||
ggml_tensor * conv1_b = nullptr;
|
||||
ggml_tensor * act2_alpha = nullptr;
|
||||
ggml_tensor * act2_beta = nullptr;
|
||||
ggml_tensor * conv2_w = nullptr;
|
||||
ggml_tensor * conv2_b = nullptr;
|
||||
};
|
||||
|
||||
// one DAC upsample block (SnakeBeta -> causal ConvTranspose1d -> 3 residual units)
|
||||
struct dac_block {
|
||||
ggml_tensor * snake_alpha = nullptr;
|
||||
ggml_tensor * snake_beta = nullptr;
|
||||
ggml_tensor * conv_w = nullptr; // causal ConvTranspose1d
|
||||
ggml_tensor * conv_b = nullptr;
|
||||
std::vector<dac_res> res;
|
||||
};
|
||||
|
||||
// quantizer: RVQ codebook decode
|
||||
ggml_tensor * quant_first_in_w = nullptr; // semantic RVQ, in_proj (1x1 conv, loaded as 2D)
|
||||
ggml_tensor * quant_first_out_w = nullptr;
|
||||
ggml_tensor * quant_first_cb_w = nullptr; // codebook (1 layer)
|
||||
ggml_tensor * quant_rest_in_w = nullptr; // acoustic RVQ
|
||||
ggml_tensor * quant_rest_out_w = nullptr;
|
||||
ggml_tensor * quant_rest_cb_w = nullptr; // codebooks, merged 3D [15, vocab, dim]
|
||||
|
||||
ggml_tensor * pre_conv_w = nullptr;
|
||||
ggml_tensor * pre_conv_b = nullptr;
|
||||
|
||||
ggml_tensor * tfm_in_proj_w = nullptr;
|
||||
ggml_tensor * tfm_in_proj_b = nullptr;
|
||||
ggml_tensor * tfm_out_proj_w = nullptr;
|
||||
ggml_tensor * tfm_out_proj_b = nullptr;
|
||||
ggml_tensor * tfm_output_norm_w = nullptr;
|
||||
std::vector<clip_layer> tfm_layers; // reuses the generic block fields (ln_1/attn/ln_2/ffn/ls_1/ls_2)
|
||||
|
||||
std::vector<upsample_block> upsample;
|
||||
|
||||
ggml_tensor * dac_entry_w = nullptr;
|
||||
ggml_tensor * dac_entry_b = nullptr;
|
||||
std::vector<dac_block> dac;
|
||||
ggml_tensor * dac_post_snake_alpha = nullptr;
|
||||
ggml_tensor * dac_post_snake_beta = nullptr;
|
||||
ggml_tensor * dac_post_conv_w = nullptr;
|
||||
ggml_tensor * dac_post_conv_b = nullptr;
|
||||
};
|
||||
|
||||
struct clip_vocos {
|
||||
ggml_tensor * input_w = nullptr;
|
||||
ggml_tensor * input_b = nullptr;
|
||||
ggml_tensor * norm_w = nullptr;
|
||||
ggml_tensor * norm_b = nullptr;
|
||||
ggml_tensor * output_norm_w = nullptr;
|
||||
ggml_tensor * output_norm_b = nullptr;
|
||||
ggml_tensor * output_w = nullptr;
|
||||
ggml_tensor * output_b = nullptr;
|
||||
std::vector<clip_code2wav::upsample_block> blocks;
|
||||
};
|
||||
|
||||
struct clip_model {
|
||||
clip_modality modality = CLIP_MODALITY_VISION;
|
||||
projector_type proj_type = PROJECTOR_TYPE_MLP;
|
||||
clip_hparams hparams;
|
||||
|
||||
// embeddings
|
||||
ggml_tensor * class_embedding = nullptr;
|
||||
ggml_tensor * patch_embeddings_0 = nullptr;
|
||||
ggml_tensor * patch_embeddings_1 = nullptr; // additional patch kernel; interpretation is model-specific (Qwen2VL, Nemotron Omni)
|
||||
ggml_tensor * patch_bias = nullptr;
|
||||
ggml_tensor * position_embeddings = nullptr;
|
||||
ggml_tensor * norm_embd_w = nullptr;
|
||||
ggml_tensor * norm_embd_b = nullptr;
|
||||
|
||||
// "indexed" patch embedding norms
|
||||
ggml_tensor * patch_norm_1_w = nullptr;
|
||||
ggml_tensor * patch_norm_1_b = nullptr;
|
||||
ggml_tensor * patch_norm_2_w = nullptr;
|
||||
ggml_tensor * patch_norm_2_b = nullptr;
|
||||
ggml_tensor * patch_norm_3_w = nullptr;
|
||||
ggml_tensor * patch_norm_3_b = nullptr;
|
||||
|
||||
ggml_tensor * pre_ln_w = nullptr;
|
||||
ggml_tensor * pre_ln_b = nullptr;
|
||||
|
||||
std::vector<clip_layer> layers;
|
||||
|
||||
// one patch embedder per audio pyramid stage (muse-spark)
|
||||
struct audio_patch {
|
||||
ggml_tensor * proj_w = nullptr;
|
||||
ggml_tensor * proj_b = nullptr;
|
||||
ggml_tensor * norm_w = nullptr;
|
||||
};
|
||||
std::vector<audio_patch> a_patches;
|
||||
|
||||
int32_t n_deepstack_layers = 0; // used by Qwen3-VL, calculated from clip_layer
|
||||
|
||||
ggml_tensor * post_ln_w;
|
||||
ggml_tensor * post_ln_b;
|
||||
|
||||
ggml_tensor * mm_fc_w;
|
||||
ggml_tensor * mm_fc_b;
|
||||
ggml_tensor * mm_ffn_up_w = nullptr;
|
||||
ggml_tensor * mm_ffn_up_b = nullptr;
|
||||
ggml_tensor * mm_ffn_gate_w = nullptr;
|
||||
ggml_tensor * mm_ffn_gate_b = nullptr;
|
||||
ggml_tensor * mm_ffn_down_w = nullptr;
|
||||
ggml_tensor * mm_ffn_down_b = nullptr;
|
||||
ggml_tensor * mm_post_norm_w = nullptr;
|
||||
ggml_tensor * mm_post_norm_b = nullptr;
|
||||
|
||||
// LLaVA projection
|
||||
ggml_tensor * mm_input_norm_w = nullptr;
|
||||
ggml_tensor * mm_input_norm_b = nullptr;
|
||||
ggml_tensor * mm_0_w = nullptr;
|
||||
ggml_tensor * mm_0_b = nullptr;
|
||||
ggml_tensor * mm_2_w = nullptr;
|
||||
ggml_tensor * mm_2_b = nullptr;
|
||||
ggml_tensor * mm_merger_fc1_w = nullptr; // minimax-m3
|
||||
ggml_tensor * mm_merger_fc1_b = nullptr;
|
||||
ggml_tensor * mm_merger_fc2_w = nullptr;
|
||||
ggml_tensor * mm_merger_fc2_b = nullptr;
|
||||
|
||||
ggml_tensor * image_newline = nullptr;
|
||||
ggml_tensor * view_seperator = nullptr;
|
||||
ggml_tensor * tok_embd_sep = nullptr; // [n_embd_text, n_sep] rows of the text model tok_embd (MiniCPM-V 4.7)
|
||||
|
||||
|
||||
// Yi type models with mlp+normalization projection
|
||||
ggml_tensor * mm_1_w = nullptr; // Yi type models have 0, 1, 3, 4
|
||||
ggml_tensor * mm_1_b = nullptr;
|
||||
ggml_tensor * mm_3_w = nullptr;
|
||||
ggml_tensor * mm_3_b = nullptr;
|
||||
ggml_tensor * mm_4_w = nullptr;
|
||||
ggml_tensor * mm_4_b = nullptr;
|
||||
ggml_tensor * mm_5_w = nullptr;
|
||||
ggml_tensor * mm_5_b = nullptr;
|
||||
ggml_tensor * mm_6_w = nullptr;
|
||||
ggml_tensor * mm_6_b = nullptr;
|
||||
|
||||
// GLMV-Edge projection
|
||||
ggml_tensor * mm_model_adapter_conv_w = nullptr;
|
||||
ggml_tensor * mm_model_adapter_conv_b = nullptr;
|
||||
|
||||
// MobileVLM projection
|
||||
ggml_tensor * mm_model_mlp_1_w = nullptr;
|
||||
ggml_tensor * mm_model_mlp_1_b = nullptr;
|
||||
ggml_tensor * mm_model_mlp_3_w = nullptr;
|
||||
ggml_tensor * mm_model_mlp_3_b = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_0_0_w = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_0_1_w = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_0_1_b = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_1_fc1_w = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_1_fc1_b = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_1_fc2_w = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_1_fc2_b = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_2_0_w = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_2_1_w = nullptr;
|
||||
ggml_tensor * mm_model_block_1_block_2_1_b = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_0_0_w = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_0_1_w = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_0_1_b = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_1_fc1_w = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_1_fc1_b = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_1_fc2_w = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_1_fc2_b = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_2_0_w = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_2_1_w = nullptr;
|
||||
ggml_tensor * mm_model_block_2_block_2_1_b = nullptr;
|
||||
|
||||
// MobileVLM_V2 projection
|
||||
ggml_tensor * mm_model_mlp_0_w = nullptr;
|
||||
ggml_tensor * mm_model_mlp_0_b = nullptr;
|
||||
ggml_tensor * mm_model_mlp_2_w = nullptr;
|
||||
ggml_tensor * mm_model_mlp_2_b = nullptr;
|
||||
ggml_tensor * mm_model_peg_0_w = nullptr;
|
||||
ggml_tensor * mm_model_peg_0_b = nullptr;
|
||||
|
||||
// MINICPMV projection
|
||||
ggml_tensor * mm_model_pos_embed_k = nullptr;
|
||||
ggml_tensor * mm_model_query = nullptr;
|
||||
ggml_tensor * mm_model_proj = nullptr;
|
||||
ggml_tensor * mm_model_proj_b = nullptr;
|
||||
ggml_tensor * mm_model_kv_proj = nullptr;
|
||||
ggml_tensor * mm_model_attn_q_w = nullptr;
|
||||
ggml_tensor * mm_model_attn_q_b = nullptr;
|
||||
ggml_tensor * mm_model_attn_k_w = nullptr;
|
||||
ggml_tensor * mm_model_attn_k_b = nullptr;
|
||||
ggml_tensor * mm_model_attn_v_w = nullptr;
|
||||
ggml_tensor * mm_model_attn_v_b = nullptr;
|
||||
ggml_tensor * mm_model_attn_o_w = nullptr;
|
||||
ggml_tensor * mm_model_attn_o_b = nullptr;
|
||||
ggml_tensor * mm_model_ln_q_w = nullptr;
|
||||
ggml_tensor * mm_model_ln_q_b = nullptr;
|
||||
ggml_tensor * mm_model_ln_kv_w = nullptr;
|
||||
ggml_tensor * mm_model_ln_kv_b = nullptr;
|
||||
ggml_tensor * mm_model_ln_post_w = nullptr;
|
||||
ggml_tensor * mm_model_ln_post_b = nullptr;
|
||||
|
||||
// MiniCPM-V 4.6 ViT merger (window self-attention + ViT MLP downsample)
|
||||
ggml_tensor * vit_merger_ln1_w = nullptr;
|
||||
ggml_tensor * vit_merger_ln1_b = nullptr;
|
||||
ggml_tensor * vit_merger_attn_q_w = nullptr;
|
||||
ggml_tensor * vit_merger_attn_q_b = nullptr;
|
||||
ggml_tensor * vit_merger_attn_k_w = nullptr;
|
||||
ggml_tensor * vit_merger_attn_k_b = nullptr;
|
||||
ggml_tensor * vit_merger_attn_v_w = nullptr;
|
||||
ggml_tensor * vit_merger_attn_v_b = nullptr;
|
||||
ggml_tensor * vit_merger_attn_o_w = nullptr;
|
||||
ggml_tensor * vit_merger_attn_o_b = nullptr;
|
||||
ggml_tensor * vit_merger_ds_ln_w = nullptr;
|
||||
ggml_tensor * vit_merger_ds_ln_b = nullptr;
|
||||
ggml_tensor * vit_merger_ds_up_w = nullptr;
|
||||
ggml_tensor * vit_merger_ds_up_b = nullptr;
|
||||
ggml_tensor * vit_merger_ds_down_w = nullptr;
|
||||
ggml_tensor * vit_merger_ds_down_b = nullptr;
|
||||
|
||||
// gemma3
|
||||
ggml_tensor * mm_input_proj_w = nullptr;
|
||||
ggml_tensor * mm_soft_emb_norm_w = nullptr;
|
||||
|
||||
// mobilenetv5 for gemma3n
|
||||
std::vector<mobilenetv5_block> mobilenet_blocks;
|
||||
std::vector<int> mobilenet_stage_ends;
|
||||
ggml_tensor * mobilenet_stem_conv_w = nullptr;
|
||||
ggml_tensor * mobilenet_stem_conv_b = nullptr;
|
||||
ggml_tensor * mobilenet_stem_norm_w = nullptr;
|
||||
ggml_tensor * mm_post_proj_norm_w = nullptr;
|
||||
|
||||
// Multi-Scale Fusion Adapter (MSFA) components
|
||||
ggml_tensor * msfa_concat_conv_w = nullptr;
|
||||
ggml_tensor * msfa_concat_norm_w = nullptr;
|
||||
ggml_tensor * msfa_ffn_expand_w = nullptr;
|
||||
ggml_tensor * msfa_ffn_project_w = nullptr;
|
||||
ggml_tensor * msfa_ffn_expand_bn = nullptr;
|
||||
ggml_tensor * msfa_ffn_project_bn = nullptr;
|
||||
|
||||
// yasa2
|
||||
ggml_tensor * yasa_patch_w = nullptr;
|
||||
ggml_tensor * yasa_patch_b = nullptr;
|
||||
ggml_tensor * yasa_patch_ln_w = nullptr;
|
||||
ggml_tensor * yasa_patch_ln_b = nullptr;
|
||||
ggml_tensor * yasa_backbone_ln_w = nullptr;
|
||||
ggml_tensor * yasa_backbone_ln_b = nullptr;
|
||||
ggml_tensor * yasa_vision_pos_embed = nullptr;
|
||||
std::vector<yasa2_stage> yasa_stages;
|
||||
|
||||
// lfm3vl
|
||||
ggml_tensor * lfm3vl_embedding_norm_w = nullptr;
|
||||
|
||||
// pixtral, glm4v
|
||||
ggml_tensor * token_embd_img_break = nullptr;
|
||||
|
||||
// deepseek4v sentinel embeddings (image_newline is reused for IMAGE_NEW_LINE)
|
||||
ggml_tensor * token_embd_img_start = nullptr;
|
||||
ggml_tensor * token_embd_img_end = nullptr;
|
||||
ggml_tensor * token_embd_img_pad = nullptr;
|
||||
ggml_tensor * mm_patch_merger_w = nullptr;
|
||||
ggml_tensor * mm_patch_merger_b = nullptr;
|
||||
|
||||
// ultravox / whisper encoder
|
||||
ggml_tensor * conv1d_1_w = nullptr;
|
||||
ggml_tensor * conv1d_1_b = nullptr;
|
||||
ggml_tensor * conv1d_2_w = nullptr;
|
||||
ggml_tensor * conv1d_2_b = nullptr;
|
||||
ggml_tensor * conv_out_w = nullptr;
|
||||
ggml_tensor * conv_out_b = nullptr;
|
||||
ggml_tensor * mm_norm_pre_w = nullptr;
|
||||
ggml_tensor * mm_norm_pre_b = nullptr;
|
||||
ggml_tensor * mm_norm_mid_w = nullptr;
|
||||
|
||||
// mimo-audio-tokenizer: post-transformer downsample + RVQ codebook
|
||||
ggml_tensor * downsample_conv_w = nullptr; // no bias
|
||||
ggml_tensor * downsample_norm_w = nullptr;
|
||||
ggml_tensor * downsample_norm_b = nullptr;
|
||||
ggml_tensor * rvq_codebook = nullptr; // merged 3D [n_q, max_bins, dim]
|
||||
|
||||
// mimo-v2.5: text-side RVQ code embedding ("text codebook")
|
||||
ggml_tensor * mm_a_code_embd = nullptr; // merged 3D [n_channels, vocab, dim]
|
||||
|
||||
// mimo-v2.5: LLM-side connector (input_local_transformer, separate from the
|
||||
// audio_tokenizer's own encoder `layers`)
|
||||
std::vector<clip_layer> mm_a_local_layers;
|
||||
ggml_tensor * mm_a_local_norm_w = nullptr;
|
||||
|
||||
// qwen3a
|
||||
ggml_tensor * conv2d_1_w = nullptr;
|
||||
ggml_tensor * conv2d_1_b = nullptr;
|
||||
ggml_tensor * conv2d_2_w = nullptr;
|
||||
ggml_tensor * conv2d_2_b = nullptr;
|
||||
ggml_tensor * conv2d_3_w = nullptr;
|
||||
ggml_tensor * conv2d_3_b = nullptr;
|
||||
|
||||
// qwen3tts speaker encoder (ECAPA-TDNN)
|
||||
// reused tensors: stem conv is conv1d_1_w/b, feature aggregation is conv_out_w/b, output proj is mm_fc_w/b
|
||||
ggml_tensor * spk_asp_attn_w = nullptr;
|
||||
ggml_tensor * spk_asp_attn_b = nullptr;
|
||||
ggml_tensor * spk_asp_tdnn_w = nullptr;
|
||||
ggml_tensor * spk_asp_tdnn_b = nullptr;
|
||||
|
||||
// qwen3tts code_predictor
|
||||
ggml_tensor * gen_code_proj_in_w = nullptr; // small_to_mtp_projection
|
||||
ggml_tensor * gen_code_proj_in_b = nullptr;
|
||||
ggml_tensor * gen_code_embd_w = nullptr; // per-codebook embedding, merged 3D
|
||||
ggml_tensor * gen_code_head_w = nullptr; // per-codebook output head, merged 3D
|
||||
ggml_tensor * gen_code_out_embd_w = nullptr; // codebook-0 embedding, fed back into the talker
|
||||
ggml_tensor * gen_code_norm_w = nullptr; // final norm
|
||||
|
||||
// qwen3tts code2wav: RVQ codes -> raw PCM
|
||||
clip_code2wav c2w;
|
||||
|
||||
// pocket-tts: SEANet stack, shared by the encoder (speaker path) and the decoder (gen path)
|
||||
clip_seanet seanet;
|
||||
|
||||
// pocket-tts: voice latent -> backbone embd (speaker path)
|
||||
ggml_tensor * spk_proj_w = nullptr;
|
||||
ggml_tensor * downsample_w = nullptr;
|
||||
|
||||
// pocket-tts: flow-matching decoder, backbone hidden state -> next latent
|
||||
clip_flow_net flow;
|
||||
ggml_tensor * gen_out_eos_w = nullptr;
|
||||
ggml_tensor * gen_out_eos_b = nullptr;
|
||||
ggml_tensor * gen_input_lin_w = nullptr; // latent -> backbone embd
|
||||
ggml_tensor * gen_emb_mean = nullptr;
|
||||
ggml_tensor * gen_emb_std = nullptr;
|
||||
ggml_tensor * gen_quant_out_w = nullptr; // latent -> decoder dim
|
||||
ggml_tensor * gen_upsample_w = nullptr; // depthwise convtr, frame rate -> encoder frame rate
|
||||
std::vector<clip_layer> gen_tfm_layers; // mimi decoder_transformer
|
||||
|
||||
// soprano: Vocos decoder, hidden states -> STFT coefficients
|
||||
clip_vocos vocos;
|
||||
|
||||
// cogvlm
|
||||
ggml_tensor * mm_post_fc_norm_w = nullptr;
|
||||
ggml_tensor * mm_post_fc_norm_b = nullptr;
|
||||
ggml_tensor * mm_h_to_4h_w = nullptr;
|
||||
ggml_tensor * mm_gate_w = nullptr;
|
||||
ggml_tensor * mm_4h_to_h_w = nullptr;
|
||||
ggml_tensor * mm_boi = nullptr;
|
||||
ggml_tensor * mm_eoi = nullptr;
|
||||
|
||||
// hunyuanvl perceiver
|
||||
ggml_tensor * mm_pre_norm_w = nullptr;
|
||||
ggml_tensor * mm_img_begin = nullptr;
|
||||
ggml_tensor * mm_img_end = nullptr;
|
||||
|
||||
// deepseek ocr sam
|
||||
ggml_tensor * patch_embed_proj_w = nullptr;
|
||||
ggml_tensor * patch_embed_proj_b = nullptr;
|
||||
ggml_tensor * pos_embed = nullptr;
|
||||
|
||||
ggml_tensor * neck_0_w;
|
||||
ggml_tensor * neck_1_w;
|
||||
ggml_tensor * neck_1_b;
|
||||
ggml_tensor * neck_2_w;
|
||||
ggml_tensor * neck_3_w;
|
||||
ggml_tensor * neck_3_b;
|
||||
ggml_tensor * net_2;
|
||||
ggml_tensor * net_3;
|
||||
|
||||
int32_t n_sam_layers = 12; // used by deepseek-ocr sam encoder
|
||||
|
||||
std::vector<clip_layer> sam_layers;
|
||||
|
||||
// deepseek-ocr-2
|
||||
ggml_tensor * resample_query_768 = nullptr;
|
||||
ggml_tensor * resample_query_1024 = nullptr;
|
||||
|
||||
// lfm2 audio
|
||||
std::array<ggml_tensor *, 7> pre_encode_conv_X_w = {nullptr};
|
||||
std::array<ggml_tensor *, 7> pre_encode_conv_X_b = {nullptr};
|
||||
ggml_tensor * pre_encode_out_w = nullptr;
|
||||
ggml_tensor * pre_encode_out_b = nullptr;
|
||||
|
||||
// gemma4
|
||||
ggml_tensor * std_bias = nullptr;
|
||||
ggml_tensor * std_scale = nullptr;
|
||||
// Gemma4ClippableLinear
|
||||
struct clamp_info {
|
||||
float inp_max;
|
||||
float inp_min;
|
||||
float out_max;
|
||||
float out_min;
|
||||
};
|
||||
std::map<std::string, clamp_info> clamp_info_map;
|
||||
|
||||
// gemma4 audio conformer
|
||||
std::array<ggml_tensor *, 2> sscp_conv_w = {nullptr};
|
||||
std::array<ggml_tensor *, 2> sscp_conv_b = {nullptr};
|
||||
std::array<ggml_tensor *, 2> sscp_norm_w = {nullptr};
|
||||
ggml_tensor * sscp_inp_proj_w = nullptr;
|
||||
ggml_tensor * sscp_inp_proj_b = nullptr;
|
||||
ggml_tensor * audio_out_proj_w = nullptr;
|
||||
ggml_tensor * audio_out_proj_b = nullptr;
|
||||
|
||||
// granite_speech encoder
|
||||
ggml_tensor * inp_proj_w = nullptr;
|
||||
ggml_tensor * inp_proj_b = nullptr;
|
||||
ggml_tensor * ctc_out_w = nullptr;
|
||||
ggml_tensor * ctc_out_b = nullptr;
|
||||
ggml_tensor * ctc_out_mid_w = nullptr;
|
||||
ggml_tensor * ctc_out_mid_b = nullptr;
|
||||
// qformer projector(s)
|
||||
std::vector<qf_block> qf_proj_blocks;
|
||||
|
||||
bool audio_has_avgpool() const {
|
||||
return proj_type == PROJECTOR_TYPE_QWEN2A
|
||||
|| proj_type == PROJECTOR_TYPE_VOXTRAL
|
||||
|| proj_type == PROJECTOR_TYPE_MUSIC_FLAMINGO;
|
||||
}
|
||||
|
||||
bool audio_has_stack_frames() const {
|
||||
return proj_type == PROJECTOR_TYPE_ULTRAVOX
|
||||
|| proj_type == PROJECTOR_TYPE_VOXTRAL
|
||||
|| proj_type == PROJECTOR_TYPE_MERALION;
|
||||
}
|
||||
};
|
||||
|
||||
const clip_hparams * clip_get_hparams(const struct clip_ctx * ctx);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,160 @@
|
||||
#pragma once
|
||||
|
||||
#include "ggml.h"
|
||||
#include "mtmd.h"
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <map>
|
||||
|
||||
// !!! Internal header, to be used by mtmd only !!!
|
||||
|
||||
#define MTMD_INTERNAL_HEADER
|
||||
|
||||
struct clip_ctx;
|
||||
struct clip_enc_state;
|
||||
struct clip_gen_state;
|
||||
|
||||
struct clip_image_size {
|
||||
int width;
|
||||
int height;
|
||||
bool operator==(const clip_image_size & other) const {
|
||||
return width == other.width && height == other.height;
|
||||
}
|
||||
bool operator!=(const clip_image_size & other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
int area() const {
|
||||
// avoid overflow when computing area
|
||||
GGML_ASSERT(width >= 0 && width <= 46000);
|
||||
GGML_ASSERT(height >= 0 && height <= 46000);
|
||||
return width * height;
|
||||
}
|
||||
};
|
||||
|
||||
struct clip_image_f32;
|
||||
struct clip_image_f32_batch;
|
||||
|
||||
enum clip_modality {
|
||||
CLIP_MODALITY_VISION,
|
||||
CLIP_MODALITY_AUDIO,
|
||||
CLIP_MODALITY_GEN_AUDIO,
|
||||
};
|
||||
|
||||
enum clip_flash_attn_type {
|
||||
CLIP_FLASH_ATTN_TYPE_AUTO = -1,
|
||||
CLIP_FLASH_ATTN_TYPE_DISABLED = 0,
|
||||
CLIP_FLASH_ATTN_TYPE_ENABLED = 1,
|
||||
};
|
||||
|
||||
struct clip_context_params {
|
||||
bool use_gpu;
|
||||
ggml_backend_dev_t device;
|
||||
enum clip_flash_attn_type flash_attn_type;
|
||||
int image_min_tokens;
|
||||
int image_max_tokens;
|
||||
bool warmup;
|
||||
ggml_backend_sched_eval_callback cb_eval;
|
||||
void * cb_eval_user_data;
|
||||
bool no_alloc;
|
||||
mtmd_progress_callback progress_callback;
|
||||
void * progress_callback_user_data;
|
||||
};
|
||||
|
||||
struct clip_init_result {
|
||||
struct clip_ctx * ctx_v; // vision context
|
||||
struct clip_ctx * ctx_a; // audio context
|
||||
struct clip_ctx * ctx_gen_a; // audio generation context
|
||||
};
|
||||
|
||||
struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params);
|
||||
|
||||
// max number of output tokens per image, -1 if not dynamic size
|
||||
int clip_get_image_max_tokens(const struct clip_ctx * ctx);
|
||||
|
||||
void clip_free(struct clip_ctx * ctx);
|
||||
|
||||
// TODO: should be enum, not string
|
||||
const char * clip_patch_merge_type(const struct clip_ctx * ctx);
|
||||
|
||||
int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img);
|
||||
|
||||
// for M-RoPE, this will be the number of token positions in X and Y directions
|
||||
// for other models, X will be the total number of tokens and Y will be 1
|
||||
int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img);
|
||||
int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img);
|
||||
|
||||
// this should be equal to the embedding dimension of the text model
|
||||
int clip_n_mmproj_embd(const struct clip_ctx * ctx);
|
||||
|
||||
// TODO: remove clip_image_encode() and always use batched version
|
||||
bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector<float> & out_vec);
|
||||
bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector<float> & out_batch_embd);
|
||||
|
||||
enum clip_gen_process_type {
|
||||
CLIP_GEN_PROCESS_GEN_UNKNOWN,
|
||||
CLIP_GEN_PROCESS_GEN_CODE, // h_state to codes
|
||||
CLIP_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio
|
||||
};
|
||||
struct clip_encode_params {
|
||||
int n_threads = 1;
|
||||
const clip_image_f32_batch * imgs = nullptr;
|
||||
std::vector<float> * out_embd = nullptr;
|
||||
|
||||
// for audio gen, imgs has exactly one entry: hidden state from backbone (GEN_CODE) or unused (GEN_WAV)
|
||||
clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_UNKNOWN;
|
||||
|
||||
// GEN_CODE: out_embd receives the embd to feed back to the backbone
|
||||
int32_t code0 = 0; // semantic code sampled by the backbone
|
||||
int32_t top_k = 50;
|
||||
float top_p = 1.0f;
|
||||
std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
|
||||
std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
|
||||
float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders
|
||||
bool * out_is_eos = nullptr;
|
||||
|
||||
// GEN_WAV
|
||||
const std::vector<int32_t> * codes = nullptr; // this frame's 16 RVQ codes
|
||||
const std::vector<float> * feats = nullptr; // continuous counterpart of codes
|
||||
std::vector<float> * out_audio = nullptr; // decoded PCM samples, F32
|
||||
|
||||
// audio gen: state of one sequence, clip_encode() updates it in place
|
||||
struct clip_gen_state * gen_state = nullptr;
|
||||
|
||||
// streaming audio input, updated in place by clip_encode()
|
||||
struct clip_enc_state * enc_state = nullptr;
|
||||
};
|
||||
bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params);
|
||||
|
||||
// persistent state of a causal audio encoder, for streaming input (one per stream)
|
||||
// it holds the sliding window of past keys/values, so each clip_encode() call only takes the new frames
|
||||
// n_frames_max is the maximum number of encoder frames per call
|
||||
struct clip_enc_state * clip_enc_state_init(struct clip_ctx * ctx, int32_t n_frames_max);
|
||||
void clip_enc_state_free(struct clip_enc_state * state);
|
||||
|
||||
// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls
|
||||
// seed UINT32_MAX means random; reset() clears the slots and keeps the RNG stream
|
||||
struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed);
|
||||
void clip_gen_state_free(struct clip_gen_state * state);
|
||||
void clip_gen_state_reset(struct clip_gen_state * state);
|
||||
|
||||
bool clip_is_llava(const struct clip_ctx * ctx);
|
||||
// note for contributor: this clip_is_(model) pattern is deprecated
|
||||
// do NOT add new functions like this
|
||||
|
||||
bool clip_has_vision_encoder(const struct clip_ctx * ctx);
|
||||
bool clip_has_audio_encoder(const struct clip_ctx * ctx);
|
||||
|
||||
bool clip_support_batch(const struct clip_ctx * ctx);
|
||||
|
||||
int clip_model_n_temporal_merge(const struct clip_ctx * ctx); // TODO @ngxson : remove, refactor this
|
||||
|
||||
std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx);
|
||||
|
||||
struct clip_cap {
|
||||
bool has_vision;
|
||||
bool has_audio;
|
||||
bool has_gen_audio;
|
||||
};
|
||||
struct clip_cap clip_get_cap(const char * fname);
|
||||
@@ -0,0 +1,294 @@
|
||||
#include "mtmd-debug.h"
|
||||
|
||||
#include "arg.h"
|
||||
#include "debug.h"
|
||||
#include "log.h"
|
||||
#include "common.h"
|
||||
#include "llama.h"
|
||||
#include "ggml.h"
|
||||
#include "mtmd.h"
|
||||
#include "mtmd-helper.h"
|
||||
|
||||
#include <vector>
|
||||
#include <cmath>
|
||||
#include <limits.h>
|
||||
#include <cinttypes>
|
||||
#include <clocale>
|
||||
|
||||
// INTERNAL TOOL FOR DEBUGGING PURPOSES ONLY
|
||||
// NOT INTENDED FOR PUBLIC USE
|
||||
|
||||
static void show_additional_info(int /*argc*/, char ** argv) {
|
||||
LOG(
|
||||
"Internal debugging tool for mtmd; See mtmd-debug.md for the pytorch equivalent code\n"
|
||||
"Note: we repurpose some args from other examples, they will have different meaning here\n"
|
||||
"\n"
|
||||
"Usage: %s -m <model> --mmproj <mmproj> -p <mode> -n <size> --image <image> --audio <audio>\n"
|
||||
"\n"
|
||||
" -n <size>: number of pixels per edge for image (always square image), or number of samples for audio\n"
|
||||
"\n"
|
||||
" -p \"encode\" (debugging encode pass, default case):\n"
|
||||
" --image can be:\n"
|
||||
" \"white\", \"black\", \"gray\": filled 1.0f, 0.0f and 0.5f respectively\n"
|
||||
" \"red\", \"green\", \"blue\": filled with respective colors\n"
|
||||
" \"cb\": checkerboard pattern, alternate 1.0f and 0.0f\n"
|
||||
" \"rainbow\": raspberry-pi-like rainbow pattern\n"
|
||||
" --audio can be:\n"
|
||||
" \"one\", \"zero\", \"half\": filled 1.0f, 0.0f and 0.5f respectively\n"
|
||||
" \"1010\": checkerboard pattern, alternate 1.0f and 0.0f\n"
|
||||
"\n"
|
||||
" -p \"preproc\" (debugging preprocessing pass):\n"
|
||||
" --image can be:\n"
|
||||
" \"white\", \"black\", \"gray\": filled image with respective colors\n"
|
||||
" \"cb\": checkerboard pattern\n"
|
||||
" --audio can be:\n"
|
||||
" \"one\", \"zero\", \"half\": filled 1.0f, 0.0f and 0.5f respectively\n"
|
||||
" \"440\": sine wave with 440 Hz frequency\n"
|
||||
"\n",
|
||||
argv[0]
|
||||
);
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
|
||||
ggml_time_init();
|
||||
|
||||
common_params params;
|
||||
|
||||
common_init();
|
||||
|
||||
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_MTMD, show_additional_info)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
mtmd_helper_log_set(common_log_default_callback, nullptr);
|
||||
|
||||
if (params.mmproj.path.empty()) {
|
||||
show_additional_info(argc, argv);
|
||||
LOG_ERR("ERR: Missing --mmproj argument\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
ggml_backend_load_all();
|
||||
|
||||
LOG_INF("%s: loading model: %s\n", __func__, params.model.path.c_str());
|
||||
|
||||
mtmd::context_ptr ctx_mtmd;
|
||||
common_init_result_ptr llama_init;
|
||||
common_debug_cb_user_data cb_data;
|
||||
|
||||
llama_init = common_init_from_params(params);
|
||||
{
|
||||
auto * model = llama_init->model();
|
||||
const char * clip_path = params.mmproj.path.c_str();
|
||||
mtmd_context_params mparams = mtmd_context_params_default();
|
||||
mparams.use_gpu = params.mmproj_use_gpu;
|
||||
mparams.device = params.mmproj_device;
|
||||
mparams.print_timings = true;
|
||||
mparams.n_threads = params.cpuparams.n_threads;
|
||||
mparams.flash_attn_type = params.flash_attn_type;
|
||||
mparams.warmup = params.warmup;
|
||||
mparams.image_min_tokens = params.image_min_tokens;
|
||||
mparams.image_max_tokens = params.image_max_tokens;
|
||||
{
|
||||
// always enable debug callback
|
||||
mparams.cb_eval_user_data = &cb_data;
|
||||
mparams.cb_eval = common_debug_cb_eval;
|
||||
}
|
||||
ctx_mtmd.reset(mtmd_init_from_file(clip_path, model, mparams));
|
||||
if (!ctx_mtmd.get()) {
|
||||
LOG_ERR("Failed to load vision model from %s\n", clip_path);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
std::string input;
|
||||
int32_t inp_size = params.n_predict;
|
||||
if (params.image.empty()) {
|
||||
LOG_ERR("ERR: At least one of --image or --audio must be specified\n");
|
||||
return 1;
|
||||
}
|
||||
if (inp_size <= 0) {
|
||||
LOG_ERR("ERR: Invalid size specified with -n, must be greater than 0\n");
|
||||
return 1;
|
||||
}
|
||||
input = params.image[0];
|
||||
|
||||
if (params.prompt.empty() || params.prompt == "encode") {
|
||||
std::vector<std::vector<float>> image;
|
||||
std::vector<float> samples;
|
||||
|
||||
if (input == "black") {
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
auto row = std::vector<float>(inp_size * 3, 0.0f);
|
||||
image.push_back(row);
|
||||
}
|
||||
} else if (input == "white") {
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
auto row = std::vector<float>(inp_size * 3, 1.0f);
|
||||
image.push_back(row);
|
||||
}
|
||||
} else if (input == "gray") {
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
auto row = std::vector<float>(inp_size * 3, 0.5f);
|
||||
image.push_back(row);
|
||||
}
|
||||
} else if (input == "cb") {
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
auto row = std::vector<float>(inp_size * 3, 0.0f);
|
||||
image.push_back(row);
|
||||
}
|
||||
for (int y = 0; y < inp_size; ++y) {
|
||||
for (int x = 0; x < inp_size; ++x) {
|
||||
float v = ((x + y) % 2) ? 0.0f : 1.0f;
|
||||
image[y][x * 3 + 0] = v;
|
||||
image[y][x * 3 + 1] = v;
|
||||
image[y][x * 3 + 2] = v;
|
||||
}
|
||||
}
|
||||
} else if (input == "red") {
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
auto row = std::vector<float>(inp_size * 3, 0.0f);
|
||||
for (int j = 0; j < inp_size; ++j) {
|
||||
row[j * 3 + 0] = 1.0f; // R channel
|
||||
}
|
||||
image.push_back(row);
|
||||
}
|
||||
} else if (input == "green") {
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
auto row = std::vector<float>(inp_size * 3, 0.0f);
|
||||
for (int j = 0; j < inp_size; ++j) {
|
||||
row[j * 3 + 1] = 1.0f; // G channel
|
||||
}
|
||||
image.push_back(row);
|
||||
}
|
||||
} else if (input == "blue") {
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
auto row = std::vector<float>(inp_size * 3, 0.0f);
|
||||
for (int j = 0; j < inp_size; ++j) {
|
||||
row[j * 3 + 2] = 1.0f; // B channel
|
||||
}
|
||||
image.push_back(row);
|
||||
}
|
||||
} else if (input == "rainbow") {
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
image.push_back(std::vector<float>(inp_size * 3, 0.0f));
|
||||
}
|
||||
float cx = inp_size / 2.0f;
|
||||
float cy = inp_size / 2.0f;
|
||||
float max_dist = std::sqrt(cx * cx + cy * cy);
|
||||
for (int y = 0; y < inp_size; ++y) {
|
||||
for (int x = 0; x < inp_size; ++x) {
|
||||
float dx = x - cx;
|
||||
float dy = y - cy;
|
||||
float hue = std::atan2(dy, dx) / (2.0f * 3.14159265f);
|
||||
if (hue < 0) hue += 1.0f;
|
||||
float sat = std::sqrt(dx * dx + dy * dy) / max_dist;
|
||||
if (sat > 1.0f) sat = 1.0f;
|
||||
float h6 = hue * 6.0f;
|
||||
int i6 = (int)h6;
|
||||
float f = h6 - i6;
|
||||
float p = 1.0f - sat;
|
||||
float q = 1.0f - sat * f;
|
||||
float t = 1.0f - sat * (1.0f - f);
|
||||
float r, g, b;
|
||||
switch (i6 % 6) {
|
||||
case 0: r=1; g=t; b=p; break;
|
||||
case 1: r=q; g=1; b=p; break;
|
||||
case 2: r=p; g=1; b=t; break;
|
||||
case 3: r=p; g=q; b=1; break;
|
||||
case 4: r=t; g=p; b=1; break;
|
||||
default: r=1; g=p; b=q; break;
|
||||
}
|
||||
image[y][x * 3 + 0] = r;
|
||||
image[y][x * 3 + 1] = g;
|
||||
image[y][x * 3 + 2] = b;
|
||||
}
|
||||
}
|
||||
} else if (input == "one") {
|
||||
samples = std::vector<float>(inp_size, 1.0f);
|
||||
} else if (input == "zero") {
|
||||
samples = std::vector<float>(inp_size, 0.0f);
|
||||
} else if (input == "half") {
|
||||
samples = std::vector<float>(inp_size, 0.5f);
|
||||
} else if (input == "1010") {
|
||||
samples.resize(inp_size);
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
samples[i] = (i % 2) ? 0.0f : 1.0f;
|
||||
}
|
||||
} else {
|
||||
LOG_ERR("ERR: Invalid input specified with --image/--audio\n");
|
||||
show_additional_info(argc, argv);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// run encode pass
|
||||
LOG_INF("Running encode pass for input type: %s\n", input.c_str());
|
||||
if (samples.size() > 0) {
|
||||
LOG_INF("Input audio with %zu samples, type: %s\n", samples.size(), input.c_str());
|
||||
mtmd_debug_encode_audio(ctx_mtmd.get(), samples);
|
||||
} else {
|
||||
LOG_INF("Input image with dimensions %d x %d, type: %s\n", inp_size, inp_size, input.c_str());
|
||||
mtmd_debug_encode_image(ctx_mtmd.get(), image);
|
||||
}
|
||||
|
||||
} else if (params.prompt == "preproc") {
|
||||
std::vector<uint8_t> rgb_values;
|
||||
std::vector<float> pcm_samples;
|
||||
|
||||
if (input == "black") {
|
||||
rgb_values = std::vector<uint8_t>(inp_size * inp_size * 3, 0);
|
||||
} else if (input == "white") {
|
||||
rgb_values = std::vector<uint8_t>(inp_size * inp_size * 3, 255);
|
||||
} else if (input == "gray") {
|
||||
rgb_values = std::vector<uint8_t>(inp_size * inp_size * 3, 128);
|
||||
} else if (input == "cb") {
|
||||
rgb_values.resize(inp_size * inp_size * 3);
|
||||
for (int y = 0; y < inp_size; ++y) {
|
||||
for (int x = 0; x < inp_size; ++x) {
|
||||
uint8_t v = ((x + y) % 2) ? 0 : 255;
|
||||
rgb_values[(y * inp_size + x) * 3 + 0] = v;
|
||||
rgb_values[(y * inp_size + x) * 3 + 1] = v;
|
||||
rgb_values[(y * inp_size + x) * 3 + 2] = v;
|
||||
}
|
||||
}
|
||||
} else if (input == "one") {
|
||||
pcm_samples = std::vector<float>(inp_size, 1.0f);
|
||||
} else if (input == "zero") {
|
||||
pcm_samples = std::vector<float>(inp_size, 0.0f);
|
||||
} else if (input == "half") {
|
||||
pcm_samples = std::vector<float>(inp_size, 0.5f);
|
||||
} else if (input == "440") {
|
||||
pcm_samples.resize(inp_size);
|
||||
float freq = 440.0f;
|
||||
float sample_rate = mtmd_get_audio_sample_rate(ctx_mtmd.get());
|
||||
float pi = 3.14159265f;
|
||||
for (int i = 0; i < inp_size; ++i) {
|
||||
pcm_samples[i] = sinf(2 * pi * freq * i / sample_rate);
|
||||
}
|
||||
} else {
|
||||
LOG_ERR("ERR: Invalid input specified with --image/--audio\n");
|
||||
show_additional_info(argc, argv);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// run preprocessing pass
|
||||
LOG_INF("Running preprocessing pass for input type: %s\n", input.c_str());
|
||||
if (pcm_samples.size() > 0) {
|
||||
LOG_INF("Input audio with %zu samples, type: %s\n", pcm_samples.size(), input.c_str());
|
||||
mtmd_debug_preprocess_audio(ctx_mtmd.get(), pcm_samples);
|
||||
} else {
|
||||
LOG_INF("Input image with dimensions %d x %d, type: %s\n", inp_size, inp_size, input.c_str());
|
||||
mtmd_debug_preprocess_image(ctx_mtmd.get(), rgb_values, inp_size, inp_size);
|
||||
}
|
||||
|
||||
} else {
|
||||
LOG_ERR("ERR: Invalid mode specified with -p\n");
|
||||
show_additional_info(argc, argv);
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
#pragma once
|
||||
|
||||
#include "mtmd.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
// INTERNAL HEADER FOR DEBUGGING PURPOSES ONLY
|
||||
// NOT INTENDED FOR PUBLIC USE
|
||||
// Do not raise issues related to this debugging API
|
||||
|
||||
// encode take the pre-processed f32 values, print the intermidiate values via cb_eval callback
|
||||
MTMD_API void mtmd_debug_encode_image(mtmd_context * ctx, const std::vector<std::vector<float>> & image);
|
||||
MTMD_API void mtmd_debug_encode_audio(mtmd_context * ctx, const std::vector<float> & input); // will be broadcasted to fit n_mel
|
||||
|
||||
// preprocess take the raw input values
|
||||
MTMD_API void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector<uint8_t> & rgb_values, int nx, int ny);
|
||||
MTMD_API void mtmd_debug_preprocess_audio(mtmd_context * ctx, const std::vector<float> & pcm_samples);
|
||||
@@ -0,0 +1,62 @@
|
||||
# mtmd-debug
|
||||
|
||||
## Debugging encode pass
|
||||
|
||||
Example of debugging an input gray image (raw, not preprocessed):
|
||||
|
||||
```py
|
||||
from transformers import AutoModel
|
||||
|
||||
model = AutoModel.from_pretrained(...)
|
||||
|
||||
def test_vision():
|
||||
img_size = 896 # number of patches per side
|
||||
pixel_values = torch.zeros(1, 3, img_size, img_size) + 0.5 # gray image
|
||||
with torch.no_grad():
|
||||
outputs = model.model.get_image_features(pixel_values=pixel_values)
|
||||
print("last_hidden_state shape:", outputs.last_hidden_state.shape)
|
||||
print("last_hidden_state:", outputs.last_hidden_state)
|
||||
|
||||
test_vision()
|
||||
```
|
||||
|
||||
Example of debugging a rainbow image:
|
||||
|
||||
```py
|
||||
import torch
|
||||
import math
|
||||
|
||||
def make_rainbow(img_size):
|
||||
cx, cy = img_size / 2.0, img_size / 2.0
|
||||
max_dist = math.sqrt(cx * cx + cy * cy)
|
||||
img = torch.zeros(1, 3, img_size, img_size)
|
||||
for y in range(img_size):
|
||||
for x in range(img_size):
|
||||
dx, dy = x - cx, y - cy
|
||||
hue = math.atan2(dy, dx) / (2 * math.pi)
|
||||
if hue < 0:
|
||||
hue += 1
|
||||
sat = math.sqrt(dx * dx + dy * dy) / max_dist
|
||||
sat = min(sat, 1.0)
|
||||
h6 = hue * 6
|
||||
i6 = int(h6)
|
||||
f = h6 - i6
|
||||
p = 1 - sat
|
||||
q = 1 - sat * f
|
||||
t = 1 - sat * (1 - f)
|
||||
rgb = [(1,t,p),(q,1,p),(p,1,t),(p,q,1),(t,p,1),(1,p,q)][i6 % 6]
|
||||
img[0, 0, y, x] = rgb[0]
|
||||
img[0, 1, y, x] = rgb[1]
|
||||
img[0, 2, y, x] = rgb[2]
|
||||
return img
|
||||
|
||||
img_size = 896
|
||||
pixel_values = make_rainbow(img_size)
|
||||
with torch.no_grad():
|
||||
outputs = model.model.get_image_features(pixel_values=pixel_values)
|
||||
print("last_hidden_state:", outputs.last_hidden_state)
|
||||
```
|
||||
|
||||
## Debugging preprocess pass
|
||||
|
||||
(TODO)
|
||||
@@ -0,0 +1,26 @@
|
||||
#include <clocale>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
|
||||
std::string filename = "main";
|
||||
if (argc >= 1) {
|
||||
filename = argv[0];
|
||||
}
|
||||
|
||||
// Get only the program name from the full path
|
||||
size_t pos = filename.find_last_of("/\\");
|
||||
if (pos != std::string::npos) {
|
||||
filename = filename.substr(pos+1);
|
||||
}
|
||||
|
||||
fprintf(stdout, "\n");
|
||||
fprintf(stdout, "WARNING: The binary '%s' is deprecated.\n", filename.c_str());
|
||||
fprintf(stdout, "Please use 'llama-mtmd-cli' instead.\n");
|
||||
fprintf(stdout, "\n");
|
||||
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
@@ -0,0 +1,412 @@
|
||||
import argparse
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
from gguf import *
|
||||
from transformers import CLIPModel, CLIPProcessor, CLIPVisionModel, SiglipVisionModel
|
||||
|
||||
TEXT = "clip.text"
|
||||
VISION = "clip.vision"
|
||||
|
||||
|
||||
def k(raw_key: str, arch: str) -> str:
|
||||
return raw_key.format(arch=arch)
|
||||
|
||||
|
||||
def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: bool) -> bool:
|
||||
if name in (
|
||||
"logit_scale",
|
||||
"text_model.embeddings.position_ids",
|
||||
"vision_model.embeddings.position_ids",
|
||||
):
|
||||
return True
|
||||
|
||||
if has_llava and name in ["visual_projection.weight", "vision_model.post_layernorm.weight", "vision_model.post_layernorm.bias"]:
|
||||
return True
|
||||
|
||||
if name.startswith("v") and not has_vision:
|
||||
return True
|
||||
|
||||
if name.startswith("t") and not has_text:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_tensor_name(name: str) -> str:
|
||||
# Standardize the transformers llava next keys for
|
||||
# image newline / mm projector with the classes in haotian-liu LLaVA
|
||||
if name == "image_newline":
|
||||
return "model.image_newline"
|
||||
if name.startswith("multi_modal_projector"):
|
||||
name = name.replace("multi_modal_projector", "mm")
|
||||
if "linear_1" in name:
|
||||
name = name.replace("linear_1", "0")
|
||||
if "linear_2" in name:
|
||||
name = name.replace("linear_2", "2")
|
||||
return name
|
||||
|
||||
if "projection" in name:
|
||||
return name
|
||||
if "mm_projector" in name:
|
||||
name = name.replace("model.mm_projector", "mm")
|
||||
name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
|
||||
name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
|
||||
return name
|
||||
|
||||
return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
|
||||
|
||||
|
||||
def bytes_to_unicode():
|
||||
"""
|
||||
Returns list of utf-8 byte and a corresponding list of unicode strings.
|
||||
The reversible bpe codes work on unicode strings.
|
||||
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
|
||||
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
|
||||
This is a significant percentage of your normal, say, 32K bpe vocab.
|
||||
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
|
||||
And avoids mapping to whitespace/control characters the bpe code barfs on.
|
||||
"""
|
||||
bs = (
|
||||
list(range(ord("!"), ord("~") + 1))
|
||||
+ list(range(ord("¡"), ord("¬") + 1))
|
||||
+ list(range(ord("®"), ord("ÿ") + 1))
|
||||
)
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(2**8):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(2**8 + n)
|
||||
n += 1
|
||||
cs = [chr(n) for n in cs]
|
||||
return dict(zip(bs, cs))
|
||||
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True)
|
||||
ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16")
|
||||
ap.add_argument('--bigendian', action="store_true", default=False, help="Model is executed on big-endian machine")
|
||||
ap.add_argument("--text-only", action="store_true", required=False,
|
||||
help="Save a text-only model. It can't be used to encode images")
|
||||
ap.add_argument("--vision-only", action="store_true", required=False,
|
||||
help="Save a vision-only model. It can't be used to encode texts")
|
||||
ap.add_argument("--clip-model-is-vision", action="store_true", required=False,
|
||||
help="The clip model is a pure vision model (ShareGPT4V vision extract for example)")
|
||||
|
||||
# Selectable visual encoders that are compatible with this script
|
||||
encoder_group = ap.add_mutually_exclusive_group()
|
||||
encoder_group.add_argument("--clip-model-is-openclip", action="store_true", required=False,
|
||||
help="The clip model is from openclip (for ViT-SO400M type))")
|
||||
encoder_group.add_argument("--clip-model-is-siglip", action="store_true", required=False,
|
||||
help="the visual encoder is Siglip.")
|
||||
|
||||
ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.")
|
||||
ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp")
|
||||
ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None)
|
||||
# Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711
|
||||
# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5
|
||||
default_image_mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
default_image_std = [0.26862954, 0.26130258, 0.27577711]
|
||||
ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
|
||||
ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
|
||||
|
||||
# with proper
|
||||
args = ap.parse_args()
|
||||
|
||||
|
||||
if args.text_only and args.vision_only:
|
||||
print("--text-only and --image-only arguments cannot be specified at the same time.")
|
||||
exit(1)
|
||||
|
||||
if args.use_f32:
|
||||
print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.")
|
||||
|
||||
# output in the same directory as the model if output_dir is None
|
||||
dir_model = args.model_dir
|
||||
|
||||
if (
|
||||
args.clip_model_is_vision or
|
||||
not os.path.exists(dir_model + "/vocab.json") or
|
||||
args.clip_model_is_openclip or
|
||||
args.clip_model_is_siglip
|
||||
):
|
||||
vocab = None
|
||||
tokens = None
|
||||
else:
|
||||
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
|
||||
vocab = json.load(f)
|
||||
tokens = [key for key in vocab]
|
||||
|
||||
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
if args.clip_model_is_vision:
|
||||
v_hparams = config
|
||||
t_hparams = None
|
||||
else:
|
||||
v_hparams = config["vision_config"]
|
||||
t_hparams = config["text_config"]
|
||||
|
||||
# possible data types
|
||||
# ftype == 0 -> float32
|
||||
# ftype == 1 -> float16
|
||||
#
|
||||
# map from ftype to string
|
||||
ftype_str = ["f32", "f16"]
|
||||
|
||||
ftype = 1
|
||||
if args.use_f32:
|
||||
ftype = 0
|
||||
|
||||
if args.clip_model_is_siglip:
|
||||
model = SiglipVisionModel.from_pretrained(dir_model)
|
||||
processor = None
|
||||
elif args.clip_model_is_vision or args.clip_model_is_openclip:
|
||||
model = CLIPVisionModel.from_pretrained(dir_model)
|
||||
processor = None
|
||||
else:
|
||||
model = CLIPModel.from_pretrained(dir_model)
|
||||
processor = CLIPProcessor.from_pretrained(dir_model)
|
||||
|
||||
fname_middle = None
|
||||
has_text_encoder = True
|
||||
has_vision_encoder = True
|
||||
has_llava_projector = False
|
||||
if args.text_only:
|
||||
fname_middle = "text-"
|
||||
has_vision_encoder = False
|
||||
elif args.llava_projector is not None:
|
||||
fname_middle = "mmproj-"
|
||||
has_text_encoder = False
|
||||
has_llava_projector = True
|
||||
elif args.vision_only:
|
||||
fname_middle = "vision-"
|
||||
has_text_encoder = False
|
||||
else:
|
||||
fname_middle = ""
|
||||
|
||||
output_dir = args.output_dir if args.output_dir is not None else dir_model
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_prefix = os.path.basename(output_dir).replace("ggml_", "")
|
||||
fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
|
||||
fout = GGUFWriter(path=fname_out, arch="clip", endianess=GGUFEndian.LITTLE if not args.bigendian else GGUFEndian.BIG)
|
||||
|
||||
fout.add_bool("clip.has_text_encoder", has_text_encoder)
|
||||
fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
|
||||
fout.add_bool("clip.has_llava_projector", has_llava_projector)
|
||||
fout.add_file_type(ftype)
|
||||
model_name = config["_name_or_path"] if "_name_or_path" in config else os.path.basename(dir_model)
|
||||
fout.add_name(model_name)
|
||||
if args.text_only:
|
||||
fout.add_description("text-only CLIP model")
|
||||
elif args.vision_only and not has_llava_projector:
|
||||
fout.add_description("vision-only CLIP model")
|
||||
elif has_llava_projector:
|
||||
fout.add_description("image encoder for LLaVA")
|
||||
# add projector type
|
||||
fout.add_string("clip.projector_type", args.projector_type)
|
||||
else:
|
||||
fout.add_description("two-tower CLIP model")
|
||||
|
||||
if has_text_encoder:
|
||||
assert t_hparams is not None
|
||||
assert tokens is not None
|
||||
if args.clip_model_is_siglip:
|
||||
text_projection_dim = 0
|
||||
else:
|
||||
text_projection_dim = t_hparams.get("projection_dim", config["projection_dim"])
|
||||
# text_model hparams
|
||||
fout.add_uint32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"])
|
||||
fout.add_uint32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"])
|
||||
fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"])
|
||||
fout.add_uint32("clip.text.projection_dim", text_projection_dim)
|
||||
fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"])
|
||||
fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, TEXT), t_hparams["layer_norm_eps"])
|
||||
fout.add_uint32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"])
|
||||
fout.add_token_list(tokens)
|
||||
|
||||
|
||||
|
||||
def get_non_negative_vision_feature_layers(v_hparams):
|
||||
"""
|
||||
Determine the vision feature layer(s) for the llava model, which are indices into the
|
||||
hidden states of the visual encoder. Note that the hidden states array generally takes the
|
||||
form:
|
||||
|
||||
[<emb input>, <output of enc block 0>, ... <output of enc block num_hidden_layers>]
|
||||
|
||||
so feature indices should be offset as n+1 to get the output of encoder block n.
|
||||
We convert all vision feature layers to non-negative so that -1 can be used in
|
||||
the model as an unset value. If no vision feature layer is found, we leave it unset.
|
||||
"""
|
||||
num_hidden_layers = v_hparams["num_hidden_layers"]
|
||||
to_non_negative = lambda layer_idx: layer_idx if layer_idx >= 0 else num_hidden_layers + layer_idx + 1
|
||||
feature_layers_key = None
|
||||
# Key used for llava models in transformers
|
||||
if "vision_feature_layer" in config:
|
||||
feature_layers_key = "vision_feature_layer"
|
||||
# Key used for llava models in the original format
|
||||
elif "mm_vision_select_layer" in config:
|
||||
feature_layers_key = "mm_vision_select_layer"
|
||||
if feature_layers_key is not None:
|
||||
feature_layers = config[feature_layers_key]
|
||||
if isinstance(feature_layers, int):
|
||||
feature_layers = [feature_layers]
|
||||
return [to_non_negative(feature_layer) for feature_layer in feature_layers]
|
||||
|
||||
# Determine if we have explicitly specified vision feature layers in our config
|
||||
feature_layers = get_non_negative_vision_feature_layers(v_hparams)
|
||||
|
||||
if has_vision_encoder:
|
||||
# Siglip does not have a visual projector; set projection dim to 0
|
||||
if args.clip_model_is_siglip:
|
||||
visual_projection_dim = 0
|
||||
else:
|
||||
visual_projection_dim = v_hparams.get("projection_dim", config["projection_dim"])
|
||||
|
||||
# set vision_model hparams
|
||||
fout.add_uint32("clip.vision.image_size", v_hparams["image_size"])
|
||||
fout.add_uint32("clip.vision.patch_size", v_hparams["patch_size"])
|
||||
fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"])
|
||||
fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"])
|
||||
fout.add_uint32("clip.vision.projection_dim", visual_projection_dim)
|
||||
fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"])
|
||||
fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), v_hparams["layer_norm_eps"])
|
||||
if feature_layers:
|
||||
block_count = max(feature_layers)
|
||||
else:
|
||||
block_count = v_hparams["num_hidden_layers"] - 1 if has_llava_projector else v_hparams["num_hidden_layers"]
|
||||
fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), block_count)
|
||||
# /**
|
||||
# "image_grid_pinpoints": [
|
||||
# [
|
||||
# 336,
|
||||
# 672
|
||||
# ],
|
||||
# [
|
||||
# 672,
|
||||
# 336
|
||||
# ],
|
||||
# [
|
||||
# 672,
|
||||
# 672
|
||||
# ],
|
||||
# [
|
||||
# 1008,
|
||||
# 336
|
||||
# ],
|
||||
# [
|
||||
# 336,
|
||||
# 1008
|
||||
# ]
|
||||
# ],
|
||||
# Flattened:
|
||||
# [
|
||||
# 336, 672,
|
||||
# 672, 336,
|
||||
# 672, 672,
|
||||
# 1008, 336,
|
||||
# 336, 1008
|
||||
# ]
|
||||
# *
|
||||
# */
|
||||
if "image_grid_pinpoints" in v_hparams:
|
||||
# flatten it
|
||||
image_grid_pinpoints = []
|
||||
for pinpoint in v_hparams["image_grid_pinpoints"]:
|
||||
for p in pinpoint:
|
||||
image_grid_pinpoints.append(p)
|
||||
fout.add_array("clip.vision.image_grid_pinpoints", image_grid_pinpoints)
|
||||
if "image_crop_resolution" in v_hparams:
|
||||
fout.add_uint32("clip.vision.image_crop_resolution", v_hparams["image_crop_resolution"])
|
||||
if "image_aspect_ratio" in v_hparams:
|
||||
fout.add_string("clip.vision.image_aspect_ratio", v_hparams["image_aspect_ratio"])
|
||||
if "image_split_resolution" in v_hparams:
|
||||
fout.add_uint32("clip.vision.image_split_resolution", v_hparams["image_split_resolution"])
|
||||
if "mm_patch_merge_type" in v_hparams:
|
||||
fout.add_string("clip.vision.mm_patch_merge_type", v_hparams["mm_patch_merge_type"])
|
||||
if "mm_projector_type" in v_hparams:
|
||||
fout.add_string("clip.vision.mm_projector_type", v_hparams["mm_projector_type"])
|
||||
if feature_layers:
|
||||
fout.add_array("clip.vision.feature_layer", feature_layers)
|
||||
|
||||
if processor is not None:
|
||||
image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean # pyright: ignore[reportAttributeAccessIssue]
|
||||
image_std = processor.image_processor.image_std if args.image_std is None or args.image_std == default_image_std else args.image_std # pyright: ignore[reportAttributeAccessIssue]
|
||||
else:
|
||||
image_mean = args.image_mean if args.image_mean is not None else default_image_mean
|
||||
image_std = args.image_std if args.image_std is not None else default_image_std
|
||||
fout.add_array("clip.vision.image_mean", image_mean)
|
||||
fout.add_array("clip.vision.image_std", image_std)
|
||||
|
||||
use_gelu = v_hparams["hidden_act"] == "gelu"
|
||||
fout.add_bool("clip.use_gelu", use_gelu)
|
||||
|
||||
|
||||
if has_llava_projector:
|
||||
# By default, we drop the last layer for llava projector
|
||||
# models unless we have explicitly set vision feature layers
|
||||
if feature_layers is None:
|
||||
model.vision_model.encoder.layers.pop(-1)
|
||||
else:
|
||||
model.vision_model.encoder.layers = model.vision_model.encoder.layers[:max(feature_layers)]
|
||||
|
||||
projector = torch.load(args.llava_projector)
|
||||
for name, data in projector.items():
|
||||
name = get_tensor_name(name)
|
||||
# pw and dw conv ndim==4
|
||||
if data.ndim == 2 or data.ndim == 4:
|
||||
data = data.squeeze().numpy().astype(np.float16)
|
||||
else:
|
||||
data = data.squeeze().numpy().astype(np.float32)
|
||||
|
||||
fout.add_tensor(name, data)
|
||||
|
||||
print("Projector tensors added\n")
|
||||
|
||||
state_dict = model.state_dict()
|
||||
for name, data in state_dict.items():
|
||||
if should_skip_tensor(name, has_text_encoder, has_vision_encoder, has_llava_projector):
|
||||
# we don't need this
|
||||
print(f"skipping parameter: {name}")
|
||||
continue
|
||||
|
||||
name = get_tensor_name(name)
|
||||
data = data.squeeze().numpy()
|
||||
|
||||
n_dims = len(data.shape)
|
||||
|
||||
# ftype == 0 -> float32, ftype == 1 -> float16
|
||||
ftype_cur = 0
|
||||
if n_dims == 4:
|
||||
print(f"tensor {name} is always saved in f16")
|
||||
data = data.astype(np.float16)
|
||||
ftype_cur = 1
|
||||
elif ftype == 1:
|
||||
if name[-7:] == ".weight" and n_dims == 2:
|
||||
print(" Converting to float16")
|
||||
data = data.astype(np.float16)
|
||||
ftype_cur = 1
|
||||
else:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
else:
|
||||
if data.dtype != np.float32:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
|
||||
print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
|
||||
fout.add_tensor(name, data)
|
||||
|
||||
|
||||
fout.write_header_to_file()
|
||||
fout.write_kv_data_to_file()
|
||||
fout.write_tensors_to_file()
|
||||
fout.close()
|
||||
|
||||
print("Done. Output file: " + fname_out)
|
||||
@@ -0,0 +1,280 @@
|
||||
import argparse
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
from gguf import *
|
||||
|
||||
TEXT = "clip.text"
|
||||
VISION = "clip.vision"
|
||||
from transformers import SiglipVisionModel, SiglipVisionConfig
|
||||
|
||||
def k(raw_key: str, arch: str) -> str:
|
||||
return raw_key.format(arch=arch)
|
||||
|
||||
|
||||
def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_llava: bool) -> bool:
|
||||
if name in (
|
||||
"logit_scale",
|
||||
"text_model.embeddings.position_ids",
|
||||
"vision_model.embeddings.position_ids",
|
||||
):
|
||||
return True
|
||||
|
||||
if name in (
|
||||
"vision_model.head.probe",
|
||||
"vision_model.head.attention.in_proj_weight",
|
||||
"vision_model.head.attention.in_proj_bias",
|
||||
"vision_model.head.attention.out_proj.weight",
|
||||
"vision_model.head.attention.out_proj.bias",
|
||||
"vision_model.head.layernorm.weight",
|
||||
"vision_model.head.layernorm.bias",
|
||||
"vision_model.head.mlp.fc1.weight",
|
||||
"vision_model.head.mlp.fc1.bias",
|
||||
"vision_model.head.mlp.fc2.weight",
|
||||
"vision_model.head.mlp.fc2.bias"
|
||||
):
|
||||
return True
|
||||
|
||||
if name.startswith("v") and not has_vision:
|
||||
return True
|
||||
|
||||
if name.startswith("t") and not has_text:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_tensor_name(name: str) -> str:
|
||||
if "projection" in name:
|
||||
return name
|
||||
if "mm_projector" in name:
|
||||
name = name.replace("model.mm_projector", "mm")
|
||||
name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
|
||||
name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
|
||||
return name
|
||||
|
||||
return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
|
||||
|
||||
|
||||
def bytes_to_unicode():
|
||||
"""
|
||||
Returns list of utf-8 byte and a corresponding list of unicode strings.
|
||||
The reversible bpe codes work on unicode strings.
|
||||
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
|
||||
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
|
||||
This is a significant percentage of your normal, say, 32K bpe vocab.
|
||||
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
|
||||
And avoids mapping to whitespace/control characters the bpe code barfs on.
|
||||
"""
|
||||
bs = (
|
||||
list(range(ord("!"), ord("~") + 1))
|
||||
+ list(range(ord("¡"), ord("¬") + 1))
|
||||
+ list(range(ord("®"), ord("ÿ") + 1))
|
||||
)
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(2**8):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(2**8 + n)
|
||||
n += 1
|
||||
cs = [chr(n) for n in cs]
|
||||
return dict(zip(bs, cs))
|
||||
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True)
|
||||
ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16")
|
||||
ap.add_argument("--text-only", action="store_true", required=False,
|
||||
help="Save a text-only model. It can't be used to encode images")
|
||||
ap.add_argument("--vision-only", action="store_true", required=False,
|
||||
help="Save a vision-only model. It can't be used to encode texts")
|
||||
ap.add_argument("--clip-model-is-vision", action="store_true", required=False,
|
||||
help="The clip model is a pure vision model (ShareGPT4V vision extract for example)")
|
||||
ap.add_argument("--clip-model-is-openclip", action="store_true", required=False,
|
||||
help="The clip model is from openclip (for ViT-SO400M type))")
|
||||
ap.add_argument("--llava-projector", help="Path to llava.projector file. If specified, save an image encoder for LLaVA models.")
|
||||
ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2","adapter"], default="adapter")
|
||||
ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None)
|
||||
# Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711
|
||||
# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5
|
||||
default_image_mean = [0.5, 0.5, 0.5]
|
||||
default_image_std = [0.5, 0.5, 0.5]
|
||||
ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
|
||||
ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
|
||||
|
||||
# with proper
|
||||
args = ap.parse_args()
|
||||
|
||||
|
||||
if args.text_only and args.vision_only:
|
||||
print("--text-only and --image-only arguments cannot be specified at the same time.")
|
||||
exit(1)
|
||||
|
||||
if args.use_f32:
|
||||
print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.")
|
||||
|
||||
# output in the same directory as the model if output_dir is None
|
||||
dir_model = args.model_dir
|
||||
|
||||
if args.clip_model_is_vision or not os.path.exists(dir_model + "/vocab.json") or args.clip_model_is_openclip:
|
||||
vocab = None
|
||||
tokens = None
|
||||
else:
|
||||
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
|
||||
vocab = json.load(f)
|
||||
tokens = [key for key in vocab]
|
||||
|
||||
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
if args.clip_model_is_vision:
|
||||
v_hparams = config
|
||||
t_hparams = None
|
||||
else:
|
||||
v_hparams = config["vision_config"]
|
||||
t_hparams = None
|
||||
|
||||
# possible data types
|
||||
# ftype == 0 -> float32
|
||||
# ftype == 1 -> float16
|
||||
#
|
||||
# map from ftype to string
|
||||
ftype_str = ["f32", "f16"]
|
||||
|
||||
ftype = 1
|
||||
if args.use_f32:
|
||||
ftype = 0
|
||||
|
||||
vision_config = SiglipVisionConfig(**v_hparams)
|
||||
model = SiglipVisionModel(vision_config)
|
||||
model.load_state_dict(torch.load(os.path.join(dir_model, "glm.clip")))
|
||||
|
||||
fname_middle = None
|
||||
has_text_encoder = False
|
||||
has_vision_encoder = True
|
||||
has_glm_projector = True
|
||||
if args.text_only:
|
||||
fname_middle = "text-"
|
||||
has_vision_encoder = False
|
||||
elif args.llava_projector is not None:
|
||||
fname_middle = "mmproj-"
|
||||
has_text_encoder = False
|
||||
has_glm_projector = True
|
||||
elif args.vision_only:
|
||||
fname_middle = "vision-"
|
||||
has_text_encoder = False
|
||||
else:
|
||||
fname_middle = ""
|
||||
|
||||
output_dir = args.output_dir if args.output_dir is not None else dir_model
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_prefix = os.path.basename(output_dir).replace("ggml_", "")
|
||||
fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
|
||||
fout = GGUFWriter(path=fname_out, arch="clip")
|
||||
|
||||
fout.add_bool("clip.has_text_encoder", has_text_encoder)
|
||||
fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
|
||||
fout.add_bool("clip.has_glm_projector", has_glm_projector)
|
||||
fout.add_file_type(ftype)
|
||||
model_name = config["_name_or_path"] if "_name_or_path" in config else os.path.basename(dir_model)
|
||||
fout.add_name(model_name)
|
||||
if has_glm_projector:
|
||||
fout.add_description("image encoder for glm4v")
|
||||
fout.add_string("clip.projector_type", "adapter")
|
||||
else:
|
||||
fout.add_description("two-tower CLIP model")
|
||||
|
||||
if has_text_encoder:
|
||||
assert t_hparams is not None
|
||||
assert tokens is not None
|
||||
# text_model hparams
|
||||
fout.add_uint32(k(KEY_CONTEXT_LENGTH, TEXT), t_hparams["max_position_embeddings"])
|
||||
fout.add_uint32(k(KEY_EMBEDDING_LENGTH, TEXT), t_hparams["hidden_size"])
|
||||
fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, TEXT), t_hparams["intermediate_size"])
|
||||
fout.add_uint32("clip.text.projection_dim", t_hparams.get("projection_dim", config["projection_dim"]))
|
||||
fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, TEXT), t_hparams["num_attention_heads"])
|
||||
fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, TEXT), t_hparams["layer_norm_eps"])
|
||||
fout.add_uint32(k(KEY_BLOCK_COUNT, TEXT), t_hparams["num_hidden_layers"])
|
||||
fout.add_token_list(tokens)
|
||||
|
||||
if has_vision_encoder:
|
||||
# vision_model hparams
|
||||
fout.add_uint32("clip.vision.image_size", v_hparams["image_size"])
|
||||
fout.add_uint32("clip.vision.patch_size", v_hparams["patch_size"])
|
||||
fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), v_hparams["hidden_size"])
|
||||
fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), v_hparams["intermediate_size"])
|
||||
fout.add_uint32("clip.vision.projection_dim", 0)
|
||||
fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), v_hparams["num_attention_heads"])
|
||||
fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
|
||||
fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), v_hparams["num_hidden_layers"])
|
||||
|
||||
image_mean = args.image_mean if args.image_mean is not None else default_image_mean
|
||||
image_std = args.image_std if args.image_std is not None else default_image_std
|
||||
fout.add_array("clip.vision.image_mean", image_mean)
|
||||
fout.add_array("clip.vision.image_std", image_std)
|
||||
|
||||
fout.add_bool("clip.use_gelu", True)
|
||||
|
||||
|
||||
if has_glm_projector:
|
||||
# model.vision_model.encoder.layers.pop(-1) # pyright: ignore[reportAttributeAccessIssue]
|
||||
projector = torch.load(args.llava_projector)
|
||||
for name, data in projector.items():
|
||||
name = get_tensor_name(name)
|
||||
# pw and dw conv ndim==4
|
||||
if data.ndim == 2 or data.ndim == 4:
|
||||
data = data.squeeze().numpy().astype(np.float16)
|
||||
else:
|
||||
data = data.squeeze().numpy().astype(np.float32)
|
||||
if name.startswith("vision."):
|
||||
name=name.replace("vision.","")
|
||||
fout.add_tensor(name, data)
|
||||
print(f"Projector {name} - {data.dtype} - shape = {data.shape}")
|
||||
# print(f"Projector {name} tensors added\n")
|
||||
|
||||
state_dict = model.state_dict() # pyright: ignore[reportAttributeAccessIssue]
|
||||
for name, data in state_dict.items():
|
||||
if should_skip_tensor(name, has_text_encoder, has_vision_encoder, has_glm_projector):
|
||||
# we don't need this
|
||||
print(f"skipping parameter: {name}")
|
||||
continue
|
||||
|
||||
name = get_tensor_name(name)
|
||||
data = data.squeeze().numpy()
|
||||
|
||||
n_dims = len(data.shape)
|
||||
|
||||
# ftype == 0 -> float32, ftype == 1 -> float16
|
||||
ftype_cur = 0
|
||||
if n_dims == 4:
|
||||
print(f"tensor {name} is always saved in f16")
|
||||
data = data.astype(np.float16)
|
||||
ftype_cur = 1
|
||||
elif ftype == 1:
|
||||
if name[-7:] == ".weight" and n_dims == 2:
|
||||
# print(" Converting to float16")
|
||||
data = data.astype(np.float16)
|
||||
ftype_cur = 1
|
||||
else:
|
||||
# print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
else:
|
||||
if data.dtype != np.float32:
|
||||
# print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
print(f"siglip {name} - {data.dtype} - shape = {data.shape}")
|
||||
# print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
|
||||
fout.add_tensor(name, data)
|
||||
|
||||
|
||||
fout.write_header_to_file()
|
||||
fout.write_kv_data_to_file()
|
||||
fout.write_tensors_to_file()
|
||||
fout.close()
|
||||
|
||||
print("Done. Output file: " + fname_out)
|
||||
@@ -0,0 +1,33 @@
|
||||
import argparse
|
||||
import os
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-m", "--model", help="Path to GLM model")
|
||||
args = ap.parse_args()
|
||||
|
||||
# find the model part that includes the the multimodal projector weights
|
||||
model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True)
|
||||
checkpoint = model.state_dict()
|
||||
|
||||
# get a list of mm tensor names
|
||||
mm_tensors = [k for k, v in checkpoint.items() if k.startswith("vision.adapter.")]
|
||||
|
||||
# store these tensors in a new dictionary and torch.save them
|
||||
projector = {name: checkpoint[name].float() for name in mm_tensors}
|
||||
torch.save(projector, f"{args.model}/glm.projector")
|
||||
|
||||
clip_tensors = [k for k, v in checkpoint.items() if k.startswith("vision.vit.model.vision_model.")]
|
||||
if len(clip_tensors) > 0:
|
||||
clip = {name.replace("vision.vit.model.", ""): checkpoint[name].float() for name in clip_tensors}
|
||||
torch.save(clip, f"{args.model}/glm.clip")
|
||||
|
||||
# added tokens should be removed to be able to convert Mistral models
|
||||
if os.path.exists(f"{args.model}/added_tokens.json"):
|
||||
with open(f"{args.model}/added_tokens.json", "w") as f:
|
||||
f.write("{}\n")
|
||||
|
||||
print("Done!")
|
||||
print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
|
||||
print(f"Also, use {args.model}glm.projector to prepare a glm-encoder.gguf file.")
|
||||
@@ -0,0 +1,38 @@
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import torch
|
||||
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-m", "--model", help="Path to LLaVA v1.5 model")
|
||||
args = ap.parse_args()
|
||||
|
||||
# find the model part that includes the the multimodal projector weights
|
||||
path = sorted(glob.glob(f"{args.model}/pytorch_model*.bin"))[-1]
|
||||
checkpoint = torch.load(path)
|
||||
|
||||
# get a list of mm tensor names
|
||||
mm_tensors = [k for k, v in checkpoint.items() if k.startswith("model.mm_projector")]
|
||||
|
||||
# store these tensors in a new dictionary and torch.save them
|
||||
projector = {name: checkpoint[name].float() for name in mm_tensors}
|
||||
torch.save(projector, f"{args.model}/llava.projector")
|
||||
|
||||
# BakLLaVA models contain CLIP tensors in it
|
||||
clip_tensors = [k for k, v in checkpoint.items() if k.startswith("model.vision_tower")]
|
||||
if len(clip_tensors) > 0:
|
||||
clip = {name.replace("vision_tower.vision_tower.", ""): checkpoint[name].float() for name in clip_tensors}
|
||||
torch.save(clip, f"{args.model}/llava.clip")
|
||||
|
||||
|
||||
# added tokens should be removed to be able to convert Mistral models
|
||||
if os.path.exists(f"{args.model}/added_tokens.json"):
|
||||
with open(f"{args.model}/added_tokens.json", "w") as f:
|
||||
f.write("{}\n")
|
||||
|
||||
|
||||
|
||||
print("Done!")
|
||||
print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
|
||||
print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.")
|
||||
@@ -0,0 +1,180 @@
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import save_file
|
||||
from typing import Any, ContextManager, cast
|
||||
|
||||
# Function to determine if file is a SafeTensor file
|
||||
def is_safetensor_file(file_path):
|
||||
return file_path.endswith('.safetensors')
|
||||
|
||||
|
||||
# Unified loading function
|
||||
def load_model(file_path):
|
||||
if is_safetensor_file(file_path):
|
||||
tensors = {}
|
||||
with cast(ContextManager[Any], safe_open(file_path, framework="pt", device="cpu")) as f:
|
||||
for key in f.keys():
|
||||
tensors[key] = f.get_tensor(key).clone()
|
||||
# output shape
|
||||
print(f"{key} : {tensors[key].shape}")
|
||||
return tensors, 'safetensor'
|
||||
else:
|
||||
return torch.load(file_path, map_location=torch.device('cpu')), 'pytorch'
|
||||
|
||||
|
||||
# Unified saving function
|
||||
def save_model(model, file_path, file_type):
|
||||
if file_type == 'safetensor':
|
||||
# safe_save(model, file_path)
|
||||
save_file(model, file_path)
|
||||
else:
|
||||
torch.save(model, file_path)
|
||||
|
||||
# Helpers to match weight names from specific components or
|
||||
# determine if a saved shard contains that component
|
||||
def is_vision_tower(weight_name):
|
||||
return (
|
||||
weight_name.startswith("model.vision_tower") or
|
||||
weight_name.startswith("vit.") or
|
||||
weight_name.startswith("vision_tower")
|
||||
)
|
||||
|
||||
def is_newline(weight_name):
|
||||
return (
|
||||
weight_name.startswith("model.image_newline") or
|
||||
weight_name.startswith("image_newline")
|
||||
)
|
||||
|
||||
def is_mm_projector(weight_name):
|
||||
return (
|
||||
weight_name.startswith("model.mm_projector") or
|
||||
weight_name.startswith("vision_proj.") or
|
||||
weight_name.startswith("multi_modal_projector")
|
||||
)
|
||||
|
||||
def newline_criteria(checkpoint):
|
||||
return any(is_newline(k) for k in checkpoint.keys())
|
||||
|
||||
def proj_criteria(checkpoint):
|
||||
return any(is_mm_projector(k) for k in checkpoint.keys())
|
||||
|
||||
# Adapted function to clean vision tower from checkpoint
|
||||
def clean_vision_tower_from_checkpoint(checkpoint_path):
|
||||
checkpoint, file_type = load_model(checkpoint_path)
|
||||
# file_type = 'pytorch'
|
||||
model_path = os.path.dirname(checkpoint_path)
|
||||
print(f"Searching for vision tower tensors in {checkpoint_path}")
|
||||
clip_tensors = [k for k, v in checkpoint.items() if is_vision_tower(k)]
|
||||
|
||||
if len(clip_tensors) > 0:
|
||||
print(f"Found {len(clip_tensors)} tensors to extract from {checkpoint_path}")
|
||||
# Adapted for file type
|
||||
clip_path = os.path.join(model_path, "llava.clip")
|
||||
|
||||
if os.path.exists(clip_path):
|
||||
print(f"Loading existing llava.clip from {clip_path}")
|
||||
existing_clip, _ = load_model(clip_path)
|
||||
else:
|
||||
print(f"Creating new llava.clip at {clip_path}")
|
||||
existing_clip = {}
|
||||
# Update existing_clip with new tensors, avoid duplicates
|
||||
for name in clip_tensors:
|
||||
simple_name = name[name.index('vision_model.'):] if 'vision_model.' in name else name
|
||||
print(f"Adding {simple_name} to llava.clip")
|
||||
if simple_name not in existing_clip:
|
||||
existing_clip[simple_name] = checkpoint[name]
|
||||
|
||||
# Save the updated clip tensors back to llava.clip
|
||||
save_model(existing_clip, clip_path, 'pytorch')
|
||||
|
||||
# Remove the tensors from the original checkpoint
|
||||
for name in clip_tensors:
|
||||
del checkpoint[name]
|
||||
|
||||
checkpoint_path = checkpoint_path
|
||||
return True
|
||||
return False
|
||||
|
||||
def find_relevant_checkpoints(checkpoint_paths, newline_criteria, projector):
|
||||
newline_checkpoint_path = None
|
||||
projector_checkpoint_path = None
|
||||
|
||||
for path in checkpoint_paths:
|
||||
checkpoint, _ = load_model(path)
|
||||
if newline_criteria(checkpoint) and newline_checkpoint_path is None:
|
||||
newline_checkpoint_path = path
|
||||
if projector(checkpoint):
|
||||
projector_checkpoint_path = path
|
||||
|
||||
return newline_checkpoint_path, projector_checkpoint_path
|
||||
|
||||
|
||||
# Command-line interface setup
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-m", "--model", required=True, help="Path to LLaVA v1.5+ model")
|
||||
ap.add_argument("-C", "--clean-vision-tower", action="store_true", help="Remove any vision tower from the model files")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.clean_vision_tower:
|
||||
# Generalized to handle both PyTorch and SafeTensors models
|
||||
model_files = sorted(glob.glob(f"{args.model}/*"), key=os.path.getmtime, reverse=True)
|
||||
# checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and path.startswith('pytorch')) or (path.endswith('.safetensors') and path.startswith('model'))]
|
||||
checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and 'pytorch' in path.split('/')[-1].split('\\')[-1]) or (path.endswith('.safetensors') and 'model' in path.split('/')[-1].split('\\')[-1])]
|
||||
for projector_checkpoint_path in checkpoint_paths:
|
||||
print(f"Cleaning {projector_checkpoint_path}")
|
||||
if not clean_vision_tower_from_checkpoint(projector_checkpoint_path):
|
||||
print(f"No vision tower found in {projector_checkpoint_path}")
|
||||
# we break once none is found, so far all models append them at the end
|
||||
# break
|
||||
print("Done! All vision tower tensors are removed from the model files and stored in llava.clip file.")
|
||||
|
||||
# Now we look for the projector in the last checkpoint
|
||||
model_files = sorted(glob.glob(f"{args.model}/*"), key=os.path.getmtime, reverse=True)
|
||||
checkpoint_paths = [path for path in model_files if (path.endswith('.bin') and 'pytorch' in path.split('/')[-1].split('\\')[-1]) or (path.endswith('.safetensors') and 'model' in path.split('/')[-1].split('\\')[-1])]
|
||||
# last_checkpoint_path = checkpoint_paths[0]
|
||||
# first_checkpoint_path = checkpoint_paths[-1]
|
||||
newline_checkpoint_path, projector_checkpoint_path = find_relevant_checkpoints(checkpoint_paths, newline_criteria, proj_criteria)
|
||||
|
||||
print(f"Taking projector from {projector_checkpoint_path}")
|
||||
first_mm_tensors = []
|
||||
first_checkpoint = None
|
||||
if newline_checkpoint_path is not None:
|
||||
print(f"Taking newline from {newline_checkpoint_path}")
|
||||
first_checkpoint, file_type = load_model(newline_checkpoint_path)
|
||||
first_mm_tensors = [k for k, v in first_checkpoint.items() if is_newline(k)]
|
||||
|
||||
# Load the checkpoint
|
||||
mm_tensors = []
|
||||
last_checkpoint = None
|
||||
if projector_checkpoint_path is not None:
|
||||
last_checkpoint, file_type = load_model(projector_checkpoint_path)
|
||||
mm_tensors = [k for k, v in last_checkpoint.items() if is_mm_projector(k)]
|
||||
|
||||
if len(mm_tensors) == 0:
|
||||
if last_checkpoint is not None:
|
||||
for k, v in last_checkpoint.items():
|
||||
print(k)
|
||||
print(f"Found {len(mm_tensors)} tensors to extract out of {len(last_checkpoint) if last_checkpoint is not None else 0} tensors.")
|
||||
print("No tensors found. Is this a LLaVA model?")
|
||||
exit()
|
||||
|
||||
print(f"Found {len(mm_tensors)} tensors to extract.")
|
||||
print(f"Found additional {len(first_mm_tensors)} tensors to extract.")
|
||||
# projector = {name: checkpoint.[name].float() for name in mm_tensors}
|
||||
projector = {}
|
||||
for name in mm_tensors:
|
||||
assert last_checkpoint is not None
|
||||
projector[name] = last_checkpoint[name].float()
|
||||
for name in first_mm_tensors:
|
||||
assert first_checkpoint is not None
|
||||
projector[name] = first_checkpoint[name].float()
|
||||
|
||||
if len(projector) > 0:
|
||||
save_model(projector, f"{args.model}/llava.projector", 'pytorch')
|
||||
|
||||
print("Done!")
|
||||
print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
|
||||
print(f"Also, use {args.model}/llava.projector to prepare a llava-encoder.gguf file.")
|
||||
@@ -0,0 +1,892 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 Google AI and The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" PyTorch Siglip model. """
|
||||
# Copied from HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit and add tgt_sizes
|
||||
|
||||
|
||||
import os
|
||||
import math
|
||||
import warnings
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from torch.nn.init import _calculate_fan_in_and_fan_out
|
||||
|
||||
from transformers.activations import ACT2FN
|
||||
from transformers.modeling_utils import PreTrainedModel
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import (
|
||||
logging,
|
||||
)
|
||||
from transformers.utils import logging
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
class SiglipVisionConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`SiglipVisionModel`]. It is used to instantiate a
|
||||
Siglip vision encoder according to the specified arguments, defining the model architecture. Instantiating a
|
||||
configuration with the defaults will yield a similar configuration to that of the vision encoder of the Siglip
|
||||
[google/siglip-base-patch16-224](https://huggingface.co/google/siglip-base-patch16-224) architecture.
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
Args:
|
||||
hidden_size (`int`, *optional*, defaults to 768):
|
||||
Dimensionality of the encoder layers and the pooler layer.
|
||||
intermediate_size (`int`, *optional*, defaults to 3072):
|
||||
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 12):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
num_attention_heads (`int`, *optional*, defaults to 12):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
num_channels (`int`, *optional*, defaults to 3):
|
||||
Number of channels in the input images.
|
||||
image_size (`int`, *optional*, defaults to 224):
|
||||
The size (resolution) of each image.
|
||||
patch_size (`int`, *optional*, defaults to 16):
|
||||
The size (resolution) of each patch.
|
||||
hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
|
||||
The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
|
||||
`"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
|
||||
layer_norm_eps (`float`, *optional*, defaults to 1e-06):
|
||||
The epsilon used by the layer normalization layers.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
Example:
|
||||
```python
|
||||
>>> from transformers import SiglipVisionConfig, SiglipVisionModel
|
||||
>>> # Initializing a SiglipVisionConfig with google/siglip-base-patch16-224 style configuration
|
||||
>>> configuration = SiglipVisionConfig()
|
||||
>>> # Initializing a SiglipVisionModel (with random weights) from the google/siglip-base-patch16-224 style configuration
|
||||
>>> model = SiglipVisionModel(configuration)
|
||||
>>> # Accessing the model configuration
|
||||
>>> configuration = model.config
|
||||
```"""
|
||||
|
||||
model_type = "siglip_vision_model"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size=768,
|
||||
intermediate_size=3072,
|
||||
num_hidden_layers=12,
|
||||
num_attention_heads=12,
|
||||
num_channels=3,
|
||||
image_size=224,
|
||||
patch_size=16,
|
||||
hidden_act="gelu_pytorch_tanh",
|
||||
layer_norm_eps=1e-6,
|
||||
attention_dropout=0.0,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_channels = num_channels
|
||||
self.patch_size = patch_size
|
||||
self.image_size = image_size
|
||||
self.attention_dropout = attention_dropout
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
self.hidden_act = hidden_act
|
||||
|
||||
_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
|
||||
|
||||
SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
|
||||
"google/siglip-base-patch16-224",
|
||||
# See all SigLIP models at https://huggingface.co/models?filter=siglip
|
||||
]
|
||||
|
||||
# Copied from transformers.models.llama.modeling_llama._get_unpad_data
|
||||
def _get_unpad_data(attention_mask):
|
||||
seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
|
||||
indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
|
||||
max_seqlen_in_batch = seqlens_in_batch.max().item()
|
||||
cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
|
||||
return (
|
||||
indices,
|
||||
cu_seqlens,
|
||||
max_seqlen_in_batch,
|
||||
)
|
||||
|
||||
|
||||
def _trunc_normal_(tensor, mean, std, a, b):
|
||||
# Cut & paste from PyTorch official master until it's in a few official releases - RW
|
||||
# Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
|
||||
def norm_cdf(x):
|
||||
# Computes standard normal cumulative distribution function
|
||||
return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
|
||||
|
||||
if (mean < a - 2 * std) or (mean > b + 2 * std):
|
||||
warnings.warn(
|
||||
"mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
|
||||
"The distribution of values may be incorrect.",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
# Values are generated by using a truncated uniform distribution and
|
||||
# then using the inverse CDF for the normal distribution.
|
||||
# Get upper and lower cdf values
|
||||
l = norm_cdf((a - mean) / std)
|
||||
u = norm_cdf((b - mean) / std)
|
||||
|
||||
# Uniformly fill tensor with values from [l, u], then translate to
|
||||
# [2l-1, 2u-1].
|
||||
tensor.uniform_(2 * l - 1, 2 * u - 1)
|
||||
|
||||
# Use inverse cdf transform for normal distribution to get truncated
|
||||
# standard normal
|
||||
if tensor.dtype in [torch.float16, torch.bfloat16]:
|
||||
# The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
|
||||
og_dtype = tensor.dtype
|
||||
tensor = tensor.to(torch.float32)
|
||||
tensor.erfinv_()
|
||||
tensor = tensor.to(og_dtype)
|
||||
else:
|
||||
tensor.erfinv_()
|
||||
|
||||
# Transform to proper mean, std
|
||||
tensor.mul_(std * math.sqrt(2.0))
|
||||
tensor.add_(mean)
|
||||
|
||||
# Clamp to ensure it's in the proper range
|
||||
if tensor.dtype == torch.float16:
|
||||
# The `clamp_` op is not (yet?) defined in float16+cpu
|
||||
tensor = tensor.to(torch.float32)
|
||||
tensor.clamp_(min=a, max=b)
|
||||
tensor = tensor.to(torch.float16)
|
||||
else:
|
||||
tensor.clamp_(min=a, max=b)
|
||||
|
||||
|
||||
def trunc_normal_tf_(
|
||||
tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0, a: float = -2.0, b: float = 2.0
|
||||
):
|
||||
"""Fills the input Tensor with values drawn from a truncated
|
||||
normal distribution. The values are effectively drawn from the
|
||||
normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
|
||||
with values outside :math:`[a, b]` redrawn until they are within
|
||||
the bounds. The method used for generating the random values works
|
||||
best when :math:`a \\leq \text{mean} \\leq b`.
|
||||
NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
|
||||
bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
|
||||
and the result is subsequently scaled and shifted by the mean and std args.
|
||||
Args:
|
||||
tensor: an n-dimensional `torch.Tensor`
|
||||
mean: the mean of the normal distribution
|
||||
std: the standard deviation of the normal distribution
|
||||
a: the minimum cutoff value
|
||||
b: the maximum cutoff value
|
||||
"""
|
||||
with torch.no_grad():
|
||||
_trunc_normal_(tensor, 0, 1.0, a, b)
|
||||
tensor.mul_(std).add_(mean)
|
||||
|
||||
|
||||
def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
|
||||
fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
|
||||
denom = fan_in
|
||||
if mode == "fan_in":
|
||||
denom = fan_in
|
||||
elif mode == "fan_out":
|
||||
denom = fan_out
|
||||
elif mode == "fan_avg":
|
||||
denom = (fan_in + fan_out) / 2
|
||||
|
||||
variance = scale / denom
|
||||
|
||||
if distribution == "truncated_normal":
|
||||
# constant is stddev of standard normal truncated to (-2, 2)
|
||||
trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
|
||||
elif distribution == "normal":
|
||||
with torch.no_grad():
|
||||
tensor.normal_(std=math.sqrt(variance))
|
||||
elif distribution == "uniform":
|
||||
bound = math.sqrt(3 * variance)
|
||||
with torch.no_grad():
|
||||
tensor.uniform_(-bound, bound)
|
||||
else:
|
||||
raise ValueError(f"invalid distribution {distribution}")
|
||||
|
||||
|
||||
def lecun_normal_(tensor):
|
||||
variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
|
||||
|
||||
|
||||
def default_flax_embed_init(tensor):
|
||||
variance_scaling_(tensor, mode="fan_in", distribution="normal")
|
||||
|
||||
class SiglipVisionEmbeddings(nn.Module):
|
||||
def __init__(self, config: SiglipVisionConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.embed_dim = config.hidden_size
|
||||
self.image_size = config.image_size
|
||||
self.patch_size = config.patch_size
|
||||
|
||||
self.patch_embedding = nn.Conv2d(
|
||||
in_channels=config.num_channels,
|
||||
out_channels=self.embed_dim,
|
||||
kernel_size=self.patch_size,
|
||||
stride=self.patch_size,
|
||||
padding="valid",
|
||||
)
|
||||
|
||||
self.num_patches_per_side = self.image_size // self.patch_size
|
||||
self.num_patches = self.num_patches_per_side**2
|
||||
self.num_positions = self.num_patches
|
||||
self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
|
||||
|
||||
class SiglipAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
||||
|
||||
# Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.embed_dim = config.hidden_size
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.head_dim = self.embed_dim // self.num_heads
|
||||
if self.head_dim * self.num_heads != self.embed_dim:
|
||||
raise ValueError(
|
||||
f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
|
||||
f" {self.num_heads})."
|
||||
)
|
||||
self.scale = self.head_dim**-0.5
|
||||
self.dropout = config.attention_dropout
|
||||
|
||||
self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
||||
self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
||||
self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
||||
self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
|
||||
|
||||
# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
|
||||
class SiglipMLP(nn.Module):
|
||||
def __init__(self, config):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.activation_fn = ACT2FN[config.hidden_act]
|
||||
self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
|
||||
self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
|
||||
|
||||
|
||||
# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->Siglip
|
||||
class SiglipEncoderLayer(nn.Module):
|
||||
def __init__(self, config: SiglipVisionConfig):
|
||||
super().__init__()
|
||||
self.embed_dim = config.hidden_size
|
||||
self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
|
||||
self.self_attn = (
|
||||
SiglipAttention(config)
|
||||
)
|
||||
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
||||
self.mlp = SiglipMLP(config)
|
||||
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
|
||||
|
||||
class SiglipPreTrainedModel(PreTrainedModel):
|
||||
"""
|
||||
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
|
||||
models.
|
||||
"""
|
||||
|
||||
config_class = SiglipVisionConfig
|
||||
base_model_prefix = "siglip"
|
||||
supports_gradient_checkpointing = True
|
||||
|
||||
def _init_weights(self, module):
|
||||
"""Initialize the weights"""
|
||||
|
||||
if isinstance(module, SiglipVisionEmbeddings):
|
||||
width = self.config.hidden_size
|
||||
nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
|
||||
elif isinstance(module, nn.Embedding):
|
||||
default_flax_embed_init(module.weight)
|
||||
elif isinstance(module, SiglipAttention):
|
||||
nn.init.normal_(module.q_proj.weight)
|
||||
nn.init.normal_(module.k_proj.weight)
|
||||
nn.init.normal_(module.v_proj.weight)
|
||||
nn.init.normal_(module.out_proj.weight)
|
||||
nn.init.zeros_(module.q_proj.bias)
|
||||
nn.init.zeros_(module.k_proj.bias)
|
||||
nn.init.zeros_(module.v_proj.bias)
|
||||
nn.init.zeros_(module.out_proj.bias)
|
||||
elif isinstance(module, SiglipMLP):
|
||||
nn.init.normal_(module.fc1.weight)
|
||||
nn.init.normal_(module.fc2.weight)
|
||||
nn.init.normal_(module.fc1.bias, std=1e-6)
|
||||
nn.init.normal_(module.fc2.bias, std=1e-6)
|
||||
elif isinstance(module, (nn.Linear, nn.Conv2d)):
|
||||
lecun_normal_(module.weight)
|
||||
if module.bias is not None:
|
||||
nn.init.zeros_(module.bias)
|
||||
elif isinstance(module, nn.LayerNorm):
|
||||
module.bias.data.zero_()
|
||||
module.weight.data.fill_(1.0)
|
||||
|
||||
|
||||
SIGLIP_START_DOCSTRING = r"""
|
||||
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
||||
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
|
||||
etc.)
|
||||
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
||||
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
||||
and behavior.
|
||||
Parameters:
|
||||
config ([`SiglipVisionConfig`]): Model configuration class with all the parameters of the model.
|
||||
Initializing with a config file does not load the weights associated with the model, only the
|
||||
configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
|
||||
"""
|
||||
|
||||
|
||||
SIGLIP_VISION_INPUTS_DOCSTRING = r"""
|
||||
Args:
|
||||
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
|
||||
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
|
||||
[`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
|
||||
output_attentions (`bool`, *optional*):
|
||||
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
||||
tensors for more detail.
|
||||
output_hidden_states (`bool`, *optional*):
|
||||
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
||||
more detail.
|
||||
return_dict (`bool`, *optional*):
|
||||
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
||||
"""
|
||||
|
||||
|
||||
# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->Siglip
|
||||
class SiglipEncoder(nn.Module):
|
||||
"""
|
||||
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
|
||||
[`SiglipEncoderLayer`].
|
||||
Args:
|
||||
config: SiglipConfig
|
||||
"""
|
||||
|
||||
def __init__(self, config: SiglipVisionConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
|
||||
self.gradient_checkpointing = False
|
||||
|
||||
class SiglipVisionTransformer(SiglipPreTrainedModel):
|
||||
config_class = SiglipVisionConfig
|
||||
main_input_name = "pixel_values"
|
||||
_supports_flash_attn_2 = True
|
||||
|
||||
def __init__(self, config: SiglipVisionConfig):
|
||||
super().__init__(config)
|
||||
self.config = config
|
||||
embed_dim = config.hidden_size
|
||||
|
||||
self.embeddings = SiglipVisionEmbeddings(config)
|
||||
self.encoder = SiglipEncoder(config)
|
||||
self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
|
||||
self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
|
||||
|
||||
# Initialize weights and apply final processing
|
||||
self.post_init()
|
||||
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return self.embeddings.patch_embedding
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
|
||||
import numpy as np
|
||||
from gguf import *
|
||||
from transformers.models.idefics2.modeling_idefics2 import Idefics2VisionTransformer
|
||||
from transformers.models.idefics2.configuration_idefics2 import Idefics2VisionConfig
|
||||
|
||||
TEXT = "clip.text"
|
||||
VISION = "clip.vision"
|
||||
|
||||
|
||||
def add_key_str(raw_key: str, arch: str) -> str:
|
||||
return raw_key.format(arch=arch)
|
||||
|
||||
|
||||
def should_skip_tensor(name: str, has_text: bool, has_vision: bool, has_minicpmv: bool) -> bool:
|
||||
if name in (
|
||||
"logit_scale",
|
||||
"text_model.embeddings.position_ids",
|
||||
"vision_model.embeddings.position_ids",
|
||||
):
|
||||
return True
|
||||
|
||||
if has_minicpmv and name in ["visual_projection.weight"]:
|
||||
return True
|
||||
|
||||
if name.startswith("v") and not has_vision:
|
||||
return True
|
||||
|
||||
if name.startswith("t") and not has_text:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_tensor_name(name: str) -> str:
|
||||
if "projection" in name:
|
||||
return name
|
||||
if "mm_projector" in name:
|
||||
name = name.replace("model.mm_projector", "mm")
|
||||
name = re.sub(r'mm\.mlp\.mlp', 'mm.model.mlp', name, count=1)
|
||||
name = re.sub(r'mm\.peg\.peg', 'mm.model.peg', name, count=1)
|
||||
return name
|
||||
|
||||
return name.replace("text_model", "t").replace("vision_model", "v").replace("encoder.layers", "blk").replace("embeddings.", "").replace("_proj", "").replace("self_attn.", "attn_").replace("layer_norm", "ln").replace("layernorm", "ln").replace("mlp.fc1", "ffn_down").replace("mlp.fc2", "ffn_up").replace("embedding", "embd").replace("final", "post").replace("layrnorm", "ln")
|
||||
|
||||
|
||||
def bytes_to_unicode():
|
||||
"""
|
||||
Returns list of utf-8 byte and a corresponding list of unicode strings.
|
||||
The reversible bpe codes work on unicode strings.
|
||||
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
|
||||
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
|
||||
This is a significant percentage of your normal, say, 32K bpe vocab.
|
||||
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
|
||||
And avoids mapping to whitespace/control characters the bpe code barfs on.
|
||||
"""
|
||||
bs = (
|
||||
list(range(ord("!"), ord("~") + 1))
|
||||
+ list(range(ord("¡"), ord("¬") + 1))
|
||||
+ list(range(ord("®"), ord("ÿ") + 1))
|
||||
)
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(2**8):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(2**8 + n)
|
||||
n += 1
|
||||
cs = [chr(n) for n in cs]
|
||||
return dict(zip(bs, cs))
|
||||
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-m", "--model-dir", help="Path to model directory cloned from HF Hub", required=True)
|
||||
ap.add_argument("--use-f32", action="store_true", default=False, help="Use f32 instead of f16")
|
||||
ap.add_argument("--text-only", action="store_true", required=False,
|
||||
help="Save a text-only model. It can't be used to encode images")
|
||||
ap.add_argument("--vision-only", action="store_true", required=False,
|
||||
help="Save a vision-only model. It can't be used to encode texts")
|
||||
ap.add_argument("--clip-model-is-vision", action="store_true", required=False,
|
||||
help="The clip model is a pure vision model (ShareGPT4V vision extract for example)")
|
||||
ap.add_argument("--clip-model-is-openclip", action="store_true", required=False,
|
||||
help="The clip model is from openclip (for ViT-SO400M type))")
|
||||
ap.add_argument("--minicpmv-projector", help="Path to minicpmv.projector file. If specified, save an image encoder for MiniCPM-V models.")
|
||||
ap.add_argument("--projector-type", help="Type of projector. Possible values: mlp, ldp, ldpv2", choices=["mlp", "ldp", "ldpv2"], default="mlp")
|
||||
ap.add_argument("-o", "--output-dir", help="Directory to save GGUF files. Default is the original model directory", default=None)
|
||||
# Example --image_mean 0.48145466 0.4578275 0.40821073 --image_std 0.26862954 0.26130258 0.27577711
|
||||
# Example --image_mean 0.5 0.5 0.5 --image_std 0.5 0.5 0.5
|
||||
default_image_mean = [0.5, 0.5, 0.5]
|
||||
default_image_std = [0.5, 0.5, 0.5]
|
||||
ap.add_argument('--image-mean', type=float, nargs='+', help='Mean of the images for normalization (overrides processor) ', default=None)
|
||||
ap.add_argument('--image-std', type=float, nargs='+', help='Standard deviation of the images for normalization (overrides processor)', default=None)
|
||||
ap.add_argument('--minicpmv_version', type=int, help='minicpmv_version: MiniCPM-V-2 use 1; MiniCPM-V-2.5 use 2; MiniCPM-V-2.6 use 3; MiniCPM-o-2.6 use 4; MiniCPM-V 4.0 use 5; MiniCPM-o-4.0 use 6; MiniCPM-o-4.5 use 100045', default=2)
|
||||
|
||||
# with proper
|
||||
args = ap.parse_args()
|
||||
|
||||
|
||||
if args.text_only and args.vision_only:
|
||||
print("--text-only and --image-only arguments cannot be specified at the same time.")
|
||||
exit(1)
|
||||
|
||||
if args.use_f32:
|
||||
print("WARNING: Weights for the convolution op is always saved in f16, as the convolution op in GGML does not support 32-bit kernel weights yet.")
|
||||
|
||||
# output in the same directory as the model if output_dir is None
|
||||
dir_model = args.model_dir
|
||||
|
||||
# Read config.json to get actual model configuration
|
||||
config_path = os.path.join(dir_model, "config.json")
|
||||
model_config = {}
|
||||
if os.path.isfile(config_path):
|
||||
with open(config_path, "r", encoding="utf-8") as f:
|
||||
model_config = json.load(f)
|
||||
print(f"Loaded config from {config_path}")
|
||||
else:
|
||||
print(f"Warning: config.json not found at {config_path}")
|
||||
|
||||
# If minicpmv_projector is not specified but the default path exists, use the default path
|
||||
if args.minicpmv_projector is None:
|
||||
default_projector_path = os.path.join(dir_model, "minicpmv.projector")
|
||||
if os.path.isfile(default_projector_path):
|
||||
args.minicpmv_projector = default_projector_path
|
||||
print(f"Found default projector file: {default_projector_path}")
|
||||
|
||||
# If output_dir is not specified, use model_dir as the default value
|
||||
if args.output_dir is None:
|
||||
args.output_dir = dir_model
|
||||
|
||||
if args.clip_model_is_vision or not os.path.exists(dir_model + "/vocab.json") or args.clip_model_is_openclip:
|
||||
vocab = None
|
||||
tokens = None
|
||||
else:
|
||||
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
|
||||
vocab = json.load(f)
|
||||
tokens = [key for key in vocab]
|
||||
|
||||
# possible data types
|
||||
# ftype == 0 -> float32
|
||||
# ftype == 1 -> float16
|
||||
#
|
||||
# map from ftype to string
|
||||
ftype_str = ["f32", "f16"]
|
||||
|
||||
ftype = 1
|
||||
if args.use_f32:
|
||||
ftype = 0
|
||||
|
||||
# if args.clip_model_is_vision or args.clip_model_is_openclip:
|
||||
# model = CLIPVisionModel.from_pretrained(dir_model)
|
||||
# processor = None
|
||||
# else:
|
||||
# model = CLIPModel.from_pretrained(dir_model)
|
||||
# processor = CLIPProcessor.from_pretrained(dir_model)
|
||||
|
||||
minicpmv_version = args.minicpmv_version
|
||||
|
||||
# Use actual config values instead of hardcoded ones
|
||||
if model_config:
|
||||
# For the projector/resampler, use the main model's hidden_size
|
||||
emb_dim = model_config.get("hidden_size", 1536)
|
||||
|
||||
# For the vision model, use vision_config values
|
||||
vision_config_dict = model_config.get("vision_config", {})
|
||||
default_vision_config = {
|
||||
"hidden_size": vision_config_dict.get("hidden_size", 1152),
|
||||
"image_size": vision_config_dict.get("image_size", 980),
|
||||
"intermediate_size": vision_config_dict.get("intermediate_size", 4304),
|
||||
"model_type": vision_config_dict.get("model_type", "siglip"),
|
||||
"num_attention_heads": vision_config_dict.get("num_attention_heads", 16),
|
||||
"num_hidden_layers": vision_config_dict.get("num_hidden_layers", 27),
|
||||
"patch_size": vision_config_dict.get("patch_size", 14),
|
||||
}
|
||||
|
||||
# Use vision model's num_hidden_layers for block_count
|
||||
block_count = vision_config_dict.get("num_hidden_layers", 27)
|
||||
|
||||
print(f"Using config values: emb_dim={emb_dim}, block_count={block_count}")
|
||||
print(f"Vision config: {default_vision_config}")
|
||||
else:
|
||||
# Fallback to original hardcoded logic if config.json not found
|
||||
emb_dim = 4096
|
||||
block_count = 26
|
||||
if minicpmv_version == 1:
|
||||
emb_dim = 2304
|
||||
block_count = 26
|
||||
elif minicpmv_version == 2:
|
||||
emb_dim = 4096
|
||||
block_count = 27
|
||||
elif minicpmv_version == 3:
|
||||
emb_dim = 3584
|
||||
block_count = 27
|
||||
elif minicpmv_version == 4:
|
||||
emb_dim = 3584
|
||||
block_count = 27
|
||||
elif minicpmv_version == 5:
|
||||
emb_dim = 2560
|
||||
block_count = 27
|
||||
elif minicpmv_version == 6:
|
||||
emb_dim = 4096
|
||||
block_count = 27
|
||||
elif minicpmv_version == 100045:
|
||||
emb_dim = 4096
|
||||
block_count = 27
|
||||
|
||||
default_vision_config = {
|
||||
"hidden_size": 1152,
|
||||
"image_size": 980,
|
||||
"intermediate_size": 4304,
|
||||
"model_type": "idefics2",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 27,
|
||||
"patch_size": 14,
|
||||
}
|
||||
|
||||
vision_config = Idefics2VisionConfig(**default_vision_config)
|
||||
model = Idefics2VisionTransformer(vision_config)
|
||||
if minicpmv_version == 3 or (model_config and model_config.get("vision_config", {}).get("model_type") == "siglip"):
|
||||
vision_config = SiglipVisionConfig(**default_vision_config)
|
||||
model = SiglipVisionTransformer(vision_config)
|
||||
elif minicpmv_version == 4:
|
||||
vision_config = SiglipVisionConfig(**default_vision_config)
|
||||
model = SiglipVisionTransformer(vision_config)
|
||||
elif minicpmv_version == 5:
|
||||
default_vision_config["model_type"] = "siglip_vision_model"
|
||||
vision_config = SiglipVisionConfig(**default_vision_config)
|
||||
model = SiglipVisionTransformer(vision_config)
|
||||
elif minicpmv_version == 6:
|
||||
default_vision_config["model_type"] = "siglip_vision_model"
|
||||
vision_config = SiglipVisionConfig(**default_vision_config)
|
||||
model = SiglipVisionTransformer(vision_config)
|
||||
elif minicpmv_version == 100045:
|
||||
default_vision_config["model_type"] = "siglip_vision_model"
|
||||
vision_config = SiglipVisionConfig(**default_vision_config)
|
||||
model = SiglipVisionTransformer(vision_config)
|
||||
|
||||
processor = None
|
||||
# if model.attn_pool is not None:
|
||||
# model.attn_pool = torch.nn.Identity()
|
||||
|
||||
# model.blocks = model.blocks[:-1]
|
||||
model.load_state_dict(torch.load(os.path.join(dir_model, "minicpmv.clip")))
|
||||
|
||||
fname_middle = None
|
||||
has_text_encoder = True
|
||||
has_vision_encoder = True
|
||||
has_minicpmv_projector = False
|
||||
|
||||
if args.text_only:
|
||||
fname_middle = "text-"
|
||||
has_vision_encoder = False
|
||||
elif args.minicpmv_projector is not None:
|
||||
fname_middle = "mmproj-"
|
||||
has_text_encoder = False
|
||||
has_minicpmv_projector = True
|
||||
elif args.vision_only:
|
||||
fname_middle = "vision-"
|
||||
has_text_encoder = False
|
||||
else:
|
||||
fname_middle = ""
|
||||
|
||||
output_dir = args.output_dir
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_prefix = os.path.basename(output_dir).replace("ggml_", "")
|
||||
fname_out = os.path.join(output_dir, f"{fname_middle}model-{ftype_str[ftype]}.gguf")
|
||||
fout = GGUFWriter(path=fname_out, arch="clip")
|
||||
|
||||
fout.add_bool("clip.has_text_encoder", has_text_encoder)
|
||||
fout.add_bool("clip.has_vision_encoder", has_vision_encoder)
|
||||
fout.add_bool("clip.has_minicpmv_projector", has_minicpmv_projector)
|
||||
fout.add_file_type(ftype)
|
||||
if args.text_only:
|
||||
fout.add_description("text-only CLIP model")
|
||||
elif args.vision_only and not has_minicpmv_projector:
|
||||
fout.add_description("vision-only CLIP model")
|
||||
elif has_minicpmv_projector:
|
||||
fout.add_description("image encoder for MiniCPM-V")
|
||||
# add projector type
|
||||
fout.add_string("clip.projector_type", "resampler")
|
||||
fout.add_int32("clip.minicpmv_version", minicpmv_version)
|
||||
else:
|
||||
fout.add_description("two-tower CLIP model")
|
||||
|
||||
if has_vision_encoder:
|
||||
# vision_model hparams - use actual config values
|
||||
vision_image_size = model_config.get("image_size", 448) if model_config else 448
|
||||
vision_patch_size = default_vision_config.get("patch_size", 14)
|
||||
vision_hidden_size = default_vision_config.get("hidden_size", 1152)
|
||||
vision_intermediate_size = default_vision_config.get("intermediate_size", 4304)
|
||||
vision_attention_heads = default_vision_config.get("num_attention_heads", 16)
|
||||
|
||||
fout.add_uint32("clip.vision.image_size", vision_image_size)
|
||||
fout.add_uint32("clip.vision.patch_size", vision_patch_size)
|
||||
fout.add_uint32(add_key_str(KEY_EMBEDDING_LENGTH, VISION), vision_hidden_size)
|
||||
fout.add_uint32(add_key_str(KEY_FEED_FORWARD_LENGTH, VISION), vision_intermediate_size)
|
||||
fout.add_uint32("clip.vision.projection_dim", 0)
|
||||
fout.add_uint32(add_key_str(KEY_ATTENTION_HEAD_COUNT, VISION), vision_attention_heads)
|
||||
fout.add_float32(add_key_str(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
|
||||
fout.add_uint32(add_key_str(KEY_BLOCK_COUNT, VISION), block_count)
|
||||
|
||||
# Add MiniCPM-V specific parameters
|
||||
query_num = model_config.get("query_num", 0) if model_config else 0
|
||||
resampler_emb_dim = model_config.get("hidden_size", 0) if model_config else 0
|
||||
fout.add_uint32("clip.minicpmv_query_num", query_num)
|
||||
|
||||
if processor is not None:
|
||||
image_mean = processor.image_processor.image_mean if args.image_mean is None or args.image_mean == default_image_mean else args.image_mean
|
||||
image_std = processor.image_processor.image_std if args.image_std is None or args.image_std == default_image_std else args.image_std
|
||||
else:
|
||||
image_mean = args.image_mean if args.image_mean is not None else default_image_mean
|
||||
image_std = args.image_std if args.image_std is not None else default_image_std
|
||||
fout.add_array("clip.vision.image_mean", image_mean)
|
||||
fout.add_array("clip.vision.image_std", image_std)
|
||||
|
||||
use_gelu = True
|
||||
fout.add_bool("clip.use_gelu", use_gelu)
|
||||
|
||||
def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
|
||||
"""
|
||||
embed_dim: output dimension for each position
|
||||
pos: a list of positions to be encoded: size (M,)
|
||||
out: (M, D)
|
||||
"""
|
||||
assert embed_dim % 2 == 0
|
||||
omega = np.arange(embed_dim // 2, dtype=np.float32)
|
||||
omega /= embed_dim / 2.
|
||||
omega = 1. / 10000 ** omega # (D/2,)
|
||||
|
||||
pos = pos.reshape(-1) # (M,)
|
||||
out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
|
||||
|
||||
emb_sin = np.sin(out) # (M, D/2)
|
||||
emb_cos = np.cos(out) # (M, D/2)
|
||||
|
||||
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
|
||||
return emb
|
||||
|
||||
def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
|
||||
assert embed_dim % 2 == 0
|
||||
|
||||
# use half of dimensions to encode grid_h
|
||||
emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2)
|
||||
emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2)
|
||||
|
||||
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
|
||||
return emb
|
||||
|
||||
|
||||
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
|
||||
def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
|
||||
"""
|
||||
grid_size: int of the grid height and width
|
||||
return:
|
||||
pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
|
||||
"""
|
||||
if isinstance(grid_size, int):
|
||||
grid_h_size, grid_w_size = grid_size, grid_size
|
||||
else:
|
||||
grid_h_size, grid_w_size = grid_size[0], grid_size[1]
|
||||
|
||||
grid_h = np.arange(grid_h_size, dtype=np.float32)
|
||||
grid_w = np.arange(grid_w_size, dtype=np.float32)
|
||||
grid = np.meshgrid(grid_w, grid_h) # here w goes first
|
||||
grid = np.stack(grid, axis=0)
|
||||
|
||||
grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
|
||||
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
|
||||
if cls_token:
|
||||
pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
|
||||
return pos_embed
|
||||
|
||||
def _replace_name_resampler(s, v):
|
||||
if re.match("resampler.pos_embed", s):
|
||||
return {
|
||||
s: v,
|
||||
re.sub("pos_embed", "pos_embed_k", s): torch.from_numpy(get_2d_sincos_pos_embed(emb_dim, (70, 70))),
|
||||
}
|
||||
if re.match("resampler.proj", s):
|
||||
return {
|
||||
re.sub("proj", "pos_embed_k", s): torch.from_numpy(get_2d_sincos_pos_embed(emb_dim, (70, 70))),
|
||||
re.sub("proj", "proj.weight", s): v.transpose(-1, -2).contiguous(),
|
||||
}
|
||||
if re.match("resampler.attn.in_proj_.*", s):
|
||||
return {
|
||||
re.sub("attn.in_proj_", "attn.q.", s): v.chunk(3, dim=0)[0],
|
||||
re.sub("attn.in_proj_", "attn.k.", s): v.chunk(3, dim=0)[1],
|
||||
re.sub("attn.in_proj_", "attn.v.", s): v.chunk(3, dim=0)[2],
|
||||
}
|
||||
return {s: v}
|
||||
|
||||
if has_minicpmv_projector:
|
||||
projector = torch.load(args.minicpmv_projector)
|
||||
new_state_dict = {}
|
||||
for k, v in projector.items():
|
||||
kvs = _replace_name_resampler(k, v)
|
||||
for nk, nv in kvs.items():
|
||||
new_state_dict[nk] = nv
|
||||
projector = new_state_dict
|
||||
ftype_cur = 0
|
||||
for name, data in projector.items():
|
||||
name = get_tensor_name(name)
|
||||
data = data.squeeze().numpy()
|
||||
|
||||
n_dims = len(data.shape)
|
||||
if ftype == 1:
|
||||
if name[-7:] == ".weight" and n_dims == 2:
|
||||
print(" Converting to float16")
|
||||
data = data.astype(np.float16)
|
||||
ftype_cur = 1
|
||||
else:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
else:
|
||||
if data.dtype != np.float32:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
|
||||
fout.add_tensor(name, data)
|
||||
print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
|
||||
|
||||
print("Projector tensors added\n")
|
||||
|
||||
def _replace_name(s, v):
|
||||
s = "vision_model." + s
|
||||
if re.match("vision_model.embeddings.position_embedding", s):
|
||||
v = v.unsqueeze(0)
|
||||
return {s: v}
|
||||
|
||||
return {s: v}
|
||||
|
||||
state_dict = model.state_dict()
|
||||
new_state_dict = {}
|
||||
for k, v in state_dict.items():
|
||||
kvs = _replace_name(k, v)
|
||||
for nk, nv in kvs.items():
|
||||
new_state_dict[nk] = nv
|
||||
state_dict = new_state_dict
|
||||
for name, data in state_dict.items():
|
||||
if should_skip_tensor(name, has_text_encoder, has_vision_encoder, has_minicpmv_projector):
|
||||
# we don't need this
|
||||
print(f"skipping parameter: {name}")
|
||||
continue
|
||||
|
||||
name = get_tensor_name(name)
|
||||
data = data.squeeze().numpy()
|
||||
|
||||
n_dims = len(data.shape)
|
||||
|
||||
# ftype == 0 -> float32, ftype == 1 -> float16
|
||||
ftype_cur = 0
|
||||
if n_dims == 4:
|
||||
print(f"tensor {name} is always saved in f16")
|
||||
data = data.astype(np.float16)
|
||||
ftype_cur = 1
|
||||
elif ftype == 1:
|
||||
if name[-7:] == ".weight" and n_dims == 2:
|
||||
print(" Converting to float16")
|
||||
data = data.astype(np.float16)
|
||||
ftype_cur = 1
|
||||
else:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
else:
|
||||
if data.dtype != np.float32:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
|
||||
print(f"{name} - {ftype_str[ftype_cur]} - shape = {data.shape}")
|
||||
fout.add_tensor(name, data)
|
||||
|
||||
|
||||
fout.write_header_to_file()
|
||||
fout.write_kv_data_to_file()
|
||||
fout.write_tensors_to_file()
|
||||
fout.close()
|
||||
|
||||
print("Done. Output file: " + fname_out)
|
||||
@@ -0,0 +1,47 @@
|
||||
import argparse
|
||||
import os
|
||||
import torch
|
||||
from transformers import AutoModel, AutoTokenizer
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-m", "--model", help="Path to MiniCPM-V model")
|
||||
args = ap.parse_args()
|
||||
|
||||
# find the model part that includes the the multimodal projector weights
|
||||
model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True, torch_dtype=torch.bfloat16)
|
||||
checkpoint = model.state_dict()
|
||||
|
||||
# get a list of mm tensor names
|
||||
mm_tensors = [k for k, v in checkpoint.items() if k.startswith("resampler")]
|
||||
|
||||
# store these tensors in a new dictionary and torch.save them
|
||||
projector = {name: checkpoint[name].float() for name in mm_tensors}
|
||||
if 'resampler.proj' in projector.keys() and hasattr(model.llm.config,'scale_emb') is True:
|
||||
projector['resampler.proj'] = projector['resampler.proj'] / model.llm.config.scale_emb
|
||||
torch.save(projector, f"{args.model}/minicpmv.projector")
|
||||
|
||||
clip_tensors = [k for k, v in checkpoint.items() if k.startswith("vpm")]
|
||||
if len(clip_tensors) > 0:
|
||||
clip = {name.replace("vpm.", ""): checkpoint[name].float() for name in clip_tensors}
|
||||
torch.save(clip, f"{args.model}/minicpmv.clip")
|
||||
|
||||
# added tokens should be removed to be able to convert Mistral models
|
||||
if os.path.exists(f"{args.model}/added_tokens.json"):
|
||||
with open(f"{args.model}/added_tokens.json", "w") as f:
|
||||
f.write("{}\n")
|
||||
|
||||
config = model.llm.config
|
||||
config.auto_map = {
|
||||
"AutoConfig": "configuration_minicpm.MiniCPMConfig",
|
||||
"AutoModel": "modeling_minicpm.MiniCPMModel",
|
||||
"AutoModelForCausalLM": "modeling_minicpm.MiniCPMForCausalLM",
|
||||
"AutoModelForSeq2SeqLM": "modeling_minicpm.MiniCPMForCausalLM",
|
||||
"AutoModelForSequenceClassification": "modeling_minicpm.MiniCPMForSequenceClassification"
|
||||
}
|
||||
model.llm.save_pretrained(f"{args.model}/model")
|
||||
tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
|
||||
tok.save_pretrained(f"{args.model}/model")
|
||||
|
||||
print("Done!")
|
||||
print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
|
||||
print(f"Also, use {args.model}/minicpmv.projector to prepare a minicpmv-encoder.gguf file.")
|
||||
@@ -0,0 +1,98 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_cogvlm::build() {
|
||||
GGML_ASSERT(model.class_embedding != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
|
||||
const int n_pos = n_patches + 1; // +1 for [CLS]
|
||||
|
||||
// build input and concatenate class embedding
|
||||
ggml_tensor * inp = build_inp();
|
||||
inp = ggml_concat(ctx0, inp, model.class_embedding, 1);
|
||||
|
||||
inp = ggml_add(ctx0, inp, model.position_embeddings);
|
||||
cb(inp, "inp_pos", -1);
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
auto & layer = model.layers[il];
|
||||
ggml_tensor * cur = inpL;
|
||||
|
||||
cur = build_mm(layer.qkv_w, cur);
|
||||
|
||||
cur = ggml_add(ctx0, cur, layer.qkv_b);
|
||||
|
||||
ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos, d_head*sizeof(float),
|
||||
cur->nb[1], 0);
|
||||
ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos, d_head*sizeof(float),
|
||||
cur->nb[1], n_embd * sizeof(float));
|
||||
ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos, d_head*sizeof(float),
|
||||
cur->nb[1], 2 * n_embd * sizeof(float));
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b,
|
||||
Qcur, Kcur, Vcur, nullptr, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "attn_post_norm", il);
|
||||
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
inpL = cur;
|
||||
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "ffn_post_norm", il);
|
||||
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
cb(cur, "layer_out", il);
|
||||
inpL = cur;
|
||||
|
||||
}
|
||||
|
||||
// remove CLS token (like build_llama4 does)
|
||||
ggml_tensor * cur = ggml_view_2d(ctx0, inpL,
|
||||
n_embd, n_patches,
|
||||
ggml_row_size(inpL->type, n_embd), 0);
|
||||
|
||||
// Multiply with mm_model_proj
|
||||
cur = build_mm(model.mm_model_proj, cur);
|
||||
|
||||
// Apply layernorm, weight, bias
|
||||
cur = build_norm(cur, model.mm_post_fc_norm_w, model.mm_post_fc_norm_b, NORM_TYPE_NORMAL, 1e-5, -1);
|
||||
|
||||
// Apply GELU
|
||||
cur = ggml_gelu_inplace(ctx0, cur);
|
||||
|
||||
// Branch 1: multiply with mm_h_to_4h_w
|
||||
ggml_tensor * h_to_4h = build_mm(model.mm_h_to_4h_w, cur);
|
||||
|
||||
// Branch 2: multiply with mm_gate_w
|
||||
ggml_tensor * gate = build_mm(model.mm_gate_w, cur);
|
||||
|
||||
// Apply silu
|
||||
gate = ggml_swiglu_split(ctx0, gate, h_to_4h);
|
||||
|
||||
// Apply mm_4h_to_h_w
|
||||
cur = build_mm(model.mm_4h_to_h_w, gate);
|
||||
|
||||
// Concatenate with boi and eoi
|
||||
cur = ggml_concat(ctx0, model.mm_boi, cur, 1);
|
||||
cur = ggml_concat(ctx0, cur, model.mm_eoi, 1);
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_conformer::build() {
|
||||
const int n_frames = img.nx();
|
||||
const int n_pos = n_frames / 2;
|
||||
const int n_pos_embd = (((((n_frames + 1) / 2) + 1) / 2 + 1) / 2) * 2 - 1;
|
||||
GGML_ASSERT(!model.position_embeddings || model.position_embeddings->ne[1] >= n_pos);
|
||||
|
||||
ggml_tensor * pos_emb = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 512, n_pos_embd);
|
||||
ggml_set_name(pos_emb, "pos_emb");
|
||||
ggml_set_input(pos_emb);
|
||||
ggml_build_forward_expand(gf, pos_emb);
|
||||
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
|
||||
auto * cur = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
|
||||
// pre encode, conv subsampling
|
||||
{
|
||||
// layer.0 - conv2d
|
||||
cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[0], cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[0]);
|
||||
cb(cur, "conformer.pre_encode.conv.{}", 0);
|
||||
|
||||
// layer.1 - relu
|
||||
cur = ggml_relu_inplace(ctx0, cur);
|
||||
|
||||
// layer.2 conv2d dw
|
||||
cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[2], cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[2]);
|
||||
cb(cur, "conformer.pre_encode.conv.{}", 2);
|
||||
|
||||
// layer.3 conv2d
|
||||
cur = ggml_conv_2d_direct(ctx0, model.pre_encode_conv_X_w[3], cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[3]);
|
||||
cb(cur, "conformer.pre_encode.conv.{}", 3);
|
||||
|
||||
// layer.4 - relu
|
||||
cur = ggml_relu_inplace(ctx0, cur);
|
||||
|
||||
// layer.5 conv2d dw
|
||||
cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[5], cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[5]);
|
||||
cb(cur, "conformer.pre_encode.conv.{}", 5);
|
||||
|
||||
// layer.6 conv2d
|
||||
cur = ggml_conv_2d_direct(ctx0, model.pre_encode_conv_X_w[6], cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[6]);
|
||||
cb(cur, "conformer.pre_encode.conv.{}", 6);
|
||||
|
||||
// layer.7 - relu
|
||||
cur = ggml_relu_inplace(ctx0, cur);
|
||||
|
||||
// flatten channel and frequency axis
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));
|
||||
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2]);
|
||||
|
||||
// calculate out
|
||||
cur = build_mm(model.pre_encode_out_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_out_b);
|
||||
cb(cur, "conformer.pre_encode.out", -1);
|
||||
}
|
||||
|
||||
// pos_emb
|
||||
cb(pos_emb, "pos_emb", -1);
|
||||
|
||||
for (int il = 0; il < hparams.n_layer; il++) {
|
||||
const auto & layer = model.layers[il];
|
||||
|
||||
auto * residual = cur;
|
||||
|
||||
cb(cur, "layer.in", il);
|
||||
|
||||
// feed_forward1
|
||||
cur = build_norm(cur, layer.ff_norm_w, layer.ff_norm_b, NORM_TYPE_NORMAL, 1e-5, il);
|
||||
cb(cur, "conformer.layers.{}.norm_feed_forward1", il);
|
||||
|
||||
cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b, FFN_SILU,
|
||||
il);
|
||||
cb(cur, "conformer.layers.{}.feed_forward1.linear2", il);
|
||||
|
||||
const auto fc_factor = 0.5f;
|
||||
residual = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, fc_factor));
|
||||
|
||||
// self-attention
|
||||
{
|
||||
cur = build_norm(residual, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, 1e-5, il);
|
||||
cb(cur, "conformer.layers.{}.norm_self_att", il);
|
||||
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
Qcur = ggml_add(ctx0, Qcur, layer.q_b);
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, Qcur->ne[1]);
|
||||
ggml_tensor * Q_bias_u = ggml_add(ctx0, Qcur, layer.pos_bias_u);
|
||||
Q_bias_u = ggml_permute(ctx0, Q_bias_u, 0, 2, 1, 3);
|
||||
ggml_tensor * Q_bias_v = ggml_add(ctx0, Qcur, layer.pos_bias_v);
|
||||
Q_bias_v = ggml_permute(ctx0, Q_bias_v, 0, 2, 1, 3);
|
||||
|
||||
// TODO @ngxson : some cont can/should be removed when ggml_mul_mat support these cases
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
Kcur = ggml_add(ctx0, Kcur, layer.k_b);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, Kcur->ne[1]);
|
||||
Kcur = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
|
||||
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
Vcur = ggml_add(ctx0, Vcur, layer.v_b);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, Vcur->ne[1]);
|
||||
Vcur = ggml_cont(ctx0, ggml_permute(ctx0, Vcur, 1, 2, 0, 3));
|
||||
|
||||
// build_attn won't fit due to matrix_ac and matrix_bd separation
|
||||
ggml_tensor * matrix_ac = ggml_mul_mat(ctx0, Q_bias_u, Kcur);
|
||||
matrix_ac = ggml_cont(ctx0, ggml_permute(ctx0, matrix_ac, 1, 0, 2, 3));
|
||||
cb(matrix_ac, "conformer.layers.{}.self_attn.id3", il);
|
||||
|
||||
auto * p = build_mm(layer.linear_pos_w, pos_emb);
|
||||
cb(p, "conformer.layers.{}.self_attn.linear_pos", il);
|
||||
p = ggml_reshape_3d(ctx0, p, d_head, n_head, p->ne[1]);
|
||||
p = ggml_permute(ctx0, p, 0, 2, 1, 3);
|
||||
|
||||
auto * matrix_bd = ggml_mul_mat(ctx0, Q_bias_v, p);
|
||||
matrix_bd = ggml_cont(ctx0, ggml_permute(ctx0, matrix_bd, 1, 0, 2, 3));
|
||||
|
||||
// rel shift
|
||||
{
|
||||
const auto pos_len = matrix_bd->ne[0];
|
||||
const auto q_len = matrix_bd->ne[1];
|
||||
const auto h = matrix_bd->ne[2];
|
||||
matrix_bd = ggml_pad_ext(ctx0, matrix_bd, 1, 0, 0, 0, 0, 0, 0, 0);
|
||||
matrix_bd = ggml_reshape_3d(ctx0, matrix_bd, q_len, pos_len + 1, h);
|
||||
matrix_bd = ggml_view_3d(ctx0, matrix_bd, q_len, pos_len, h, matrix_bd->nb[1],
|
||||
matrix_bd->nb[2], matrix_bd->nb[0] * q_len);
|
||||
matrix_bd = ggml_cont_3d(ctx0, matrix_bd, pos_len, q_len, h);
|
||||
}
|
||||
|
||||
matrix_bd = ggml_view_3d(ctx0, matrix_bd, matrix_ac->ne[0], matrix_bd->ne[1],
|
||||
matrix_bd->ne[2], matrix_bd->nb[1], matrix_bd->nb[2], 0);
|
||||
auto * scores = ggml_add(ctx0, matrix_ac, matrix_bd);
|
||||
scores = ggml_scale(ctx0, scores, 1.0f / std::sqrt(d_head));
|
||||
cb(scores, "conformer.layers.{}.self_attn.id0", il);
|
||||
|
||||
ggml_tensor * attn = ggml_soft_max(ctx0, scores);
|
||||
ggml_tensor * x = ggml_mul_mat(ctx0, attn, Vcur);
|
||||
x = ggml_permute(ctx0, x, 2, 0, 1, 3);
|
||||
x = ggml_cont_2d(ctx0, x, x->ne[0] * x->ne[1], x->ne[2]);
|
||||
|
||||
ggml_tensor * out = build_mm(layer.o_w, x);
|
||||
out = ggml_add(ctx0, out, layer.o_b);
|
||||
cb(out, "conformer.layers.{}.self_attn.linear_out", il);
|
||||
|
||||
cur = out;
|
||||
}
|
||||
|
||||
residual = ggml_add(ctx0, residual, cur);
|
||||
cur = build_norm(residual, layer.norm_conv_w, layer.norm_conv_b, NORM_TYPE_NORMAL, 1e-5, il);
|
||||
cb(cur, "conformer.layers.{}.norm_conv", il);
|
||||
|
||||
// conv
|
||||
{
|
||||
auto * x = cur;
|
||||
x = build_mm(layer.conv_pw1_w, x);
|
||||
x = ggml_add(ctx0, x, layer.conv_pw1_b);
|
||||
cb(x, "conformer.layers.{}.conv.pointwise_conv1", il);
|
||||
|
||||
// ggml_glu doesn't support sigmoid
|
||||
// TODO @ngxson : support this ops in ggml
|
||||
{
|
||||
int64_t d = x->ne[0] / 2;
|
||||
ggml_tensor * gate = ggml_sigmoid(ctx0, ggml_cont(ctx0, ggml_view_2d(ctx0, x, d, x->ne[1], x->nb[1], d * x->nb[0])));
|
||||
x = ggml_mul(ctx0, ggml_view_2d(ctx0, x, d, x->ne[1], x->nb[1], 0), gate);
|
||||
x = ggml_cont(ctx0, ggml_transpose(ctx0, x));
|
||||
}
|
||||
|
||||
// use ggml_ssm_conv for f32 precision
|
||||
x = ggml_pad(ctx0, x, 4, 0, 0, 0);
|
||||
x = ggml_roll(ctx0, x, 4, 0, 0, 0);
|
||||
x = ggml_pad(ctx0, x, 4, 0, 0, 0);
|
||||
x = ggml_ssm_conv(ctx0, x, layer.conv_dw_w);
|
||||
x = ggml_add(ctx0, x, layer.conv_dw_b);
|
||||
|
||||
x = ggml_add(ctx0, ggml_mul(ctx0, x, layer.conv_norm_w), layer.conv_norm_b);
|
||||
x = ggml_silu(ctx0, x);
|
||||
|
||||
// pointwise_conv2
|
||||
x = build_mm(layer.conv_pw2_w, x);
|
||||
x = ggml_add(ctx0, x, layer.conv_pw2_b);
|
||||
|
||||
cur = x;
|
||||
}
|
||||
|
||||
residual = ggml_add(ctx0, residual, cur);
|
||||
|
||||
cur = build_norm(residual, layer.ff_norm_1_w, layer.ff_norm_1_b, NORM_TYPE_NORMAL, 1e-5, il);
|
||||
cb(cur, "conformer.layers.{}.norm_feed_forward2", il);
|
||||
|
||||
cur = build_ffn(cur, layer.ff_up_1_w, layer.ff_up_1_b, nullptr, nullptr, layer.ff_down_1_w, layer.ff_down_1_b,
|
||||
FFN_SILU, il); // TODO(tarek): read activation for ffn from hparams
|
||||
cb(cur, "conformer.layers.{}.feed_forward2.linear2", il);
|
||||
|
||||
residual = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, fc_factor));
|
||||
cb(residual, "conformer.layers.{}.conv.id", il);
|
||||
|
||||
cur = build_norm(residual, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, 1e-5, il);
|
||||
cb(cur, "conformer.layers.{}.norm_out", il);
|
||||
}
|
||||
|
||||
// audio adapter
|
||||
cur = build_norm(cur, model.mm_0_w, model.mm_0_b, NORM_TYPE_NORMAL, 1e-5, -1);
|
||||
cb(cur, "audio_adapter.model.{}", 0);
|
||||
cur = build_ffn(cur, model.mm_1_w, model.mm_1_b, nullptr, nullptr, model.mm_3_w, model.mm_3_b, FFN_GELU_ERF, -1);
|
||||
|
||||
// d1omni_a: residual block after the projector
|
||||
if (model.mm_4_w) {
|
||||
ggml_tensor * x = build_norm(cur, model.mm_4_w, model.mm_4_b, NORM_TYPE_NORMAL, 1e-5, -1);
|
||||
x = build_ffn(x, model.mm_5_w, model.mm_5_b, nullptr, nullptr, model.mm_6_w, model.mm_6_b, FFN_GELU_ERF, -1);
|
||||
cur = ggml_add(ctx0, cur, x);
|
||||
}
|
||||
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
#include "models.h"
|
||||
|
||||
// DeepSeek-V4-Flash-Vision encoder (deepseek4v)
|
||||
//
|
||||
// native-resolution ViT (RMSNorm, SwiGLU, 2D RoPE, no CLS / learned pos-embd)
|
||||
// then the "aligner": 3x3 patch merge (torch.nn.functional.unfold) + 2-layer GELU MLP
|
||||
//
|
||||
// the graph outputs the complete LLM token block, built from the aligner output and 4 learned sentinel embeddings:
|
||||
//
|
||||
// [PAD]*lead_pad [START] <interleaved rows> [PAD]*pad_last [END]
|
||||
//
|
||||
// each aligner row ends with a NEWLINE, an odd row count is padded with a full row of PADs
|
||||
// pairs of adjacent rows are interleaved column-wise ("N-layout")
|
||||
// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp)
|
||||
//
|
||||
// ref: inference/vision.py and inference/image_processor.py in the HF repo
|
||||
//
|
||||
// DeepSeek-V4.1-Flash (deepseek41v) shares the encoder and aligner; its block is plain reading order
|
||||
// [START] ([IMAGE]*n_llm_w [NEWLINE])*n_llm_h [END], no lead pads, no pad row, no IMAGE_PAD tensor
|
||||
|
||||
ggml_cgraph * clip_graph_deepseek4v::build() {
|
||||
const int n_merge = hparams.n_merge;
|
||||
|
||||
// 2D input positions
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
int sections[4] = {d_head/4, d_head/4, 0, 0};
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return ggml_rope_multi(ctx0, cur, positions, nullptr,
|
||||
d_head/2, sections, GGML_ROPE_TYPE_VISION,
|
||||
0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
};
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
NORM_TYPE_RMS,
|
||||
hparams.ffn_op,
|
||||
nullptr, // no learned pos embd
|
||||
add_pos);
|
||||
cb(cur, "vit_out", -1);
|
||||
|
||||
// aligner patch merge: zero-pad the patch grid to a multiple of n_merge
|
||||
// then F.unfold == im2col with a dummy kernel (same trick as pixtral)
|
||||
{
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_embd, n_patches_x, n_patches_y);
|
||||
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); // [x, y, n_embd]
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
const int pad_x = (n_merge - n_patches_x % n_merge) % n_merge;
|
||||
const int pad_y = (n_merge - n_patches_y % n_merge) % n_merge;
|
||||
if (pad_x || pad_y) {
|
||||
cur = ggml_pad(ctx0, cur, pad_x, pad_y, 0, 0);
|
||||
}
|
||||
|
||||
// contiguous kernel view: CUDA rejects a zero-stride one and im2col falls back to CPU
|
||||
const size_t esz = ggml_element_size(cur);
|
||||
ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2],
|
||||
n_merge*esz, n_merge*n_merge*esz, 0);
|
||||
cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type);
|
||||
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
|
||||
|
||||
// aligner MLP (F.gelu in the reference == erf-based gelu)
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU_ERF,
|
||||
-1);
|
||||
cb(cur, "aligner_out", -1);
|
||||
}
|
||||
|
||||
// assemble the token block: append the sentinel embeddings as extra rows
|
||||
// then reorder everything with the precomputed layout index
|
||||
{
|
||||
const int64_t n_embd_out = cur->ne[0];
|
||||
const int64_t n_grid = cur->ne[1]; // n_llm_w * n_llm_h
|
||||
|
||||
// rows n_grid + 0..3, keep in sync with the index computation in set_input
|
||||
const bool is_v41 = proj_type == PROJECTOR_TYPE_DEEPSEEK41V;
|
||||
ggml_tensor * sentinels[] = {
|
||||
model.token_embd_img_start,
|
||||
model.token_embd_img_end,
|
||||
model.image_newline,
|
||||
is_v41 ? nullptr : model.token_embd_img_pad,
|
||||
};
|
||||
for (ggml_tensor * tok : sentinels) {
|
||||
if (tok) {
|
||||
cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1);
|
||||
}
|
||||
}
|
||||
|
||||
const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;
|
||||
const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;
|
||||
const int n_out = is_v41 ? dsv41_n_block_tokens(n_llm_w, n_llm_h)
|
||||
: dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;
|
||||
GGML_ASSERT(n_grid == n_llm_w * n_llm_h);
|
||||
|
||||
ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
|
||||
ggml_set_name(layout_idx, "layout_idx");
|
||||
ggml_set_input(layout_idx);
|
||||
|
||||
cur = ggml_get_rows(ctx0, cur, layout_idx);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
#include "models.h"
|
||||
|
||||
// Implementation based on approach suggested by Acly
|
||||
// See: https://github.com/ggml-org/llama.cpp/pull/17383#issuecomment-3554227091
|
||||
static ggml_tensor * window_partition(ggml_context * ctx0, ggml_tensor * x, const int window) {
|
||||
auto [c, w, h, b] = x->ne;
|
||||
// same as
|
||||
// x = ggml_win_part(m, x, window);
|
||||
// x = ggml_reshape_3d(m, x, c, window * window, x->ne[3]);
|
||||
|
||||
const int64_t px = (window - w % window) % window;
|
||||
const int64_t py = (window - h % window) % window;
|
||||
const int64_t npw = (w + px) / window;
|
||||
const int64_t nph = (h + py) / window;
|
||||
|
||||
ggml_tensor * cur = x;
|
||||
if (px > 0 || py > 0) {
|
||||
cur = ggml_pad(ctx0, cur, 0, static_cast<int>(px), static_cast<int>(py), 0);
|
||||
}
|
||||
cur = ggml_reshape_4d(ctx0, cur, c * window, npw, window, nph * b);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));
|
||||
cur = ggml_reshape_4d(ctx0, cur, c, window, window, npw * nph * b);
|
||||
return cur;
|
||||
}
|
||||
|
||||
// Implementation based on approach suggested by Acly
|
||||
// See: https://github.com/ggml-org/llama.cpp/pull/17383#issuecomment-3554227091
|
||||
static ggml_tensor * window_unpartition(ggml_context * ctx0,
|
||||
ggml_tensor * x,
|
||||
const int w,
|
||||
const int h,
|
||||
const int window) {
|
||||
const int64_t c = x->ne[0];
|
||||
// same as
|
||||
// x = ggml_reshape_4d(m, x, c, window, window, x->ne[2]);
|
||||
// x = ggml_win_unpart(m, x, w, h, window);
|
||||
|
||||
const int64_t px = (window - w % window) % window;
|
||||
const int64_t py = (window - h % window) % window;
|
||||
const int64_t npw = (w + px) / window;
|
||||
const int64_t nph = (h + py) / window;
|
||||
|
||||
const int64_t b = x->ne[3] / (npw * nph);
|
||||
ggml_tensor * cur = x;
|
||||
cur = ggml_reshape_4d(ctx0, cur, c * window, window, npw, nph * b);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));
|
||||
cur = ggml_reshape_4d(ctx0, cur, c, w + px, h + py, b);
|
||||
cur = ggml_view_4d(ctx0, cur, cur->ne[0], w, h, cur->ne[3], cur->nb[1], cur->nb[2], cur->nb[3], 0);
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
return cur;
|
||||
}
|
||||
|
||||
static ggml_tensor * get_rel_pos(ggml_context * ctx0,
|
||||
ggml_tensor * rel_pos, // [L, C]
|
||||
ggml_tensor * indices, // [q_size, k_size]
|
||||
const int q_size,
|
||||
const int k_size) {
|
||||
const int64_t C = rel_pos->ne[0]; // channels
|
||||
const int64_t L = rel_pos->ne[1]; // length
|
||||
|
||||
GGML_ASSERT(indices != nullptr);
|
||||
GGML_ASSERT(indices->type == GGML_TYPE_I32);
|
||||
GGML_ASSERT(indices->ne[0] == k_size);
|
||||
GGML_ASSERT(indices->ne[1] == q_size);
|
||||
|
||||
const auto max_rel_dist = 2 * std::max(q_size, k_size) - 1;
|
||||
ggml_tensor * cur = rel_pos;
|
||||
|
||||
if (max_rel_dist != L) {
|
||||
// Linear interpolation
|
||||
const int64_t ne0 = cur->ne[0];
|
||||
const int64_t ne1 = cur->ne[1];
|
||||
const int64_t ne2 = cur->ne[2];
|
||||
const int64_t ne3 = cur->ne[3];
|
||||
|
||||
cur = ggml_reshape_3d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 0, 2, 3)), ne1, 1, ne0 * ne2 * ne3);
|
||||
cur = ggml_reshape_4d(
|
||||
ctx0, ggml_interpolate(ctx0, cur, max_rel_dist, 1, ne0 * ne2 * ne3, 1, GGML_SCALE_MODE_BILINEAR),
|
||||
max_rel_dist, ne0, ne2, ne3);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 0, 2, 3));
|
||||
}
|
||||
|
||||
// Flatten indices to 1D for ggml_get_rows
|
||||
const int qk = q_size * k_size;
|
||||
|
||||
cur = ggml_reshape_3d(ctx0, ggml_get_rows(ctx0, cur, ggml_reshape_1d(ctx0, indices, qk)), C, k_size, q_size);
|
||||
|
||||
return cur; // [C, k_size, q_size]
|
||||
}
|
||||
|
||||
// ggml_conv_2d with the im2col kept in F32: the F16 im2col it emits since #23660 degrades OCR
|
||||
static ggml_tensor * conv_2d_f32(ggml_context * ctx0, ggml_tensor * a, ggml_tensor * b,
|
||||
int s0, int s1, int p0, int p1, int d0, int d1) {
|
||||
const ggml_type im2col_type = a->type == GGML_TYPE_F16 ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
ggml_tensor * im2col = ggml_im2col(ctx0, a, b, s0, s1, p0, p1, d0, d1, true, im2col_type); // [N, OH, OW, IC * KH * KW]
|
||||
|
||||
ggml_tensor * result = ggml_mul_mat(ctx0,
|
||||
ggml_reshape_2d(ctx0, im2col, im2col->ne[0], im2col->ne[3] * im2col->ne[2] * im2col->ne[1]),
|
||||
ggml_reshape_2d(ctx0, a, (a->ne[0] * a->ne[1] * a->ne[2]), a->ne[3]));
|
||||
|
||||
result = ggml_reshape_4d(ctx0, result, im2col->ne[1], im2col->ne[2], im2col->ne[3], a->ne[3]); // [OC, N, OH, OW]
|
||||
result = ggml_cont(ctx0, ggml_permute(ctx0, result, 0, 1, 3, 2)); // [N, OC, OH, OW]
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
ggml_tensor * clip_graph_deepseekocr::build_sam(ggml_tensor * inp_raw) {
|
||||
// Building SAM
|
||||
const int n_embd = hparams.sam_n_embd;
|
||||
const int n_layer = hparams.sam_n_layer;
|
||||
const int n_heads = hparams.sam_n_head;
|
||||
const int d_heads = n_embd / n_heads;
|
||||
const int window = hparams.attn_window_size;
|
||||
// SAM stage runs its layernorms at 1e-6
|
||||
const float sam_eps = 1e-6f;
|
||||
|
||||
ggml_tensor * inpL;
|
||||
|
||||
inpL = conv_2d_f32(ctx0, model.patch_embed_proj_w, inp_raw,
|
||||
(int) model.patch_embed_proj_w->ne[0], (int) model.patch_embed_proj_w->ne[1], 0, 0, 1, 1);
|
||||
inpL = ggml_add(ctx0, inpL, ggml_reshape_3d(ctx0, model.patch_embed_proj_b, 1, 1, n_embd));
|
||||
inpL = ggml_cont(ctx0, ggml_permute(ctx0, inpL, 1, 2, 0, 3));
|
||||
|
||||
ggml_tensor * rel_pos_indices_local;
|
||||
ggml_tensor * rel_pos_indices_global;
|
||||
|
||||
rel_pos_indices_local = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, window, window);
|
||||
rel_pos_indices_global = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, inpL->ne[1], inpL->ne[2]);
|
||||
ggml_set_name(rel_pos_indices_local, "rel_pos_indices_local");
|
||||
ggml_set_name(rel_pos_indices_global, "rel_pos_indices_global");
|
||||
ggml_set_input(rel_pos_indices_local);
|
||||
ggml_set_input(rel_pos_indices_global);
|
||||
|
||||
ggml_tensor * cur;
|
||||
const auto tgt_size = inpL->ne[1];
|
||||
const auto str_size = model.pos_embed->ne[1];
|
||||
|
||||
if (str_size != tgt_size) {
|
||||
ggml_tensor * old_pos_embed = nullptr;
|
||||
old_pos_embed = ggml_cont(ctx0, ggml_permute(ctx0, model.pos_embed, 2, 0, 1, 3));
|
||||
ggml_tensor * new_pos_embed =
|
||||
ggml_interpolate(ctx0, old_pos_embed, tgt_size, tgt_size, n_embd, 1, GGML_SCALE_MODE_BICUBIC);
|
||||
new_pos_embed = ggml_cont(ctx0, ggml_permute(ctx0, new_pos_embed, 1, 2, 0, 3));
|
||||
cur = ggml_add(ctx0, inpL, new_pos_embed);
|
||||
} else {
|
||||
cur = ggml_add(ctx0, inpL, model.pos_embed);
|
||||
}
|
||||
|
||||
// loop over layers
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
auto & layer = model.sam_layers[il];
|
||||
ggml_tensor * shortcut = cur;
|
||||
|
||||
// layernorm1
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, sam_eps, il);
|
||||
|
||||
const int64_t w0 = cur->ne[1];
|
||||
const int64_t h0 = cur->ne[2];
|
||||
|
||||
ggml_tensor * indices;
|
||||
|
||||
if (hparams.is_global_attn(il)) {
|
||||
indices = rel_pos_indices_global;
|
||||
} else {
|
||||
// local attention layer - apply window partition
|
||||
cur = window_partition(ctx0, cur, window);
|
||||
indices = rel_pos_indices_local;
|
||||
}
|
||||
|
||||
const int64_t W = cur->ne[1];
|
||||
const int64_t H = cur->ne[2];
|
||||
// self-attention
|
||||
{
|
||||
const int B = cur->ne[3];
|
||||
|
||||
cur = ggml_mul_mat(ctx0, layer.qkv_w, cur);
|
||||
cur = ggml_add(ctx0, cur, layer.qkv_b);
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_embd, 3, W * H, B);
|
||||
|
||||
ggml_tensor * Q;
|
||||
ggml_tensor * K;
|
||||
ggml_tensor * V;
|
||||
|
||||
Q = ggml_view_3d(ctx0, cur, n_embd, W * H, B, cur->nb[2], cur->nb[3], 0 * cur->nb[1]);
|
||||
Q = ggml_reshape_4d(ctx0, ggml_cont(ctx0, Q), d_heads, n_heads, W * H, B);
|
||||
|
||||
K = ggml_view_3d(ctx0, cur, n_embd, W * H, B, cur->nb[2], cur->nb[3], 1 * cur->nb[1]);
|
||||
K = ggml_reshape_4d(ctx0, ggml_cont(ctx0, K), d_heads, n_heads, W * H, B);
|
||||
|
||||
V = ggml_view_3d(ctx0, cur, n_embd, W * H, B, cur->nb[2], cur->nb[3], 2 * cur->nb[1]);
|
||||
V = ggml_reshape_4d(ctx0, ggml_cont(ctx0, V), d_heads, n_heads, W * H, B);
|
||||
|
||||
ggml_tensor * mask;
|
||||
ggml_tensor * rw;
|
||||
ggml_tensor * rh;
|
||||
ggml_tensor * qr;
|
||||
|
||||
rw = get_rel_pos(ctx0, layer.rel_pos_w, indices, W, W); // [W, W, C]
|
||||
rh = get_rel_pos(ctx0, layer.rel_pos_h, indices, H, H); // [H, H, C]
|
||||
qr = ggml_permute(ctx0, Q, 0, 2, 1, 3);
|
||||
qr = ggml_reshape_4d(ctx0, ggml_cont(ctx0, qr), d_heads, W, H, B * n_heads);
|
||||
|
||||
rw = ggml_mul_mat(ctx0, rw,
|
||||
ggml_cont(ctx0, ggml_permute(ctx0, qr, 0, 2, 1, 3))); // [B*n_heads, W, H, W]
|
||||
rw = ggml_cont(ctx0, ggml_permute(ctx0, rw, 0, 2, 1, 3)); // [B*n_heads, H, W, W]
|
||||
rw = ggml_reshape_4d(ctx0, rw, W, 1, W * H, n_heads * B);
|
||||
rw = ggml_repeat_4d(ctx0, rw, W, H, W * H, n_heads * B);
|
||||
rh = ggml_mul_mat(ctx0, rh, qr); // [B*n_heads, H, W, H]
|
||||
rh = ggml_reshape_4d(ctx0, rh, 1, H, W * H, n_heads * B);
|
||||
mask = ggml_add(ctx0, rw, rh); // [B*n_heads, H*W, H, W]
|
||||
mask = ggml_reshape_4d(ctx0, mask, W * H, W * H, n_heads, B);
|
||||
// casting mask to F16 only required when flash-attn is enabled
|
||||
if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
|
||||
mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
|
||||
}
|
||||
|
||||
const float scale = 1.0f / sqrtf(static_cast<float>(d_heads));
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b, Q, K, V, mask, scale,
|
||||
il); // [B, H*W, n_embd]
|
||||
cur = ggml_reshape_4d(ctx0, ggml_cont(ctx0, cur), n_embd, W, H, B);
|
||||
}
|
||||
|
||||
if (hparams.is_global_attn(il) == false) {
|
||||
// local attention layer - reverse window partition
|
||||
cur = window_unpartition(ctx0, cur, w0, h0, window);
|
||||
}
|
||||
|
||||
// re-add the layer input, e.g., residual
|
||||
cur = ggml_add(ctx0, cur, shortcut);
|
||||
|
||||
ggml_tensor * inpFF = cur;
|
||||
|
||||
// layernorm2
|
||||
cur = build_norm(inpFF, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, sam_eps, il);
|
||||
|
||||
// ffn
|
||||
cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
|
||||
// residual 2
|
||||
cur = ggml_add(ctx0, cur, inpFF);
|
||||
cb(cur, "sam_layer_out", il);
|
||||
}
|
||||
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = conv_2d_f32(ctx0, model.neck_0_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = build_norm(cur, model.neck_1_w, model.neck_1_b, NORM_TYPE_NORMAL, sam_eps, -1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = conv_2d_f32(ctx0, model.neck_2_w, cur, 1, 1, 1, 1, 1, 1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = build_norm(cur, model.neck_3_w, model.neck_3_b, NORM_TYPE_NORMAL, sam_eps, -1);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
|
||||
cur = conv_2d_f32(ctx0, model.net_2, cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = conv_2d_f32(ctx0, model.net_3, cur, 2, 2, 1, 1, 1, 1);
|
||||
cb(cur, "sam_output", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_deepseekocr::build() {
|
||||
// patch embedding
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
bool is_overview = img.add_viewsep;
|
||||
int n_tiles_per_row = 0;
|
||||
// number of separate "row" images batched together in this graph call
|
||||
// (captured now, before n_batch below gets repurposed as the SAM/ViT batch size)
|
||||
const int n_rows_batch = n_batch;
|
||||
|
||||
// note: we expect either a batch of rows or a batch of overviews, but not a mix of both
|
||||
|
||||
if (!is_overview) {
|
||||
// handle the case where we have a batch of rows
|
||||
// sanity check
|
||||
for (auto & entry : img_batch->entries) {
|
||||
if (entry.add_viewsep) {
|
||||
throw std::runtime_error("DeepSeek-OCR: mixed overview and non-overview images in batch");
|
||||
}
|
||||
if (entry.nx() != img.nx() || entry.ny() != img.ny()) {
|
||||
throw std::runtime_error("DeepSeek-OCR: mixed image sizes in batch");
|
||||
}
|
||||
}
|
||||
|
||||
GGML_ASSERT(img.ny() >= img.nx());
|
||||
GGML_ASSERT(img.ny() % img.nx() == 0);
|
||||
n_tiles_per_row = img.ny() / img.nx();
|
||||
|
||||
// each entry is one "row" image of shape [tile_size, tile_size * n_tiles_per_row, 3];
|
||||
// merge the tile axis into the batch axis, giving a combined SAM input of shape
|
||||
// [tile_size, tile_size, 3, n_tiles_per_row * n_rows_batch] (tile fast, row slow)
|
||||
inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx() * img.nx(), n_tiles_per_row, 3, n_rows_batch);
|
||||
inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 2, 1, 3));
|
||||
inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.nx(), 3, n_tiles_per_row * n_rows_batch);
|
||||
}
|
||||
|
||||
ggml_tensor * sam_out = build_sam(inp_raw);
|
||||
|
||||
if (!is_overview) {
|
||||
n_batch = n_tiles_per_row * n_rows_batch;
|
||||
}
|
||||
|
||||
const int clip_n_patches = sam_out->ne[0] * sam_out->ne[1];
|
||||
|
||||
ggml_tensor * clip_out;
|
||||
// Building DS-OCR CLIP
|
||||
{
|
||||
ggml_tensor * inp;
|
||||
|
||||
// sam_out: [patch_h, patch_w, n_embd, n_batch]
|
||||
// -> [n_embd, clip_n_patches, n_batch]
|
||||
inp = ggml_reshape_3d(ctx0, sam_out, clip_n_patches, sam_out->ne[2], sam_out->ne[3]);
|
||||
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
|
||||
|
||||
ggml_tensor * new_pos_embd = model.position_embeddings;
|
||||
|
||||
int n_pos = new_pos_embd->ne[1]; // +1 for [CLS]
|
||||
const auto tgt_size = static_cast<int>(std::sqrt(inp->ne[1]));
|
||||
const auto src_size = static_cast<int>(std::sqrt(n_pos - 1));
|
||||
|
||||
if (tgt_size != src_size) {
|
||||
ggml_tensor * old_pos_embd;
|
||||
ggml_tensor * cls_tok;
|
||||
|
||||
old_pos_embd = ggml_view_2d(ctx0, new_pos_embd, new_pos_embd->ne[0], src_size * src_size,
|
||||
ggml_row_size(new_pos_embd->type, new_pos_embd->ne[0]), 0);
|
||||
cls_tok = ggml_view_2d(ctx0, new_pos_embd, new_pos_embd->ne[0], 1,
|
||||
ggml_row_size(new_pos_embd->type, new_pos_embd->ne[0]), src_size * src_size);
|
||||
new_pos_embd = ggml_interpolate(ctx0, old_pos_embd, tgt_size, tgt_size, new_pos_embd->ne[0], 1,
|
||||
GGML_SCALE_MODE_BICUBIC);
|
||||
new_pos_embd = ggml_reshape_3d(ctx0, new_pos_embd, n_embd, tgt_size * tgt_size, 1);
|
||||
new_pos_embd = ggml_concat(ctx0, new_pos_embd, cls_tok, 1);
|
||||
n_pos = tgt_size * tgt_size + 1;
|
||||
}
|
||||
|
||||
// add CLS token per batch item
|
||||
// inp: [n_embd, clip_n_patches, n_batch]
|
||||
// class_embedding: [n_embd] -> [n_embd, 1, n_batch]
|
||||
ggml_tensor * cls_embd = ggml_repeat_4d(ctx0, model.class_embedding, n_embd, 1, n_batch, 1);
|
||||
inp = ggml_concat(ctx0, cls_embd, inp, 1);
|
||||
|
||||
// for selecting learned pos embd, used by ViT
|
||||
ggml_tensor * positions = ggml_cast(ctx0, ggml_arange(ctx0, 0, n_pos, 1), GGML_TYPE_I32);
|
||||
ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, new_pos_embd, positions);
|
||||
|
||||
ggml_tensor * cur = build_vit(inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_QUICK, learned_pos_embd, nullptr);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
clip_out = cur;
|
||||
}
|
||||
|
||||
// sam_out: [patch_h, patch_w, n_embd, n_batch]
|
||||
// -> [n_embd, clip_n_patches, n_batch]
|
||||
sam_out = ggml_cont(ctx0, ggml_permute(ctx0, sam_out, 1, 2, 0, 3));
|
||||
sam_out = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0], clip_n_patches, n_batch);
|
||||
|
||||
// clip_out: [n_embd, n_pos, n_batch] where n_pos = clip_n_patches + 1 (CLS)
|
||||
// strip CLS token: skip first position, view only the patch tokens
|
||||
clip_out = ggml_view_3d(ctx0, clip_out, n_embd, clip_n_patches, n_batch,
|
||||
clip_out->nb[1], clip_out->nb[2], clip_out->nb[1]);
|
||||
|
||||
ggml_tensor * cur;
|
||||
cur = ggml_concat(ctx0, clip_out, sam_out, 0);
|
||||
cur = ggml_mul_mat(ctx0, model.mm_fc_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_fc_b);
|
||||
|
||||
if (is_overview) {
|
||||
// global view: weave one newline per row + trailing view separator
|
||||
const auto h = static_cast<int>(std::sqrt(static_cast<float>(cur->ne[1])));
|
||||
const auto w = h;
|
||||
const auto n_dim = cur->ne[0];
|
||||
|
||||
ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, h, n_batch);
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_dim, w, h, n_batch);
|
||||
cur = ggml_reshape_3d(ctx0, ggml_concat(ctx0, cur, imgnl, 1), n_dim, (w + 1) * h, n_batch);
|
||||
ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, n_dim, 1, n_batch, 1);
|
||||
cur = ggml_concat(ctx0, cur, vs, 1); // (n_dim, h*(w+1) + 1, n_batch)
|
||||
} else {
|
||||
// tile row: interleave tiles within each row, add newline per row
|
||||
const int grid_x = static_cast<int>(std::sqrt(static_cast<float>(clip_n_patches)));
|
||||
const int grid_y = grid_x;
|
||||
const auto n_dim = cur->ne[0];
|
||||
|
||||
// merge n_dim into the grid_x axis, freeing the 4th axis for n_rows_batch
|
||||
// (n_dim, clip_n_patches, n_tiles_per_row * n_rows_batch) -> (n_dim*grid_x, grid_y, n_tiles_per_row, n_rows_batch)
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_dim * grid_x, grid_y, n_tiles_per_row, n_rows_batch);
|
||||
|
||||
// tiles: re-order from A.row0 A.row1 B.row0 B.row1 ...
|
||||
// to A.row0 B.row0 A.row1 B.row1 ...
|
||||
// then add nl: A.row0 B.row0 [nl] A.row1 B.row1 [nl] ...
|
||||
// interleave tiles: -> (n_dim*grid_x, n_tiles_per_row, grid_y, n_rows_batch)
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));
|
||||
|
||||
// merge: -> (n_dim, grid_x*n_tiles_per_row, grid_y, n_rows_batch)
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_dim, grid_x * n_tiles_per_row, grid_y, n_rows_batch);
|
||||
|
||||
// append newline per row: (n_dim, grid_x*n_tiles_per_row+1, grid_y, n_rows_batch)
|
||||
ggml_tensor * imgnl = ggml_repeat_4d(ctx0, model.image_newline, n_dim, 1, grid_y, n_rows_batch);
|
||||
cur = ggml_concat(ctx0, cur, imgnl, 1);
|
||||
|
||||
// flatten: (n_dim, (grid_x*n_tiles_per_row+1)*grid_y, n_rows_batch)
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_dim, (grid_x * n_tiles_per_row + 1) * grid_y, n_rows_batch);
|
||||
}
|
||||
|
||||
cb(cur, "dsocr_output", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_deepseekocr2::build() {
|
||||
GGML_ASSERT(hparams.n_head_kv > 0);
|
||||
GGML_ASSERT(n_head % hparams.n_head_kv == 0);
|
||||
|
||||
// patch embedding
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
ggml_tensor * sam_out = build_sam(inp_raw);
|
||||
|
||||
ggml_tensor * qwen2_out;
|
||||
// Building Qwen2 encoder
|
||||
{
|
||||
ggml_tensor * inp;
|
||||
|
||||
// H*W, C, B
|
||||
inp = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0] * sam_out->ne[1], sam_out->ne[2], sam_out->ne[3]);
|
||||
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); // C, H*W, B
|
||||
|
||||
auto num_image_tokens = inp->ne[1]; // H*W
|
||||
GGML_ASSERT(num_image_tokens == 144 || num_image_tokens == 256);
|
||||
|
||||
// query based on numbers of image tokens (in SAM output)
|
||||
// 16x16 -> query_1024 (1024x1024 images)
|
||||
// 12x12 -> query_768 (768x768 images)
|
||||
|
||||
ggml_tensor * query_embed = model.resample_query_1024;
|
||||
int num_queries = 256;
|
||||
|
||||
if (num_image_tokens == 144) {
|
||||
query_embed = model.resample_query_768;
|
||||
num_queries = 144;
|
||||
}
|
||||
|
||||
// repeat the query embedding per batch item, then append: (C, num_image_tokens + num_queries, B)
|
||||
query_embed = ggml_cast(ctx0, query_embed, inp->type);
|
||||
query_embed = ggml_repeat_4d(ctx0, query_embed, query_embed->ne[0], num_queries, inp->ne[2], 1);
|
||||
inp = ggml_concat(ctx0, inp, query_embed, 1);
|
||||
|
||||
auto seq_len = inp->ne[1];
|
||||
|
||||
// qwen2 encoder attention mask
|
||||
ggml_tensor * attn_mask = build_inp_attn_mask(seq_len, seq_len);
|
||||
|
||||
ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_arange(ctx0, 0, seq_len, 1), GGML_TYPE_I32);
|
||||
|
||||
auto add_rope = [&](ggml_tensor * x, const clip_layer &) {
|
||||
return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head,
|
||||
GGML_ROPE_TYPE_NEOX, 131072, 1000000, 1, 0, 1, 0, 0);
|
||||
};
|
||||
|
||||
build_vit_opts vit_opts;
|
||||
vit_opts.attn_mask = attn_mask;
|
||||
|
||||
// build_vit applies model.post_ln_w internally; do not re-apply
|
||||
ggml_tensor * cur = build_vit(inp, seq_len, NORM_TYPE_RMS, FFN_SILU,
|
||||
/* learned_pos_embd */ nullptr, add_rope, vit_opts);
|
||||
|
||||
cur = ggml_cont(ctx0,
|
||||
ggml_view_3d(ctx0, cur, cur->ne[0], num_queries, cur->ne[2], cur->nb[1], cur->nb[2],
|
||||
cur->nb[1] * (cur->ne[1] - num_queries))); // only take query tokens for output
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
qwen2_out = cur;
|
||||
}
|
||||
|
||||
ggml_tensor * cur;
|
||||
|
||||
cur = ggml_mul_mat(ctx0, model.mm_fc_w, qwen2_out);
|
||||
cur = ggml_add(ctx0, cur, model.mm_fc_b);
|
||||
|
||||
// view_seperator only after the global view
|
||||
if (img.add_viewsep) {
|
||||
ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, model.view_seperator->ne[0], 1, cur->ne[2], 1);
|
||||
cur = ggml_concat(ctx0, cur, vs, 1); // (n_dim, 257, n_batch)
|
||||
}
|
||||
|
||||
cb(cur, "dsocr2_output", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_dots3note_a::build() {
|
||||
// inp_raw: [n_frames, n_mel, 1], one 60s chunk, mel frames not padded
|
||||
// the reference impl zero-masks conv inputs beyond the valid length at each stage;
|
||||
// running on exactly the valid frames with the convs' zero padding is equivalent
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
GGML_ASSERT(inp->type == GGML_TYPE_F32);
|
||||
|
||||
// 3x conv2d (k=3, s=2, p=1) + gelu
|
||||
{
|
||||
auto conv_block = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) {
|
||||
x = ggml_conv_2d(ctx0, w, x, 2, 2, 1, 1, 1, 1);
|
||||
x = ggml_add(ctx0, x, ggml_reshape_4d(ctx0, b, 1, 1, x->ne[2], 1));
|
||||
return ggml_gelu_erf(ctx0, x);
|
||||
};
|
||||
|
||||
inp = conv_block(inp, model.conv2d_1_w, model.conv2d_1_b);
|
||||
inp = conv_block(inp, model.conv2d_2_w, model.conv2d_2_b);
|
||||
inp = conv_block(inp, model.conv2d_3_w, model.conv2d_3_b);
|
||||
// inp: [OW=n_frames/8, OH=n_mel/8, OC=480, 1]
|
||||
cb(inp, "after_conv_stem", -1);
|
||||
}
|
||||
|
||||
// [OW, OH, OC, 1] -> [OH*OC, OW], feature index f + OH*c (matches the reference permute+reshape)
|
||||
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
|
||||
inp = ggml_reshape_2d(ctx0, inp, inp->ne[0] * inp->ne[1], inp->ne[2]);
|
||||
|
||||
// project to d_model (no bias)
|
||||
inp = ggml_mul_mat(ctx0, model.conv_out_w, inp);
|
||||
cb(inp, "after_conv_out", -1);
|
||||
|
||||
const int64_t n_pos = inp->ne[1];
|
||||
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
// partial rotary: first half of each head, NEOX style
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return ggml_rope_ext(ctx0, cur, positions, nullptr, d_head/2,
|
||||
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
};
|
||||
|
||||
ggml_tensor * cur = build_vit(inp, n_pos,
|
||||
NORM_TYPE_RMS, hparams.ffn_op,
|
||||
nullptr, add_pos);
|
||||
cb(cur, "after_transformer", -1);
|
||||
|
||||
// adapter: LayerNorm -> Linear -> GELU -> Linear
|
||||
cur = build_norm(cur, model.mm_norm_pre_w, model.mm_norm_pre_b, NORM_TYPE_NORMAL, 1e-5, -1);
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU_ERF, -1);
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_dotsocr::build() {
|
||||
const int n_pos = n_patches;
|
||||
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
|
||||
|
||||
// note: similar to PaddleOCR
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return ggml_rope_multi(
|
||||
ctx0, cur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
|
||||
32768, 10000, 1, 0, 1, 32, 1);
|
||||
};
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
NORM_TYPE_RMS,
|
||||
hparams.ffn_op,
|
||||
nullptr,
|
||||
add_pos);
|
||||
|
||||
cb(cur, "vit_out", -1);
|
||||
|
||||
// dots.ocr patch merger + projector
|
||||
{
|
||||
GGML_ASSERT(hparams.n_merge > 0);
|
||||
cur = build_norm(cur, model.mm_input_norm_w, model.mm_input_norm_b, NORM_TYPE_NORMAL, 1e-6, -1);
|
||||
cur = build_patch_merge_permute(cur, hparams.n_merge);
|
||||
cb(cur, "after_patch_merger", -1);
|
||||
cur = build_ffn(cur,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr, // no gate
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU_ERF, -1); // nn.GELU() defaults to exact erf-based GELU
|
||||
cb(cur, "after_projector", -1);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
// similar to qwen2vl, except for GQA attention
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_exaone4_5::build() {
|
||||
GGML_ASSERT(model.patch_bias == nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
|
||||
const int batch_size = 1;
|
||||
const bool use_window_attn = hparams.n_wa_pattern > 0;
|
||||
const int n_wa_pattern = hparams.n_wa_pattern;
|
||||
const int n_pos = n_patches;
|
||||
const int num_position_ids = n_pos * 4;
|
||||
|
||||
const norm_type norm_t = NORM_TYPE_RMS;
|
||||
|
||||
const int64_t n_kv_head = hparams.n_head_kv > 0 ? hparams.n_head_kv : n_head;
|
||||
GGML_ASSERT(n_head % n_kv_head == 0);
|
||||
|
||||
int rope_sections[4] = { d_head / 4, d_head / 4, d_head / 4, d_head / 4 };
|
||||
const float rope_freq_base = hparams.rope_theta > 0.0f ? hparams.rope_theta : 10000.0f;
|
||||
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
|
||||
|
||||
GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
|
||||
GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
|
||||
|
||||
{
|
||||
ggml_tensor * inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
|
||||
inp = ggml_add(ctx0, inp, inp_1);
|
||||
inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);
|
||||
inp = ggml_cont_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
inp = ggml_reshape_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
|
||||
inp = ggml_cont_3d(
|
||||
ctx0, inp,
|
||||
n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
ggml_tensor * window_mask = nullptr;
|
||||
ggml_tensor * window_idx = nullptr;
|
||||
ggml_tensor * inv_window_idx = nullptr;
|
||||
|
||||
if (use_window_attn) {
|
||||
window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);
|
||||
ggml_set_name(window_idx, "window_idx");
|
||||
ggml_set_input(window_idx);
|
||||
|
||||
inv_window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);
|
||||
ggml_set_name(inv_window_idx, "inv_window_idx");
|
||||
ggml_set_input(inv_window_idx);
|
||||
|
||||
window_mask = build_inp_attn_mask(n_pos, n_pos);
|
||||
}
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
|
||||
if (use_window_attn) {
|
||||
GGML_ASSERT(batch_size == 1);
|
||||
inpL = ggml_reshape_2d(ctx0, inpL, n_embd * 4, n_patches_x * n_patches_y * batch_size / 4);
|
||||
inpL = ggml_get_rows(ctx0, inpL, inv_window_idx);
|
||||
inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
const auto & layer = model.layers[il];
|
||||
const bool full_attn = use_window_attn ? (il + 1) % n_wa_pattern == 0 : true;
|
||||
ggml_tensor * cur = inpL;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
|
||||
cb(cur, "ln1", il);
|
||||
|
||||
{
|
||||
GGML_ASSERT(layer.qkv_w != nullptr);
|
||||
cur = build_mm(layer.qkv_w, cur);
|
||||
if (layer.qkv_b) {
|
||||
cur = ggml_add(ctx0, cur, layer.qkv_b);
|
||||
}
|
||||
|
||||
const int64_t n_embd_kv = d_head * n_kv_head;
|
||||
ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_patches,
|
||||
ggml_row_size(cur->type, d_head),
|
||||
cur->nb[1],
|
||||
0);
|
||||
ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, d_head, n_kv_head, n_patches,
|
||||
ggml_row_size(cur->type, d_head),
|
||||
cur->nb[1],
|
||||
ggml_row_size(cur->type, n_embd));
|
||||
ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, d_head, n_kv_head, n_patches,
|
||||
ggml_row_size(cur->type, d_head),
|
||||
cur->nb[1],
|
||||
ggml_row_size(cur->type, n_embd + n_embd_kv));
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
Qcur = ggml_rope_multi(
|
||||
ctx0, Qcur, positions, nullptr,
|
||||
d_head / 2, rope_sections, GGML_ROPE_TYPE_VISION, 32768, rope_freq_base, 1, 0, 1, 32, 1);
|
||||
Kcur = ggml_rope_multi(
|
||||
ctx0, Kcur, positions, nullptr,
|
||||
d_head / 2, rope_sections, GGML_ROPE_TYPE_VISION, 32768, rope_freq_base, 1, 0, 1, 32, 1);
|
||||
|
||||
cb(Qcur, "Qcur_rope", il);
|
||||
cb(Kcur, "Kcur_rope", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
ggml_tensor * attn_mask = full_attn ? nullptr : window_mask;
|
||||
cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, attn_mask, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
inpL = cur;
|
||||
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
|
||||
cb(cur, "ffn_inp_normed", il);
|
||||
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
ggml_tensor * embeddings = inpL;
|
||||
embeddings = build_norm(embeddings, model.post_ln_w, model.post_ln_b, norm_t, eps, n_layer);
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
|
||||
embeddings = build_ffn(embeddings,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
FFN_GELU,
|
||||
-1);
|
||||
|
||||
if (use_window_attn) {
|
||||
GGML_ASSERT(batch_size == 1);
|
||||
embeddings = ggml_reshape_2d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4);
|
||||
embeddings = ggml_get_rows(ctx0, embeddings, window_idx);
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4, batch_size);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
/**
|
||||
* Gemma 4 Audio Conformer Encoder (clip_graph_gemma4a)
|
||||
*
|
||||
* Architecture: Conformer with dual half-step FFN, full self-attention
|
||||
* with sinusoidal RPE, depthwise light conv, and output projection.
|
||||
*/
|
||||
|
||||
#include "models.h"
|
||||
#include <cmath>
|
||||
|
||||
ggml_cgraph * clip_graph_gemma4a::build() {
|
||||
const float res_weight = 0.5f;
|
||||
const float norm_eps = 1e-6f;
|
||||
|
||||
// 1. Input
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
auto * cur = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
|
||||
// 2. Subsampling Conv2D (symmetric padding=1, matching PyTorch)
|
||||
{
|
||||
for (int i = 0; i < 2; i++) {
|
||||
cur = ggml_conv_2d(ctx0, model.sscp_conv_w[i], cur, 2, 2, 1, 1, 1, 1);
|
||||
if (model.sscp_conv_b[i]) {
|
||||
cur = ggml_add(ctx0, cur, model.sscp_conv_b[i]);
|
||||
}
|
||||
// nn.LayerNorm(channels): permute ch to ne[0], normalize, permute back
|
||||
if (model.sscp_norm_w[i]) {
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = ggml_norm(ctx0, cur, norm_eps);
|
||||
cur = ggml_mul(ctx0, cur, model.sscp_norm_w[i]);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3));
|
||||
}
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
}
|
||||
// Flatten [freq, time, ch, 1] -> [ch*freq, time]
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 2, 0, 3));
|
||||
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2]);
|
||||
if (model.sscp_inp_proj_w) {
|
||||
cur = build_mm(model.sscp_inp_proj_w, cur);
|
||||
if (model.sscp_inp_proj_b) {
|
||||
cur = ggml_add(ctx0, cur, model.sscp_inp_proj_b);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const int64_t n_pos = cur->ne[1];
|
||||
|
||||
// Chunked local attention parameters
|
||||
const int64_t C = 12; // chunk_size
|
||||
const int64_t P = 12; // max_past_horizon (context_left - 1)
|
||||
const int64_t S = C + P; // context_size = 24
|
||||
const int64_t R = P + 1; // RPE positions = 13
|
||||
const int64_t B = (n_pos + C - 1) / C; // num_blocks
|
||||
const int64_t Np = B * C; // padded sequence length
|
||||
const int64_t pad_seq = Np - n_pos;
|
||||
|
||||
// Input tensors: blocked RPE and blocked attention mask
|
||||
ggml_tensor * pos_emb = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_head * d_head, R);
|
||||
ggml_set_name(pos_emb, "pos_emb");
|
||||
ggml_set_input(pos_emb);
|
||||
|
||||
ggml_tensor * kq_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, S, C, B);
|
||||
ggml_set_name(kq_mask, "kq_mask");
|
||||
ggml_set_input(kq_mask);
|
||||
|
||||
// 3. Conformer Blocks
|
||||
for (int il = 0; il < hparams.n_layer; il++) {
|
||||
const auto & layer = model.layers[il];
|
||||
auto * residual = cur;
|
||||
|
||||
// FFN 1 (half-step)
|
||||
if (layer.ff_norm_w && layer.ff_up_w && layer.ff_down_w) {
|
||||
cur = build_norm(cur, layer.ff_norm_w, nullptr, NORM_TYPE_RMS, norm_eps, il);
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, nullptr, nullptr, nullptr,
|
||||
layer.ff_down_w, nullptr, FFN_SILU, il);
|
||||
if (layer.ff_post_norm_w) {
|
||||
cur = build_norm(cur, layer.ff_post_norm_w, nullptr, NORM_TYPE_RMS, norm_eps, il);
|
||||
}
|
||||
residual = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, res_weight));
|
||||
}
|
||||
|
||||
// Chunked local self-attention with RPE
|
||||
if (layer.q_w && layer.k_w && layer.v_w && layer.o_w) {
|
||||
const float q_scale = (1.0f / sqrtf((float)d_head)) / logf(2.0f);
|
||||
const float k_scale = logf(1.0f + expf(1.0f)) / logf(2.0f);
|
||||
const float softcap = 50.0f;
|
||||
|
||||
ggml_tensor * attn_norm_w = layer.attn_pre_norm_w ? layer.attn_pre_norm_w : layer.ln_1_w;
|
||||
cur = attn_norm_w
|
||||
? build_norm(residual, attn_norm_w, nullptr, NORM_TYPE_RMS, norm_eps, il)
|
||||
: residual;
|
||||
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
|
||||
// [n_embd, n_pos] -> [D, H, N]
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
|
||||
|
||||
// Q/K scaling
|
||||
Qcur = ggml_scale(ctx0, Qcur, q_scale);
|
||||
if (layer.per_dim_scale_w) {
|
||||
Qcur = ggml_mul(ctx0, Qcur, ggml_reshape_3d(ctx0, layer.per_dim_scale_w, d_head, 1, 1));
|
||||
}
|
||||
Kcur = ggml_scale(ctx0, Kcur, k_scale);
|
||||
if (layer.per_dim_k_scale_w) {
|
||||
Kcur = ggml_mul(ctx0, Kcur, ggml_reshape_3d(ctx0, layer.per_dim_k_scale_w, d_head, 1, 1));
|
||||
}
|
||||
|
||||
// Q blocking: [D, H, N] -> pad to Np -> reshape [D, H, C, B]
|
||||
// ggml permute: ne[ax_i] = src->ne[i], so (0,3,1,2) sends H->3, C->1, B->2
|
||||
Qcur = ggml_pad(ctx0, Qcur, 0, 0, pad_seq, 0); // [D, H, Np]
|
||||
Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, C, B); // [D, H, C, B]
|
||||
Qcur = ggml_cont(ctx0, ggml_permute(ctx0, Qcur, 0, 3, 1, 2)); // [D, C, B, H]
|
||||
|
||||
// K/V block context extraction via overlapping view:
|
||||
// Left pad by P and right pad to S*B elements,
|
||||
// then view with stride C in the block dimension (overlapping windows).
|
||||
auto extract_blocks = [&](ggml_tensor * t) -> ggml_tensor * {
|
||||
// [D, H, N] -> left pad by P, right pad to S*B
|
||||
const int64_t pad_kv = S * B - n_pos;
|
||||
t = ggml_pad_ext(ctx0, t, 0, 0, 0, 0, P, pad_kv - P, 0, 0); // [D, H, S*B]
|
||||
// Overlapping view: stride for B dim is C positions, not S
|
||||
// ne = [D, H, S, B], data_size = D*H*S*B*sizeof = source_nbytes (exact fit)
|
||||
// nb1=D*sizeof, nb2=D*H*sizeof, nb3=C*D*H*sizeof (overlap: C < S)
|
||||
t = ggml_view_4d(ctx0, t, d_head, n_head, S, B,
|
||||
t->nb[1], t->nb[2], C * t->nb[2], 0);
|
||||
t = ggml_cont(ctx0, t); // materialize overlapping windows
|
||||
return t;
|
||||
};
|
||||
|
||||
ggml_tensor * Kblk = extract_blocks(Kcur);
|
||||
// [D, H, S, B] -> [D, S, B, H] via permute(0,3,1,2)
|
||||
Kblk = ggml_cont(ctx0, ggml_permute(ctx0, Kblk, 0, 3, 1, 2));
|
||||
|
||||
ggml_tensor * Vblk = extract_blocks(Vcur);
|
||||
// [D, H, S, B] -> [S, D, B, H] via permute(1,3,0,2)
|
||||
Vblk = ggml_cont(ctx0, ggml_permute(ctx0, Vblk, 1, 3, 0, 2));
|
||||
|
||||
// Content attention: Q @ K^T
|
||||
// Kblk=[D,S,B,H], Qcur=[D,C,B,H] -> mul_mat contracts on D -> [S,C,B,H]
|
||||
ggml_tensor * matrix_ac = ggml_mul_mat(ctx0, Kblk, Qcur);
|
||||
|
||||
// Relative position attention
|
||||
if (layer.attn_k_rel_w) {
|
||||
// RPE: [n_embd, R] -> project -> [D, H, R] -> [D, R, H]
|
||||
auto * p = ggml_mul_mat(ctx0, layer.attn_k_rel_w, pos_emb);
|
||||
p = ggml_reshape_3d(ctx0, p, d_head, n_head, R);
|
||||
p = ggml_cont(ctx0, ggml_permute(ctx0, p, 0, 2, 1, 3)); // [D, R, H]
|
||||
|
||||
// Q_flat @ RPE^T: [D, C*B, H] @ [D, R, H] -> [R, C*B, H]
|
||||
auto * Q_flat = ggml_reshape_3d(ctx0, Qcur, d_head, C * B, n_head);
|
||||
auto * matrix_bd = ggml_mul_mat(ctx0, p, Q_flat); // [R, C*B, H]
|
||||
matrix_bd = ggml_reshape_4d(ctx0, matrix_bd, R, C, B, n_head); // [R, C, B, H]
|
||||
|
||||
// Blocked relative shift (appendix B of Transformer-XL)
|
||||
{
|
||||
matrix_bd = ggml_pad(ctx0, matrix_bd, S + 1 - R, 0, 0, 0); // [S+1, C, B, H]
|
||||
matrix_bd = ggml_reshape_3d(ctx0, matrix_bd, (S + 1) * C, B, n_head);
|
||||
matrix_bd = ggml_view_3d(ctx0, matrix_bd,
|
||||
C * S, B, n_head,
|
||||
matrix_bd->nb[1], matrix_bd->nb[2], 0);
|
||||
matrix_bd = ggml_cont(ctx0, matrix_bd); // [C*S, B, H]
|
||||
matrix_bd = ggml_reshape_4d(ctx0, matrix_bd, S, C, B, n_head); // [S, C, B, H]
|
||||
}
|
||||
|
||||
matrix_ac = ggml_add(ctx0, matrix_ac, matrix_bd);
|
||||
}
|
||||
|
||||
auto * scores = matrix_ac; // [S, C, B, H]
|
||||
|
||||
// Softcap
|
||||
scores = ggml_scale(ctx0, scores, 1.0f / softcap);
|
||||
scores = ggml_tanh(ctx0, scores);
|
||||
scores = ggml_scale(ctx0, scores, softcap);
|
||||
|
||||
// Blocked attention mask: [S, C, B] broadcasts over H
|
||||
scores = ggml_add(ctx0, scores, kq_mask);
|
||||
|
||||
ggml_tensor * attn = ggml_soft_max(ctx0, scores);
|
||||
|
||||
// attn @ V: [S,C,B,H] @ [S,D,B,H] -> [D,C,B,H]
|
||||
ggml_tensor * x = ggml_mul_mat(ctx0, Vblk, attn);
|
||||
|
||||
// [D,C,B,H] -> [D,H,C,B] via permute(0,2,3,1) -> flatten -> trim
|
||||
x = ggml_cont(ctx0, ggml_permute(ctx0, x, 0, 2, 3, 1));
|
||||
x = ggml_cont_2d(ctx0, x, d_head * n_head, C * B);
|
||||
if (pad_seq > 0) {
|
||||
x = ggml_view_2d(ctx0, x, d_head * n_head, n_pos, x->nb[1], 0);
|
||||
x = ggml_cont(ctx0, x);
|
||||
}
|
||||
|
||||
x = build_mm(layer.o_w, x);
|
||||
if (layer.o_b) { x = ggml_add(ctx0, x, layer.o_b); }
|
||||
|
||||
if (layer.attn_post_norm_w) {
|
||||
x = build_norm(x, layer.attn_post_norm_w, nullptr, NORM_TYPE_RMS, norm_eps, il);
|
||||
}
|
||||
residual = ggml_add(ctx0, residual, x);
|
||||
}
|
||||
|
||||
// Convolution Module
|
||||
if (layer.norm_conv_w && layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w) {
|
||||
cur = build_norm(residual, layer.norm_conv_w, nullptr, NORM_TYPE_RMS, norm_eps, il);
|
||||
auto * x = build_mm(layer.conv_pw1_w, cur);
|
||||
|
||||
// GLU
|
||||
{
|
||||
int64_t d = x->ne[0] / 2;
|
||||
ggml_tensor * gate = ggml_sigmoid(ctx0,
|
||||
ggml_cont(ctx0, ggml_view_2d(ctx0, x, d, x->ne[1], x->nb[1], d * x->nb[0])));
|
||||
x = ggml_mul(ctx0,
|
||||
ggml_view_2d(ctx0, x, d, x->ne[1], x->nb[1], 0), gate);
|
||||
x = ggml_cont(ctx0, ggml_transpose(ctx0, x));
|
||||
}
|
||||
|
||||
// Causal depthwise Conv1D via ggml_ssm_conv, left padded only.
|
||||
x = ggml_pad_ext(ctx0, x, 4, 0, 0, 0, 0, 0, 0, 0);
|
||||
x = ggml_ssm_conv(ctx0, x, layer.conv_dw_w);
|
||||
if (layer.conv_dw_b) {
|
||||
x = ggml_add(ctx0, x, layer.conv_dw_b);
|
||||
}
|
||||
|
||||
if (layer.conv_norm_w) {
|
||||
x = ggml_rms_norm(ctx0, x, norm_eps);
|
||||
x = ggml_mul(ctx0, x, layer.conv_norm_w);
|
||||
}
|
||||
x = ggml_silu(ctx0, x);
|
||||
x = build_mm(layer.conv_pw2_w, x);
|
||||
residual = ggml_add(ctx0, residual, x);
|
||||
}
|
||||
|
||||
// FFN 2 (half-step)
|
||||
if (layer.ff_norm_1_w && layer.ff_up_1_w && layer.ff_down_1_w) {
|
||||
cur = build_norm(residual, layer.ff_norm_1_w, nullptr, NORM_TYPE_RMS, norm_eps, il);
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_1_w, nullptr, nullptr, nullptr,
|
||||
layer.ff_down_1_w, nullptr, FFN_SILU, il);
|
||||
if (layer.ff_post_norm_1_w) {
|
||||
cur = build_norm(cur, layer.ff_post_norm_1_w, nullptr, NORM_TYPE_RMS, norm_eps, il);
|
||||
}
|
||||
residual = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, res_weight));
|
||||
}
|
||||
|
||||
// Layer output norm
|
||||
cur = layer.ln_2_w
|
||||
? build_norm(residual, layer.ln_2_w, nullptr, NORM_TYPE_RMS, norm_eps, il)
|
||||
: residual;
|
||||
|
||||
}
|
||||
|
||||
// 4. Output Projection
|
||||
if (model.audio_out_proj_w) {
|
||||
cur = build_mm(model.audio_out_proj_w, cur);
|
||||
if (model.audio_out_proj_b) {
|
||||
cur = ggml_add(ctx0, cur, model.audio_out_proj_b);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Audio Multimodal Embedder
|
||||
cur = ggml_rms_norm(ctx0, cur, norm_eps);
|
||||
if (model.mm_soft_emb_norm_w) {
|
||||
cur = ggml_mul(ctx0, cur, model.mm_soft_emb_norm_w);
|
||||
}
|
||||
if (model.mm_input_proj_w) {
|
||||
cur = build_mm(model.mm_input_proj_w, cur);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_gemma4a::build_mm(ggml_tensor * w, ggml_tensor * x) const {
|
||||
auto it = model.clamp_info_map.find(w->name);
|
||||
if (it == model.clamp_info_map.end()) {
|
||||
return ggml_mul_mat(ctx0, w, x);
|
||||
}
|
||||
const auto & ci = it->second;
|
||||
ggml_tensor * clamped = ggml_clamp(ctx0, x, ci.inp_min, ci.inp_max);
|
||||
ggml_tensor * out = ggml_mul_mat(ctx0, w, clamped);
|
||||
return ggml_clamp(ctx0, out, ci.out_min, ci.out_max);
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
#include "models.h"
|
||||
#include <cmath>
|
||||
|
||||
ggml_cgraph * clip_graph_gemma4ua::build() {
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
|
||||
auto cur = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3));
|
||||
|
||||
// Gemma4UnifiedMultimodalEmbedder
|
||||
{
|
||||
// embedding_pre_projection_norm
|
||||
cur = ggml_rms_norm(ctx0, cur, hparams.eps);
|
||||
cur = build_mm(model.mm_input_proj_w, cur);
|
||||
cb(cur, "projected", -1);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
#include "models.h"
|
||||
#include <cmath>
|
||||
|
||||
ggml_cgraph * clip_graph_gemma4uv::build() {
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
// Gemma4UnifiedVisionEmbedder uses default pytorch LayerNorm, not RMSNorm
|
||||
float eps = 1e-5f; // default eps for pytorch LayerNorm
|
||||
|
||||
ggml_tensor * inp = nullptr;
|
||||
{
|
||||
// note: we cannot use ggml_conv_2d here because we need to apply norm after im2col
|
||||
auto c = inp_raw->ne[2];
|
||||
ggml_tensor * kernel = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, patch_size, patch_size, c);
|
||||
inp = ggml_im2col(ctx0, kernel, inp_raw, patch_size, patch_size, 0, 0, 1, 1, true, inp_raw->type);
|
||||
// inp shape: [patch_size * patch_size * c, n_patches_w, n_patches_h]
|
||||
|
||||
inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1] * inp->ne[2] * inp->ne[3]);
|
||||
inp = build_norm(inp, model.patch_norm_1_w, model.patch_norm_1_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
// inp shape: [patch_size * patch_size * c, n_patches]
|
||||
|
||||
inp = ggml_mul_mat(ctx0, model.patch_embeddings_0, inp);
|
||||
inp = ggml_add(ctx0, inp, model.patch_bias);
|
||||
// inp shape: [n_embd, n_patches]
|
||||
|
||||
inp = build_norm(inp, model.patch_norm_2_w, model.patch_norm_2_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
}
|
||||
|
||||
ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_x, "pos_x");
|
||||
ggml_set_input(pos_x);
|
||||
|
||||
ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_y, "pos_y");
|
||||
ggml_set_input(pos_y);
|
||||
|
||||
{
|
||||
const int64_t pos_size = model.position_embeddings->ne[1];
|
||||
const size_t nb1 = ggml_row_size(model.position_embeddings->type, n_embd);
|
||||
|
||||
// positional embeddings are stored as lookup tables (one for x, one for y)
|
||||
ggml_tensor * tbl_x = ggml_view_2d(ctx0, model.position_embeddings,
|
||||
n_embd, pos_size, nb1, 0);
|
||||
ggml_tensor * tbl_y = ggml_view_2d(ctx0, model.position_embeddings,
|
||||
n_embd, pos_size, nb1, pos_size * nb1);
|
||||
|
||||
// ggml_get_rows: [n_embd, n_patches]
|
||||
ggml_tensor * emb_x = ggml_get_rows(ctx0, tbl_x, pos_x);
|
||||
ggml_tensor * emb_y = ggml_get_rows(ctx0, tbl_y, pos_y);
|
||||
|
||||
inp = ggml_add(ctx0, inp, emb_x);
|
||||
inp = ggml_add(ctx0, inp, emb_y);
|
||||
cb(inp, "pos_embd", -1);
|
||||
|
||||
// pos_norm
|
||||
inp = build_norm(inp, model.patch_norm_3_w, model.patch_norm_3_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
}
|
||||
|
||||
auto cur = inp;
|
||||
|
||||
// Gemma4UnifiedMultimodalEmbedder
|
||||
{
|
||||
// embedding_pre_projection_norm
|
||||
cur = ggml_rms_norm(ctx0, cur, hparams.eps);
|
||||
cur = build_mm(model.mm_input_proj_w, cur);
|
||||
cb(cur, "projected", -1);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
#include "models.h"
|
||||
#include <cmath>
|
||||
|
||||
ggml_cgraph * clip_graph_gemma4v::build() {
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
// patches = 2 * (patches - 0.5)
|
||||
// equivalent to: patches * 2 - 1
|
||||
inp_raw = ggml_scale_bias(ctx0, inp_raw, 2.0f, -1.0f);
|
||||
ggml_set_name(inp_raw, "inp_raw_scaled");
|
||||
|
||||
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
|
||||
inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, n_batch);
|
||||
inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
ggml_set_name(inp, "inp");
|
||||
// note: no patch bias
|
||||
|
||||
ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_x, "pos_x");
|
||||
ggml_set_input(pos_x);
|
||||
|
||||
ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_y, "pos_y");
|
||||
ggml_set_input(pos_y);
|
||||
|
||||
{
|
||||
const int64_t pos_size = model.position_embeddings->ne[1];
|
||||
const size_t nb1 = ggml_row_size(model.position_embeddings->type, n_embd);
|
||||
|
||||
// positional embeddings are stored as lookup tables (one for x, one for y)
|
||||
ggml_tensor * tbl_x = ggml_view_2d(ctx0, model.position_embeddings,
|
||||
n_embd, pos_size, nb1, 0);
|
||||
ggml_tensor * tbl_y = ggml_view_2d(ctx0, model.position_embeddings,
|
||||
n_embd, pos_size, nb1, pos_size * nb1);
|
||||
|
||||
// ggml_get_rows: [n_embd, n_patches]
|
||||
ggml_tensor * emb_x = ggml_get_rows(ctx0, tbl_x, pos_x);
|
||||
ggml_tensor * emb_y = ggml_get_rows(ctx0, tbl_y, pos_y);
|
||||
|
||||
inp = ggml_add(ctx0, inp, emb_x);
|
||||
inp = ggml_add(ctx0, inp, emb_y);
|
||||
cb(inp, "pos_embd", -1);
|
||||
}
|
||||
|
||||
// similar to build_rope_2d, but use neox ordering
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
const int64_t n_dim = cur->ne[0];
|
||||
|
||||
// first half, dims [0, n_dim/2)
|
||||
cur = ggml_rope_ext(
|
||||
ctx0,
|
||||
cur,
|
||||
pos_x, // positions
|
||||
nullptr, // freq factors
|
||||
n_dim/2, // n_dims
|
||||
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
|
||||
1.0f, 0.0f, 1.0f, 0.0f, 0.0f
|
||||
);
|
||||
|
||||
// second half, dims [n_dim/2, n_dim)
|
||||
cur = ggml_rope_ext(
|
||||
ctx0,
|
||||
cur,
|
||||
pos_y, // positions
|
||||
nullptr, // freq factors
|
||||
n_dim/2, // n_dims
|
||||
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
|
||||
1.0f, 0.0f, 1.0f, 0.0f, 0.0f
|
||||
);
|
||||
cur = ggml_rope_set_offset(cur, n_dim/2);
|
||||
|
||||
return cur;
|
||||
};
|
||||
|
||||
kq_scale = 1.0f;
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
NORM_TYPE_RMS,
|
||||
hparams.ffn_op,
|
||||
nullptr, // pos embd is already handled above
|
||||
add_pos);
|
||||
|
||||
// Gemma4VisionPooler
|
||||
{
|
||||
const int kernel_size = hparams.n_merge;
|
||||
GGML_ASSERT(kernel_size > 0);
|
||||
|
||||
// [n_embd, n_patches] -> [n_patches_x, n_patches_y, n_embd, n_batch]
|
||||
cur = ggml_cont_4d(ctx0, ggml_transpose(ctx0, cur), n_patches_x, n_patches_y, n_embd, n_batch);
|
||||
cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG,
|
||||
kernel_size, kernel_size, kernel_size, kernel_size, 0, 0);
|
||||
const int out_x = n_patches_x / kernel_size;
|
||||
const int out_y = n_patches_y / kernel_size;
|
||||
// [out_x, out_y, n_embd, n_batch] -> [n_embd, out_x * out_y, n_batch]
|
||||
cur = ggml_reshape_3d(ctx0, cur, out_x * out_y, n_embd, n_batch);
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = ggml_scale(ctx0, cur, sqrtf((float)n_embd));
|
||||
cb(cur, "pooled", -1);
|
||||
}
|
||||
|
||||
// hidden_states = (hidden_states - self.std_bias) * self.std_scale
|
||||
if (model.std_bias && model.std_scale) {
|
||||
cur = ggml_sub(ctx0, cur, model.std_bias);
|
||||
cur = ggml_mul(ctx0, cur, model.std_scale);
|
||||
cb(cur, "std_scaled", -1);
|
||||
}
|
||||
|
||||
// Gemma4MultimodalEmbedder
|
||||
{
|
||||
// embedding_pre_projection_norm
|
||||
cur = ggml_rms_norm(ctx0, cur, hparams.eps);
|
||||
cur = build_mm(model.mm_input_proj_w, cur);
|
||||
cb(cur, "projected", -1);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_gemma4v::build_mm(ggml_tensor * w, ggml_tensor * x) const {
|
||||
// Gemma4ClippableLinear
|
||||
|
||||
auto it = model.clamp_info_map.find(w->name);
|
||||
if (it == model.clamp_info_map.end()) {
|
||||
return ggml_mul_mat(ctx0, w, x);
|
||||
} else {
|
||||
const auto & clamp_info = it->second;
|
||||
ggml_tensor * clamped = ggml_clamp(ctx0, x, clamp_info.inp_min, clamp_info.inp_max);
|
||||
ggml_tensor * out = ggml_mul_mat(ctx0, w, clamped);
|
||||
out = ggml_clamp(ctx0, out, clamp_info.out_min, clamp_info.out_max);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_glm4v::build() {
|
||||
GGML_ASSERT(model.patch_bias != nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
|
||||
const int batch_size = 1;
|
||||
|
||||
norm_type norm_t = NORM_TYPE_RMS;
|
||||
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
|
||||
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
|
||||
GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
|
||||
|
||||
// second conv dimension
|
||||
{
|
||||
auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
|
||||
inp = ggml_add(ctx0, inp, inp_1);
|
||||
|
||||
inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b]
|
||||
inp = ggml_cont_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
inp = ggml_reshape_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
|
||||
inp = ggml_cont_3d(
|
||||
ctx0, inp,
|
||||
n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
|
||||
// add patch bias
|
||||
inp = ggml_add(ctx0, inp, model.patch_bias);
|
||||
cb(inp, "patch_bias", -1);
|
||||
|
||||
// pos-conv norm (absent in GLM-5.3-Flash)
|
||||
if (model.norm_embd_w) {
|
||||
inp = build_norm(inp, model.norm_embd_w, model.norm_embd_b, norm_t, eps, -1);
|
||||
}
|
||||
|
||||
ggml_tensor * learned_pos_embd = nullptr;
|
||||
// Note: GLM-OCR does not have learned position embeddings
|
||||
if (model.position_embeddings != nullptr) {
|
||||
learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BICUBIC);
|
||||
learned_pos_embd = ggml_cont_4d(
|
||||
ctx0, learned_pos_embd,
|
||||
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
learned_pos_embd = ggml_reshape_4d(
|
||||
ctx0, learned_pos_embd,
|
||||
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);
|
||||
learned_pos_embd = ggml_cont_3d(
|
||||
ctx0, learned_pos_embd,
|
||||
n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
cb(learned_pos_embd, "learned_pos_embd", -1);
|
||||
}
|
||||
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return ggml_rope_multi(
|
||||
ctx0, cur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
|
||||
32768, hparams.rope_theta, 1, 0, 1, 32, 1);
|
||||
};
|
||||
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
norm_t,
|
||||
hparams.ffn_op,
|
||||
learned_pos_embd,
|
||||
add_pos);
|
||||
|
||||
cb(cur, "vit_out", -1);
|
||||
// cb(ggml_sum(ctx0, cur), "vit_out_sum", -1);
|
||||
|
||||
// GLM4V projector
|
||||
// ref: https://github.com/huggingface/transformers/blob/40dc11cd3eb4126652aa41ef8272525affd4a636/src/transformers/models/glm4v/modeling_glm4v.py#L116-L130
|
||||
|
||||
// patch merger (downsample)
|
||||
{
|
||||
int n_merge = hparams.n_merge;
|
||||
GGML_ASSERT(n_merge > 0);
|
||||
|
||||
int n_token_out = n_patches / n_merge / n_merge;
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_embd, n_merge, n_merge, n_token_out);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3)); // [n_merge, n_merge, n_embd, n_token_out]
|
||||
cur = ggml_conv_2d(ctx0, model.mm_patch_merger_w, cur, n_merge, n_merge, 0, 0, 1, 1);
|
||||
cur = ggml_reshape_2d(ctx0, cur, cur->ne[2], n_token_out); // [n_embd_out, n_token_out]
|
||||
|
||||
cur = ggml_add(ctx0, cur, model.mm_patch_merger_b);
|
||||
}
|
||||
|
||||
// FC projector
|
||||
{
|
||||
cur = build_mm(model.mm_fc_w, cur);
|
||||
// default LayerNorm (post_projection_norm)
|
||||
cur = build_norm(cur, model.mm_post_norm_w, model.mm_post_norm_b, NORM_TYPE_NORMAL, 1e-5, -1);
|
||||
cur = ggml_gelu_erf(ctx0, cur);
|
||||
cb(cur, "after_fc_proj", -1);
|
||||
}
|
||||
|
||||
// FFN projector
|
||||
{
|
||||
cur = build_ffn(cur,
|
||||
model.mm_ffn_up_w, model.mm_ffn_up_b,
|
||||
model.mm_ffn_gate_w, model.mm_ffn_gate_b,
|
||||
model.mm_ffn_down_w, model.mm_ffn_down_b,
|
||||
hparams.ffn_op, -1);
|
||||
cb(cur, "after_ffn_proj", -1);
|
||||
// cb(ggml_sum(ctx0, cur), "merged_sum", -1);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
ggml_cgraph * clip_graph_granite_speech::build() {
|
||||
const int n_frames = img.nx();
|
||||
const int context_size = hparams.audio_chunk_size;
|
||||
const int ctc_layer = n_layer / 2;
|
||||
const int conv_kernel = hparams.audio_conv_kernel_size;
|
||||
const int conv_pad = conv_kernel / 2;
|
||||
|
||||
const int num_blocks = (n_frames + context_size - 1) / context_size;
|
||||
const int padded_len = num_blocks * context_size;
|
||||
const int remainder = n_frames % context_size;
|
||||
|
||||
// Calculate projector input dimension based on feature layers
|
||||
const int proj_input_dim = n_embd * (hparams.feature_layers.size() + 1);
|
||||
const bool use_feature_concat = !hparams.feature_layers.empty();
|
||||
|
||||
ggml_tensor * attn_dists = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, context_size * context_size);
|
||||
ggml_set_name(attn_dists, "attn_dists");
|
||||
ggml_set_input(attn_dists);
|
||||
|
||||
ggml_tensor * attn_mask = nullptr;
|
||||
if (remainder > 0) {
|
||||
attn_mask = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32,
|
||||
context_size, context_size, 1, num_blocks);
|
||||
ggml_set_name(attn_mask, "attn_mask");
|
||||
ggml_set_input(attn_mask);
|
||||
}
|
||||
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
auto * cur = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
cb(cur, "inp_transposed", -1);
|
||||
|
||||
cur = build_mm(model.inp_proj_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.inp_proj_b);
|
||||
cb(cur, "inp_linear", -1);
|
||||
|
||||
// Capture layer 0 if requested (after input_linear)
|
||||
ggml_tensor * concat_result = nullptr;
|
||||
if (use_feature_concat) {
|
||||
if (std::find(hparams.feature_layers.begin(), hparams.feature_layers.end(), 0) != hparams.feature_layers.end()) {
|
||||
concat_result = cur;
|
||||
cb(concat_result, "feature_layer_0", -1);
|
||||
}
|
||||
}
|
||||
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
const auto & layer = model.layers[il];
|
||||
auto * residual = cur;
|
||||
|
||||
// ffn1 (half-step)
|
||||
{
|
||||
auto * ffn1 = build_norm(cur, layer.ff_norm_w, layer.ff_norm_b,
|
||||
NORM_TYPE_NORMAL, eps, il);
|
||||
cb(ffn1, "ffn1_norm", il);
|
||||
|
||||
ffn1 = build_ffn(ffn1,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
nullptr, nullptr,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
FFN_SILU, il);
|
||||
cb(ffn1, "ffn1_out", il);
|
||||
|
||||
residual = ggml_add(ctx0, residual, ggml_scale(ctx0, ffn1, 0.5f));
|
||||
cb(residual, "ffn1_residual", il);
|
||||
}
|
||||
|
||||
// build_attn not used here: Shaw RPE needs pos_attn = mul_mat(pos_emb, Q)
|
||||
// injected between KQ product and softmax, which build_attn doesn't support
|
||||
{
|
||||
auto * normed = build_norm(residual, layer.ln_1_w, layer.ln_1_b,
|
||||
NORM_TYPE_NORMAL, eps, il);
|
||||
cb(normed, "attn_norm", il);
|
||||
|
||||
if (n_frames < padded_len) {
|
||||
normed = ggml_pad(ctx0, normed, 0, padded_len - n_frames, 0, 0);
|
||||
}
|
||||
|
||||
ggml_tensor * Q = build_mm(layer.q_w, normed);
|
||||
ggml_tensor * K = build_mm(layer.k_w, normed);
|
||||
ggml_tensor * V = build_mm(layer.v_w, normed);
|
||||
|
||||
Q = ggml_reshape_4d(ctx0, Q, d_head, n_head, context_size, num_blocks);
|
||||
K = ggml_reshape_4d(ctx0, K, d_head, n_head, context_size, num_blocks);
|
||||
V = ggml_reshape_4d(ctx0, V, d_head, n_head, context_size, num_blocks);
|
||||
|
||||
ggml_tensor * Q_perm = ggml_permute(ctx0, Q, 0, 2, 1, 3);
|
||||
ggml_tensor * K_perm = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3));
|
||||
|
||||
ggml_tensor * kq = ggml_mul_mat(ctx0, K_perm, Q_perm);
|
||||
|
||||
// Shaw RPE: pos_emb ne[2]=1 broadcasts against Q ne[2]=num_blocks in mul_mat
|
||||
ggml_tensor * pos_emb = ggml_get_rows(ctx0, layer.attn_rel_pos_emb, attn_dists);
|
||||
pos_emb = ggml_reshape_3d(ctx0, pos_emb, d_head, context_size, context_size);
|
||||
pos_emb = ggml_reshape_4d(ctx0, pos_emb, d_head, context_size, 1, context_size);
|
||||
|
||||
ggml_tensor * Q_shaw = ggml_permute(ctx0, Q, 0, 1, 3, 2);
|
||||
ggml_tensor * pos_attn = ggml_mul_mat(ctx0, pos_emb, Q_shaw);
|
||||
pos_attn = ggml_cont(ctx0, ggml_permute(ctx0, pos_attn, 0, 2, 3, 1));
|
||||
|
||||
ggml_tensor * scores = ggml_add(ctx0, kq, pos_attn);
|
||||
ggml_tensor * attn_weights = ggml_soft_max_ext(ctx0, scores, attn_mask,
|
||||
kq_scale, 0.0f);
|
||||
|
||||
ggml_tensor * V_perm = ggml_cont(ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3));
|
||||
ggml_tensor * attn_out = ggml_mul_mat(ctx0, V_perm, attn_weights);
|
||||
|
||||
attn_out = ggml_permute(ctx0, attn_out, 0, 2, 1, 3);
|
||||
attn_out = ggml_cont_2d(ctx0, attn_out, n_embd, padded_len);
|
||||
|
||||
if (n_frames < padded_len) {
|
||||
attn_out = ggml_view_2d(ctx0, attn_out,
|
||||
n_embd, n_frames, attn_out->nb[1], 0);
|
||||
}
|
||||
|
||||
cur = build_mm(layer.o_w, attn_out);
|
||||
cur = ggml_add(ctx0, cur, layer.o_b);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
residual = ggml_add(ctx0, residual, cur);
|
||||
|
||||
// conv module
|
||||
{
|
||||
cur = build_norm(residual, layer.norm_conv_w, layer.norm_conv_b,
|
||||
NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "conv_norm", il);
|
||||
|
||||
auto * x = build_mm(layer.conv_pw1_w, cur);
|
||||
x = ggml_add(ctx0, x, layer.conv_pw1_b);
|
||||
cb(x, "conv_pw1", il);
|
||||
|
||||
// GLU: ggml has no fused op, manual split + sigmoid gate
|
||||
{
|
||||
int64_t d = x->ne[0] / 2;
|
||||
ggml_tensor * gate = ggml_sigmoid(ctx0,
|
||||
ggml_view_2d(ctx0, x, d, x->ne[1], x->nb[1], d * x->nb[0]));
|
||||
x = ggml_mul(ctx0,
|
||||
ggml_view_2d(ctx0, x, d, x->ne[1], x->nb[1], 0), gate);
|
||||
x = ggml_cont(ctx0, ggml_transpose(ctx0, x));
|
||||
}
|
||||
cb(x, "conv_glu", il);
|
||||
|
||||
x = ggml_pad_ext(ctx0, x, conv_pad, conv_pad, 0, 0, 0, 0, 0, 0);
|
||||
x = ggml_ssm_conv(ctx0, x, layer.conv_dw_w);
|
||||
cb(x, "conv_dw", il);
|
||||
|
||||
// folded batch norm
|
||||
x = ggml_add(ctx0, ggml_mul(ctx0, x, layer.conv_norm_w), layer.conv_norm_b);
|
||||
x = ggml_silu(ctx0, x);
|
||||
cb(x, "conv_bn_silu", il);
|
||||
|
||||
x = build_mm(layer.conv_pw2_w, x);
|
||||
x = ggml_add(ctx0, x, layer.conv_pw2_b);
|
||||
cb(x, "conv_pw2", il);
|
||||
|
||||
cur = x;
|
||||
}
|
||||
|
||||
residual = ggml_add(ctx0, residual, cur);
|
||||
|
||||
// ffn2 (half-step)
|
||||
{
|
||||
auto * ffn2 = build_norm(residual, layer.ff_norm_1_w, layer.ff_norm_1_b,
|
||||
NORM_TYPE_NORMAL, eps, il);
|
||||
cb(ffn2, "ffn2_norm", il);
|
||||
|
||||
ffn2 = build_ffn(ffn2,
|
||||
layer.ff_up_1_w, layer.ff_up_1_b,
|
||||
nullptr, nullptr,
|
||||
layer.ff_down_1_w, layer.ff_down_1_b,
|
||||
FFN_SILU, il);
|
||||
cb(ffn2, "ffn2_out", il);
|
||||
|
||||
residual = ggml_add(ctx0, residual, ggml_scale(ctx0, ffn2, 0.5f));
|
||||
}
|
||||
|
||||
cur = build_norm(residual, layer.ln_2_w, layer.ln_2_b,
|
||||
NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "layer_out", il);
|
||||
|
||||
// Capture intermediate layer (il + 1) if requested
|
||||
if (use_feature_concat) {
|
||||
if (hparams.is_feature_layer(il + 1)) {
|
||||
if (concat_result == nullptr) {
|
||||
concat_result = cur;
|
||||
} else {
|
||||
concat_result = ggml_concat(ctx0, concat_result, cur, 0);
|
||||
}
|
||||
cb(concat_result, string_format("feature_layer_%d", il + 1).c_str(), il);
|
||||
}
|
||||
}
|
||||
|
||||
// CTC branch
|
||||
if (il + 1 == ctc_layer) {
|
||||
auto * mid = build_mm(model.ctc_out_w, cur);
|
||||
mid = ggml_add(ctx0, mid, model.ctc_out_b);
|
||||
mid = ggml_soft_max(ctx0, mid);
|
||||
mid = build_mm(model.ctc_out_mid_w, mid);
|
||||
mid = ggml_add(ctx0, mid, model.ctc_out_mid_b);
|
||||
cur = ggml_add(ctx0, cur, mid);
|
||||
cb(cur, "ctc_branch", il);
|
||||
}
|
||||
}
|
||||
|
||||
// Append final output to concatenated features if using feature concatenation
|
||||
if (use_feature_concat && concat_result != nullptr) {
|
||||
concat_result = ggml_concat(ctx0, concat_result, cur, 0);
|
||||
cb(concat_result, "concat_final", -1);
|
||||
cur = concat_result;
|
||||
}
|
||||
|
||||
cb(cur, "encoder_out", -1);
|
||||
|
||||
// QFormer projector
|
||||
{
|
||||
const int window_size = hparams.audio_proj_window_size;
|
||||
const int num_queries = window_size / hparams.audio_proj_downsample_rate;
|
||||
const int proj_n_head = hparams.audio_proj_head_count;
|
||||
const int proj_d_head = n_embd / proj_n_head;
|
||||
const float proj_kq_scale = 1.0f / sqrtf((float)proj_d_head);
|
||||
const float proj_eps = 1e-12f;
|
||||
const int nblocks_proj = (n_frames + window_size - 1) / window_size;
|
||||
const int padded_proj = nblocks_proj * window_size;
|
||||
|
||||
if (n_frames < padded_proj) {
|
||||
cur = ggml_pad(ctx0, cur, 0, padded_proj - n_frames, 0, 0);
|
||||
}
|
||||
|
||||
ggml_tensor * enc_windows = ggml_reshape_3d(ctx0, cur, proj_input_dim, window_size, nblocks_proj);
|
||||
|
||||
ggml_tensor * queries = build_norm(model.qf_proj_blocks[0].qf_proj_query,
|
||||
model.qf_proj_blocks[0].qf_proj_norm_w, model.qf_proj_blocks[0].qf_proj_norm_b,
|
||||
NORM_TYPE_NORMAL, proj_eps, -1);
|
||||
{
|
||||
ggml_tensor * q_3d = ggml_reshape_3d(ctx0, queries, n_embd, num_queries, 1);
|
||||
ggml_tensor * q_shape = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32,
|
||||
n_embd, num_queries, nblocks_proj);
|
||||
queries = ggml_repeat(ctx0, q_3d, q_shape);
|
||||
}
|
||||
|
||||
for (int il = 0; il < (int)model.qf_proj_blocks[0].qf_proj_layers.size(); il++) {
|
||||
const auto & pl = model.qf_proj_blocks[0].qf_proj_layers[il];
|
||||
|
||||
// self-attention
|
||||
{
|
||||
ggml_tensor * Q = ggml_add(ctx0, build_mm(pl.q_w, queries), pl.q_b);
|
||||
ggml_tensor * K = ggml_add(ctx0, build_mm(pl.k_w, queries), pl.k_b);
|
||||
ggml_tensor * V = ggml_add(ctx0, build_mm(pl.v_w, queries), pl.v_b);
|
||||
|
||||
Q = ggml_reshape_4d(ctx0, Q, proj_d_head, proj_n_head, num_queries, nblocks_proj);
|
||||
K = ggml_reshape_4d(ctx0, K, proj_d_head, proj_n_head, num_queries, nblocks_proj);
|
||||
V = ggml_reshape_4d(ctx0, V, proj_d_head, proj_n_head, num_queries, nblocks_proj);
|
||||
|
||||
ggml_tensor * sa_out = build_attn(pl.o_w, pl.o_b,
|
||||
Q, K, V, nullptr, proj_kq_scale, il);
|
||||
sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, num_queries, nblocks_proj);
|
||||
|
||||
queries = build_norm(ggml_add(ctx0, sa_out, queries),
|
||||
pl.ln_1_w, pl.ln_1_b,
|
||||
NORM_TYPE_NORMAL, proj_eps, il);
|
||||
}
|
||||
|
||||
// cross-attention
|
||||
{
|
||||
ggml_tensor * Q = ggml_add(ctx0, build_mm(pl.cross_attn_q_w, queries), pl.cross_attn_q_b);
|
||||
ggml_tensor * K = ggml_add(ctx0, build_mm(pl.cross_attn_k_w, enc_windows), pl.cross_attn_k_b);
|
||||
ggml_tensor * V = ggml_add(ctx0, build_mm(pl.cross_attn_v_w, enc_windows), pl.cross_attn_v_b);
|
||||
|
||||
Q = ggml_reshape_4d(ctx0, Q, proj_d_head, proj_n_head, num_queries, nblocks_proj);
|
||||
K = ggml_reshape_4d(ctx0, K, proj_d_head, proj_n_head, window_size, nblocks_proj);
|
||||
V = ggml_reshape_4d(ctx0, V, proj_d_head, proj_n_head, window_size, nblocks_proj);
|
||||
|
||||
ggml_tensor * ca_out = build_attn(pl.cross_attn_o_w, pl.cross_attn_o_b,
|
||||
Q, K, V, nullptr, proj_kq_scale, il);
|
||||
ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, num_queries, nblocks_proj);
|
||||
|
||||
queries = build_norm(ggml_add(ctx0, ca_out, queries),
|
||||
pl.cross_attn_norm_w, pl.cross_attn_norm_b,
|
||||
NORM_TYPE_NORMAL, proj_eps, il);
|
||||
}
|
||||
|
||||
// ffn
|
||||
{
|
||||
ggml_tensor * ffn_out = build_ffn(queries,
|
||||
pl.ff_up_w, pl.ff_up_b,
|
||||
nullptr, nullptr,
|
||||
pl.ff_down_w, pl.ff_down_b,
|
||||
FFN_GELU, il);
|
||||
|
||||
queries = build_norm(ggml_add(ctx0, ffn_out, queries),
|
||||
pl.ln_2_w, pl.ln_2_b,
|
||||
NORM_TYPE_NORMAL, proj_eps, il);
|
||||
}
|
||||
}
|
||||
|
||||
cur = ggml_reshape_2d(ctx0, queries, n_embd, num_queries * nblocks_proj);
|
||||
cur = ggml_add(ctx0, build_mm(model.qf_proj_blocks[0].qf_proj_linear_w, cur), model.qf_proj_blocks[0].qf_proj_linear_b);
|
||||
cb(cur, "projector_out", -1);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,397 @@
|
||||
#include "models.h"
|
||||
#include "../clip-impl.h"
|
||||
#include "../clip-model.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
/*
|
||||
* Granite Vision 4.1 clip graph
|
||||
*
|
||||
* Stage 1a: SigLIP vision tower (N layers, post-norm)
|
||||
* Stage 1b: WindowQFormer blocks (deepstack + spatial)
|
||||
* Stage 1c: Concatenate and pack outputs
|
||||
* Stage 1d: Assemble the anyres tiles into one token sequence
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Member method implementations
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// split the stacked tiles into the batch axis, then run the usual patch embedding
|
||||
ggml_tensor * clip_graph_granite4_vision::build_tile_inp() {
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
if (n_tiles > 1) {
|
||||
const int px = img.nx();
|
||||
inp_raw = ggml_reshape_4d(ctx0, inp_raw, px * px, n_tiles, 3, 1);
|
||||
inp_raw = ggml_cont(ctx0, ggml_permute(ctx0, inp_raw, 0, 2, 1, 3));
|
||||
inp_raw = ggml_reshape_4d(ctx0, inp_raw, px, px, 3, n_tiles);
|
||||
}
|
||||
|
||||
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
|
||||
inp = ggml_reshape_3d(ctx0, inp, tile_side * tile_side, n_embd, n_tiles);
|
||||
inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
if (model.patch_bias) {
|
||||
inp = ggml_add(ctx0, inp, model.patch_bias);
|
||||
}
|
||||
return inp;
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_granite4_vision::gather(
|
||||
ggml_tensor * src,
|
||||
const std::string & name,
|
||||
int idx_len) {
|
||||
// one index row per tile, all rows hold the same permutation
|
||||
ggml_tensor * idx = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, idx_len, n_tiles);
|
||||
ggml_set_name(idx, name.c_str());
|
||||
ggml_set_input(idx);
|
||||
return ggml_get_rows(ctx0, src, idx);
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_granite4_vision::interp_down(
|
||||
ggml_tensor * src,
|
||||
int side,
|
||||
int new_side) {
|
||||
const int n_embd = src->ne[0];
|
||||
ggml_tensor * t = ggml_reshape_4d(ctx0, src, n_embd, side, side, n_tiles);
|
||||
t = ggml_cont(ctx0, ggml_permute(ctx0, t, 2, 0, 1, 3));
|
||||
// fold the tile axis into the channel axis, ggml_pool_2d only pools the first two axes
|
||||
t = ggml_reshape_3d(ctx0, t, side, side, n_embd * n_tiles);
|
||||
const int kernel = side / new_side;
|
||||
t = ggml_pool_2d(ctx0, t, GGML_OP_POOL_AVG, kernel, kernel, kernel, kernel, 0, 0);
|
||||
t = ggml_reshape_4d(ctx0, t, new_side, new_side, n_embd, n_tiles);
|
||||
t = ggml_cont(ctx0, ggml_permute(ctx0, t, 1, 2, 0, 3));
|
||||
return ggml_reshape_3d(ctx0, t, n_embd, new_side * new_side, n_tiles);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// build_block - WindowQFormer block implementation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
ggml_tensor * clip_graph_granite4_vision::build_block(
|
||||
const qf_block & blk,
|
||||
ggml_tensor * h,
|
||||
int bid,
|
||||
int spatial_offset,
|
||||
int image_side,
|
||||
int window_side,
|
||||
int query_side,
|
||||
float qformer_eps) {
|
||||
|
||||
const int n_embd = h->ne[0];
|
||||
GGML_ASSERT(h->ne[1] == image_side * image_side);
|
||||
const int n = image_side / window_side;
|
||||
const int new_side = n * query_side;
|
||||
const int n_windows = n * n;
|
||||
const int n_win_all = n_windows * n_tiles; // windows of every tile, batched together
|
||||
const int enc_len = window_side * window_side;
|
||||
const int query_len = query_side * query_side;
|
||||
|
||||
auto cbx = [&](ggml_tensor * & t, const char * step) {
|
||||
const std::string name = "g4v_blk" + std::to_string(bid) + "_" + step;
|
||||
ggml_set_name(t, name.c_str());
|
||||
};
|
||||
|
||||
// 1. Top-level LN
|
||||
cbx(h, "inp");
|
||||
ggml_tensor * x = build_norm(h, blk.qf_proj_norm_w, blk.qf_proj_norm_b, NORM_TYPE_NORMAL, eps, bid);
|
||||
cbx(x, "norm");
|
||||
|
||||
// 2. enc = _win(x, image_side, window_side)
|
||||
ggml_tensor * enc;
|
||||
{
|
||||
ggml_tensor * enc_flat = gather(x,
|
||||
"g4v_blk" + std::to_string(bid) + "_win_idx",
|
||||
image_side * image_side);
|
||||
enc = ggml_reshape_3d(ctx0, enc_flat, n_embd, enc_len, n_win_all);
|
||||
}
|
||||
cbx(enc, "enc");
|
||||
|
||||
// 3. downsampled = downsampler(x)
|
||||
ggml_tensor * d;
|
||||
(void) spatial_offset;
|
||||
if (spatial_offset >= 0) {
|
||||
d = gather(x,
|
||||
"g4v_blk" + std::to_string(bid) + "_spatial_idx",
|
||||
new_side * new_side);
|
||||
} else {
|
||||
d = interp_down(x, image_side, new_side);
|
||||
}
|
||||
cbx(d, "downsampled");
|
||||
|
||||
// 4. query_embeds = query + _win(d, new_side, query_side)
|
||||
ggml_tensor * q_in;
|
||||
{
|
||||
ggml_tensor * dw_flat = gather(d,
|
||||
"g4v_blk" + std::to_string(bid) + "_qwin_idx",
|
||||
new_side * new_side);
|
||||
ggml_tensor * dw = ggml_reshape_3d(ctx0, dw_flat, n_embd, query_len, n_win_all);
|
||||
q_in = ggml_add(ctx0, dw, blk.qf_proj_query);
|
||||
}
|
||||
cbx(q_in, "query_embeds");
|
||||
|
||||
// 5. encoder_embeds = enc + image_positions → (C, enc_len, n_windows)
|
||||
ggml_tensor * e_in = ggml_add(ctx0, enc, blk.qf_proj_img_pos);
|
||||
cbx(e_in, "encoder_embeds");
|
||||
|
||||
// 6. Qformer forward.
|
||||
ggml_tensor * q = build_norm(q_in, blk.qf_proj_post_norm_w, blk.qf_proj_post_norm_b, NORM_TYPE_NORMAL, qformer_eps, bid);
|
||||
|
||||
// Helper for linear projections with window batching
|
||||
auto linear = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) -> ggml_tensor * {
|
||||
ggml_tensor * t = ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1] * x->ne[2]);
|
||||
t = build_mm(w, t);
|
||||
if (b) t = ggml_add(ctx0, t, b);
|
||||
return t;
|
||||
};
|
||||
|
||||
// Get the single QFormer layer
|
||||
GGML_ASSERT(blk.qf_proj_layers.size() == 1);
|
||||
const auto & pl = blk.qf_proj_layers[0];
|
||||
|
||||
// 6a. Self-attention
|
||||
ggml_tensor * sa_out;
|
||||
{
|
||||
const int d_h = 64;
|
||||
const int n_head = n_embd / d_h;
|
||||
const int nq = q->ne[1];
|
||||
const float scale = 1.0f / std::sqrt((float) d_h);
|
||||
|
||||
ggml_tensor * Q = linear(q, pl.q_w, pl.q_b);
|
||||
ggml_tensor * K = linear(q, pl.k_w, pl.k_b);
|
||||
ggml_tensor * V = linear(q, pl.v_w, pl.v_b);
|
||||
|
||||
Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_win_all);
|
||||
K = ggml_reshape_4d(ctx0, K, d_h, n_head, nq, n_win_all);
|
||||
V = ggml_reshape_4d(ctx0, V, d_h, n_head, nq, n_win_all);
|
||||
|
||||
sa_out = build_attn(pl.o_w, pl.o_b, Q, K, V, nullptr, scale, bid);
|
||||
sa_out = ggml_reshape_3d(ctx0, sa_out, n_embd, nq, n_win_all);
|
||||
|
||||
sa_out = ggml_add(ctx0, sa_out, q);
|
||||
sa_out = build_norm(sa_out, pl.ln_1_w, pl.ln_1_b,
|
||||
NORM_TYPE_NORMAL, qformer_eps, bid);
|
||||
}
|
||||
cbx(sa_out, "sa_out");
|
||||
|
||||
// 6b. Cross-attention
|
||||
ggml_tensor * ca_out;
|
||||
{
|
||||
const int d_h = 64;
|
||||
const int n_head = n_embd / d_h;
|
||||
const int nq = sa_out->ne[1];
|
||||
const int nkv = e_in->ne[1];
|
||||
const float scale = 1.0f / std::sqrt((float) d_h);
|
||||
|
||||
ggml_tensor * Q = linear(sa_out, pl.cross_attn_q_w, pl.cross_attn_q_b);
|
||||
ggml_tensor * K = linear(e_in, pl.cross_attn_k_w, pl.cross_attn_k_b);
|
||||
ggml_tensor * V = linear(e_in, pl.cross_attn_v_w, pl.cross_attn_v_b);
|
||||
|
||||
Q = ggml_reshape_4d(ctx0, Q, d_h, n_head, nq, n_win_all);
|
||||
K = ggml_reshape_4d(ctx0, K, d_h, n_head, nkv, n_win_all);
|
||||
V = ggml_reshape_4d(ctx0, V, d_h, n_head, nkv, n_win_all);
|
||||
|
||||
ca_out = build_attn(pl.cross_attn_o_w, pl.cross_attn_o_b,
|
||||
Q, K, V, nullptr, scale, bid);
|
||||
ca_out = ggml_reshape_3d(ctx0, ca_out, n_embd, nq, n_win_all);
|
||||
|
||||
ca_out = ggml_add(ctx0, ca_out, sa_out);
|
||||
ca_out = build_norm(ca_out, pl.cross_attn_norm_w, pl.cross_attn_norm_b,
|
||||
NORM_TYPE_NORMAL, qformer_eps, bid);
|
||||
}
|
||||
cbx(ca_out, "ca_out");
|
||||
|
||||
// 6c. FFN
|
||||
ggml_tensor * ffn;
|
||||
{
|
||||
ggml_tensor * t = ggml_reshape_2d(ctx0, ca_out, n_embd, query_len * n_win_all);
|
||||
t = build_mm(pl.ff_up_w, t);
|
||||
if (pl.ff_up_b) t = ggml_add(ctx0, t, pl.ff_up_b);
|
||||
t = ggml_gelu_erf(ctx0, t);
|
||||
t = build_mm(pl.ff_down_w, t);
|
||||
if (pl.ff_down_b) t = ggml_add(ctx0, t, pl.ff_down_b);
|
||||
t = ggml_reshape_3d(ctx0, t, n_embd, query_len, n_win_all);
|
||||
ffn = ggml_add(ctx0, t, ca_out);
|
||||
ffn = build_norm(ffn, pl.ln_2_w, pl.ln_2_b, NORM_TYPE_NORMAL, qformer_eps, bid);
|
||||
}
|
||||
cbx(ffn, "qformer_out");
|
||||
|
||||
// 7. _unwin back to raster
|
||||
ggml_tensor * unwinned;
|
||||
{
|
||||
ggml_tensor * flat = ggml_reshape_3d(ctx0, ffn, n_embd, query_len * n_windows, n_tiles);
|
||||
unwinned = gather(flat,
|
||||
"g4v_blk" + std::to_string(bid) + "_unwin_idx",
|
||||
new_side * new_side);
|
||||
}
|
||||
cbx(unwinned, "unwin");
|
||||
|
||||
// 8. out_linear
|
||||
ggml_tensor * out = build_mm(blk.qf_proj_linear_w, unwinned);
|
||||
if (blk.qf_proj_linear_b) out = ggml_add(ctx0, out, blk.qf_proj_linear_b);
|
||||
cbx(out, "out");
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// build() - top-level graph
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Build the K-tiled, base-scaled newline row tensor.
|
||||
// Shape: (n_mmproj_embd, 1)
|
||||
ggml_tensor * clip_graph_granite4_vision::build_newline_row(ggml_context * ctx0) {
|
||||
const int K = (int) model.qf_proj_blocks.size();
|
||||
GGML_ASSERT(K > 0);
|
||||
GGML_ASSERT(n_mmproj_embd % K == 0);
|
||||
const int projection_dim = n_mmproj_embd / K;
|
||||
GGML_ASSERT(model.image_newline != nullptr);
|
||||
GGML_ASSERT(ggml_nelements(model.image_newline) == projection_dim);
|
||||
|
||||
// Build newline_row[k*projection_dim + d] = nl[d] * (k == 0 ? base : 1.0)
|
||||
ggml_tensor * nl = model.image_newline; // (projection_dim,)
|
||||
ggml_tensor * nl_first_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1);
|
||||
ggml_tensor * nl_row_2d;
|
||||
if (K == 1) {
|
||||
nl_row_2d = nl_first_2d;
|
||||
} else {
|
||||
ggml_tensor * nl_2d = ggml_reshape_2d(ctx0, nl, projection_dim, 1);
|
||||
ggml_tensor * rest_template = ggml_new_tensor_2d(
|
||||
ctx0, GGML_TYPE_F32, projection_dim, K - 1);
|
||||
ggml_tensor * nl_rest = ggml_repeat(ctx0, nl_2d, rest_template);
|
||||
nl_row_2d = ggml_concat(ctx0, nl_first_2d, nl_rest, 1); // (projection_dim, K)
|
||||
}
|
||||
nl_row_2d = ggml_cont(ctx0, nl_row_2d);
|
||||
return ggml_reshape_2d(ctx0, nl_row_2d, n_mmproj_embd, 1);
|
||||
}
|
||||
|
||||
// Assemble [overview, tile(0,0), tile(0,1), ...] into one token sequence:
|
||||
// the overview tokens first, then the tile grid read in raster order with one newline per row.
|
||||
// ref: https://github.com/huggingface/transformers/blob/v5.0.0/src/transformers/models/llava_next/modeling_llava_next.py#L266
|
||||
ggml_tensor * clip_graph_granite4_vision::build_anyres_assembly(ggml_tensor * cur, int out_side) {
|
||||
const int n_dim = cur->ne[0];
|
||||
const int grid_x = anyres.grid_x;
|
||||
const int grid_y = anyres.grid_y;
|
||||
const int cur_w = grid_x * out_side;
|
||||
const int cur_h = grid_y * out_side;
|
||||
GGML_ASSERT(cur->ne[1] == out_side * out_side);
|
||||
GGML_ASSERT(cur->ne[2] == 1 + grid_x * grid_y);
|
||||
|
||||
ggml_tensor * base = ggml_view_2d(ctx0, cur, n_dim, out_side * out_side, cur->nb[1], 0);
|
||||
|
||||
ggml_tensor * tiles = ggml_view_3d(ctx0, cur, n_dim, out_side * out_side, grid_x * grid_y,
|
||||
cur->nb[1], cur->nb[2], cur->nb[2]);
|
||||
|
||||
// (n_dim*out_side, out_side, grid_x, grid_y) -> interleave the tiles of a grid row
|
||||
tiles = ggml_reshape_4d(ctx0, tiles, n_dim * out_side, out_side, grid_x, grid_y);
|
||||
tiles = ggml_cont(ctx0, ggml_permute(ctx0, tiles, 0, 2, 1, 3));
|
||||
tiles = ggml_reshape_3d(ctx0, tiles, n_dim, cur_w, cur_h);
|
||||
|
||||
// drop the tokens that only cover the padding added when resizing to the grid
|
||||
int off_x, off_y, w, h;
|
||||
clip_anyres_unpad(cur_w, cur_h, anyres.orig_nx, anyres.orig_ny, off_x, off_y, w, h);
|
||||
if (w != cur_w || h != cur_h) {
|
||||
tiles = ggml_cont(ctx0, ggml_view_3d(ctx0, tiles, n_dim, w, h,
|
||||
tiles->nb[1], tiles->nb[2],
|
||||
off_x * tiles->nb[1] + off_y * tiles->nb[2]));
|
||||
}
|
||||
|
||||
ggml_tensor * nl = ggml_repeat_4d(ctx0, build_newline_row(ctx0), n_dim, 1, h, 1);
|
||||
tiles = ggml_concat(ctx0, tiles, nl, 1);
|
||||
tiles = ggml_reshape_2d(ctx0, tiles, n_dim, (w + 1) * h);
|
||||
|
||||
return ggml_concat(ctx0, base, tiles, 1);
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_granite4_vision::build() {
|
||||
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
GGML_ASSERT(!model.qf_proj_blocks.empty());
|
||||
|
||||
// --- Stage 1a: SigLIP encoder producing intermediate hidden states ---
|
||||
ggml_tensor * inp = build_tile_inp();
|
||||
inp = ggml_add(ctx0, inp, model.position_embeddings);
|
||||
cb(inp, "pos_embed", -1);
|
||||
|
||||
const int tile_n_patches = tile_side * tile_side;
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
std::vector<ggml_tensor *> layer_outs(n_layer, nullptr);
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const auto & layer = model.layers[il];
|
||||
ggml_tensor * cur = inpL;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
|
||||
// Self-attention
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
if (layer.q_b) Qcur = ggml_add(ctx0, Qcur, layer.q_b);
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
if (layer.k_b) Kcur = ggml_add(ctx0, Kcur, layer.k_b);
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
if (layer.v_b) Vcur = ggml_add(ctx0, Vcur, layer.v_b);
|
||||
|
||||
Qcur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, tile_n_patches, n_tiles);
|
||||
Kcur = ggml_reshape_4d(ctx0, Kcur, d_head, n_head, tile_n_patches, n_tiles);
|
||||
Vcur = ggml_reshape_4d(ctx0, Vcur, d_head, n_head, tile_n_patches, n_tiles);
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b,
|
||||
Qcur, Kcur, Vcur, nullptr, kq_scale, il);
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_embd, tile_n_patches, n_tiles);
|
||||
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
inpL = cur;
|
||||
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
layer_outs[il] = cur;
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
// --- Stage 1b/1c: WindowQFormer blocks ---
|
||||
const int projector_count = hparams.feature_layers.size();
|
||||
const float qformer_eps = 1e-12f;
|
||||
|
||||
ggml_tensor * mmproj = nullptr;
|
||||
for (int bid = 0; bid < projector_count; ++bid) {
|
||||
const auto & blk = model.qf_proj_blocks[bid];
|
||||
|
||||
int vlayer = hparams.feature_layers[bid];
|
||||
GGML_ASSERT(vlayer >= 0 && vlayer < n_layer);
|
||||
ggml_tensor * h = layer_outs[vlayer];
|
||||
|
||||
ggml_tensor * stream = build_block(
|
||||
blk, h, bid,
|
||||
hparams.proj_spatial_offsets[bid],
|
||||
tile_side,
|
||||
hparams.downsample_window_side,
|
||||
hparams.downsample_query_side,
|
||||
qformer_eps);
|
||||
cb(stream, (std::string("proj_") + std::to_string(bid) + std::string("_v_out")).c_str(), vlayer);
|
||||
mmproj = mmproj ? ggml_concat(ctx0, mmproj, stream, 0) : stream;
|
||||
}
|
||||
|
||||
// --- Stage 1d: assemble the tiles and weave in the newline tokens ---
|
||||
if (anyres.is_tiled()) {
|
||||
const int out_side = tile_side / hparams.downsample_window_side * hparams.downsample_query_side;
|
||||
mmproj = build_anyres_assembly(mmproj, out_side);
|
||||
ggml_set_name(mmproj, "g4v_mmproj_out_anyres");
|
||||
} else {
|
||||
ggml_set_name(mmproj, "g4v_mmproj_out");
|
||||
}
|
||||
ggml_build_forward_expand(gf, mmproj);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_hunyuanvl::build() {
|
||||
const int merge = hparams.n_merge;
|
||||
const int pw = n_patches_x;
|
||||
const int ph = n_patches_y;
|
||||
|
||||
// position embedding: declared as a graph input, filled on CPU
|
||||
// by clip_image_batch_encode (see PROJECTOR_TYPE_HUNYUANVL branch there).
|
||||
ggml_tensor * pos_embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, ph * pw);
|
||||
ggml_set_name(pos_embd, "hunyuanvl_pos_embd");
|
||||
ggml_set_input(pos_embd);
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * cur = build_vit(inp, n_patches, NORM_TYPE_NORMAL, hparams.ffn_op, pos_embd, nullptr);
|
||||
|
||||
// perceiver projector
|
||||
cur = build_norm(cur, model.mm_pre_norm_w, nullptr, NORM_TYPE_RMS, eps, -1);
|
||||
|
||||
// [C, W*H] -> [W, H, C] for conv2d
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_embd, pw, ph);
|
||||
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3);
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
// Conv2d(1152->2304, k=2, s=2) + GELU + Conv2d(2304->4608, k=1, s=1)
|
||||
cur = ggml_conv_2d(ctx0, model.mm_0_w, cur, merge, merge, 0, 0, 1, 1);
|
||||
if (model.mm_0_b) {
|
||||
cur = ggml_add(ctx0, cur, ggml_reshape_3d(ctx0, model.mm_0_b, 1, 1, model.mm_0_b->ne[0]));
|
||||
}
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
cur = ggml_conv_2d(ctx0, model.mm_1_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
if (model.mm_1_b) {
|
||||
cur = ggml_add(ctx0, cur, ggml_reshape_3d(ctx0, model.mm_1_b, 1, 1, model.mm_1_b->ne[0]));
|
||||
}
|
||||
|
||||
const int ow = pw / merge;
|
||||
const int oh = ph / merge;
|
||||
const int idim = (int)cur->ne[2]; // OC = 4608
|
||||
|
||||
// append newline along W (dim 0)
|
||||
ggml_tensor * nl = ggml_reshape_4d(ctx0, model.image_newline, 1, 1, idim, 1);
|
||||
nl = ggml_repeat_4d(ctx0, nl, 1, oh, idim, 1);
|
||||
cur = ggml_concat(ctx0, cur, nl, 0);
|
||||
|
||||
// [OW+1, OH, OC] -> [OC, (OW+1)*OH]
|
||||
cur = ggml_permute(ctx0, cur, 1, 2, 0, 3);
|
||||
cur = ggml_cont_2d(ctx0, cur, idim, (ow + 1) * oh);
|
||||
|
||||
// project to LLM hidden size
|
||||
cur = build_mm(model.mm_model_proj, cur);
|
||||
if (model.mm_model_proj_b) {
|
||||
cur = ggml_add(ctx0, cur, model.mm_model_proj_b);
|
||||
}
|
||||
|
||||
// wrap with begin/end tokens
|
||||
cur = ggml_concat(ctx0, ggml_reshape_2d(ctx0, model.mm_img_begin, model.mm_img_begin->ne[0], 1), cur, 1);
|
||||
cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, model.mm_img_end, model.mm_img_end->ne[0], 1), 1);
|
||||
|
||||
cur = build_norm(cur, model.mm_post_norm_w, nullptr, NORM_TYPE_RMS, eps, -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_internvl::build() {
|
||||
GGML_ASSERT(model.class_embedding != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
|
||||
const int n_pos = n_patches + 1;
|
||||
ggml_tensor * inp = build_inp();
|
||||
|
||||
// add CLS token
|
||||
ggml_tensor * cls_repeated = ggml_repeat_4d(ctx0, model.class_embedding,
|
||||
model.class_embedding->ne[0], 1, n_batch, 1);
|
||||
inp = ggml_concat(ctx0, inp, cls_repeated, 1);
|
||||
|
||||
// The larger models use a different ViT, which uses RMS norm instead of layer norm
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/13443#issuecomment-2869786188
|
||||
norm_type norm_t = (hparams.n_embd == 3200 && hparams.n_layer == 45)
|
||||
? NORM_TYPE_RMS // 6B ViT (Used by InternVL 2.5/3 - 26B, 38B, 78B)
|
||||
: NORM_TYPE_NORMAL; // 300M ViT (Used by all smaller InternVL models)
|
||||
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_pos,
|
||||
norm_t,
|
||||
hparams.ffn_op,
|
||||
model.position_embeddings,
|
||||
nullptr);
|
||||
|
||||
// remove CLS token
|
||||
cur = ggml_view_3d(ctx0, cur,
|
||||
n_embd, n_patches, n_batch,
|
||||
cur->nb[1], cur->nb[2], 0);
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
// pixel shuffle
|
||||
{
|
||||
const int scale_factor = model.hparams.n_merge;
|
||||
const int bsz = n_batch;
|
||||
const int height = n_patches_y;
|
||||
const int width = n_patches_x;
|
||||
GGML_ASSERT(scale_factor > 0);
|
||||
cur = ggml_reshape_4d(ctx0, cur, n_embd * scale_factor, height / scale_factor, width, bsz);
|
||||
cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
|
||||
cur = ggml_cont_4d(ctx0, cur,
|
||||
n_embd * scale_factor * scale_factor,
|
||||
height / scale_factor,
|
||||
width / scale_factor,
|
||||
bsz);
|
||||
cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
|
||||
// flatten to 2D
|
||||
cur = ggml_cont_3d(ctx0, cur,
|
||||
n_embd * scale_factor * scale_factor,
|
||||
cur->ne[1] * cur->ne[2],
|
||||
cur->ne[3]);
|
||||
}
|
||||
|
||||
// projector (always using GELU activation)
|
||||
{
|
||||
// projector LayerNorm uses pytorch's default eps = 1e-5
|
||||
// ref: https://huggingface.co/OpenGVLab/InternVL3-8B-Instruct/blob/a34d3e4e129a5856abfd6aa6de79776484caa14e/modeling_internvl_chat.py#L79
|
||||
cur = build_norm(cur, model.mm_0_w, model.mm_0_b, NORM_TYPE_NORMAL, 1e-5, -1);
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_3_w, model.mm_3_b,
|
||||
FFN_GELU,
|
||||
-1);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
#include "models.h"
|
||||
#include <cstring>
|
||||
#include <cmath>
|
||||
|
||||
// note: this is similar to clip_graph::resize_position_embeddings, major difference is having
|
||||
// the w/h in ne[1] and ne[2] instead of assuming with sqrt. Could try storing the tensor in 2D instead
|
||||
// with a w*h? Also the permute is a bit different at (2, 1, 0, 3) instead of (2, 0, 1, 3).
|
||||
ggml_tensor * clip_graph_kimik25::resize_position_embeddings_3d(uint32_t interpolation_mode) {
|
||||
ggml_tensor * pos_embd = model.position_embeddings;
|
||||
const int height = img.ny() / patch_size;
|
||||
const int width = img.nx() / patch_size;
|
||||
const uint32_t mode = interpolation_mode;
|
||||
|
||||
GGML_ASSERT(pos_embd);
|
||||
|
||||
const int64_t stored_c = pos_embd->ne[0]; // C = 1152
|
||||
const int64_t orig_w = pos_embd->ne[1]; // W = 64
|
||||
const int64_t orig_h = pos_embd->ne[2]; // H = 64
|
||||
|
||||
GGML_ASSERT(stored_c == n_embd);
|
||||
|
||||
if (height == (int)orig_h && width == (int)orig_w) {
|
||||
// No interpolation needed, just flatten to [C, H*W]
|
||||
return ggml_cont_2d(ctx0, pos_embd, n_embd, width * height);
|
||||
}
|
||||
|
||||
pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3);
|
||||
pos_embd = ggml_interpolate(ctx0, pos_embd, height, width, n_embd, 1, mode);
|
||||
pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3);
|
||||
pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height);
|
||||
return pos_embd;
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_kimik25::build() {
|
||||
ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_h, "pos_h");
|
||||
ggml_set_input(pos_h);
|
||||
|
||||
ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_w, "pos_w");
|
||||
ggml_set_input(pos_w);
|
||||
|
||||
ggml_tensor * learned_pos_embd = resize_position_embeddings_3d(GGML_SCALE_MODE_BICUBIC);
|
||||
|
||||
// Kimi-K2.5 uses interleaved 2D RoPE pattern natively, but
|
||||
// Q / K are permuted during conversion to use split format.
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
cur = build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
|
||||
return cur;
|
||||
};
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
|
||||
// I don't know why, but doing this in the build_vit lead to the ggml_add not occurring?
|
||||
// Doing it manually here does work.
|
||||
inp = ggml_add(ctx0, inp, learned_pos_embd);
|
||||
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
NORM_TYPE_NORMAL,
|
||||
hparams.ffn_op,
|
||||
nullptr,
|
||||
add_pos);
|
||||
|
||||
cb(cur, "vit_out", -1);
|
||||
|
||||
{
|
||||
// patch_merger
|
||||
const int scale_factor = model.hparams.n_merge;
|
||||
cur = build_patch_merge_permute(cur, scale_factor);
|
||||
|
||||
// projection norm
|
||||
int proj_inp_dim = cur->ne[0];
|
||||
int n_merged_patches = cur->ne[1];
|
||||
cur = ggml_view_2d(ctx0, cur,
|
||||
n_embd, n_merged_patches * scale_factor * scale_factor,
|
||||
ggml_row_size(cur->type, n_embd), 0);
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_mul(ctx0, cur, model.mm_input_norm_w);
|
||||
cur = ggml_add(ctx0, cur, model.mm_input_norm_b);
|
||||
cur = ggml_view_2d(ctx0, cur,
|
||||
proj_inp_dim, n_merged_patches,
|
||||
ggml_row_size(cur->type, proj_inp_dim), 0);
|
||||
cb(cur, "proj_inp_normed", -1);
|
||||
|
||||
// projection mlp
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU,
|
||||
-1);
|
||||
|
||||
cb(cur, "proj_out", -1);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_kimivl::build() {
|
||||
// 2D input positions
|
||||
ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_h, "pos_h");
|
||||
ggml_set_input(pos_h);
|
||||
|
||||
ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_w, "pos_w");
|
||||
ggml_set_input(pos_w);
|
||||
|
||||
ggml_tensor * learned_pos_embd = resize_position_embeddings();
|
||||
|
||||
// build ViT with 2D position embeddings
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
// first half is X axis and second half is Y axis
|
||||
return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
|
||||
};
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
NORM_TYPE_NORMAL,
|
||||
hparams.ffn_op,
|
||||
learned_pos_embd,
|
||||
add_pos);
|
||||
|
||||
cb(cur, "vit_out", -1);
|
||||
|
||||
{
|
||||
// patch_merger
|
||||
const int scale_factor = model.hparams.n_merge;
|
||||
cur = build_patch_merge_permute(cur, scale_factor);
|
||||
|
||||
// projection norm
|
||||
int proj_inp_dim = cur->ne[0];
|
||||
cur = ggml_view_2d(ctx0, cur,
|
||||
n_embd, cur->ne[1] * scale_factor * scale_factor,
|
||||
ggml_row_size(cur->type, n_embd), 0);
|
||||
cur = ggml_norm(ctx0, cur, 1e-5); // default nn.LayerNorm
|
||||
cur = ggml_mul(ctx0, cur, model.mm_input_norm_w);
|
||||
cur = ggml_add(ctx0, cur, model.mm_input_norm_b);
|
||||
cur = ggml_view_2d(ctx0, cur,
|
||||
proj_inp_dim, cur->ne[1] / scale_factor / scale_factor,
|
||||
ggml_row_size(cur->type, proj_inp_dim), 0);
|
||||
cb(cur, "proj_inp_normed", -1);
|
||||
|
||||
// projection mlp
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU,
|
||||
-1);
|
||||
cb(cur, "proj_out", -1);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_lfm3vl::build() {
|
||||
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
|
||||
GGML_ASSERT(model.lfm3vl_embedding_norm_w != nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
GGML_ASSERT(n_batch == 1);
|
||||
|
||||
const int batch_size = 1;
|
||||
const int n_pos = n_patches;
|
||||
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
|
||||
|
||||
const norm_type norm_t = NORM_TYPE_RMS;
|
||||
|
||||
// 2D rope: pairs (j, j+n_dims); first d_head/4 pairs rotate by y, next d_head/4 by x
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, 0, 0};
|
||||
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
|
||||
// one patchifier kernel per temporal slice: a still image fills both slices, so both kernels apply to it
|
||||
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
|
||||
if (model.patch_embeddings_1) {
|
||||
inp = ggml_add(ctx0, inp, ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
|
||||
}
|
||||
inp = ggml_reshape_2d(ctx0, inp, n_pos, n_embd);
|
||||
inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
cb(inp, "patchifier_out", -1);
|
||||
|
||||
// the 2x2 interleave waits for the projector, the conv layers need raster-scan order
|
||||
inp = build_norm(inp, model.lfm3vl_embedding_norm_w, nullptr, norm_t, eps, -1);
|
||||
cb(inp, "embedding_norm", -1);
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
|
||||
// position IDs for 2D RoPE (same format as Qwen3VL)
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
const float rope_theta = hparams.rope_theta;
|
||||
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
auto & layer = model.layers[il];
|
||||
|
||||
const bool is_conv_layer = (layer.conv2d_in_proj_w != nullptr);
|
||||
|
||||
ggml_tensor * cur = inpL;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
|
||||
cb(cur, "ln1", il);
|
||||
|
||||
if (is_conv_layer) {
|
||||
ggml_tensor * proj = build_mm(layer.conv2d_in_proj_w, cur);
|
||||
cb(proj, "conv_in_proj", il);
|
||||
|
||||
ggml_tensor * first = ggml_view_2d(ctx0, proj, n_embd, n_pos,
|
||||
proj->nb[1], 0);
|
||||
ggml_tensor * middle = ggml_view_2d(ctx0, proj, n_embd, n_pos,
|
||||
proj->nb[1], ggml_row_size(proj->type, n_embd));
|
||||
ggml_tensor * third = ggml_view_2d(ctx0, proj, n_embd, n_pos,
|
||||
proj->nb[1], ggml_row_size(proj->type, 2 * n_embd));
|
||||
|
||||
ggml_tensor * gated = ggml_mul(ctx0, first, third);
|
||||
cb(gated, "conv_gated", il);
|
||||
|
||||
ggml_tensor * conv_out;
|
||||
if (layer.conv2d_spatial_w) {
|
||||
ggml_tensor * gated_3d = ggml_reshape_4d(ctx0, gated,
|
||||
n_embd, n_patches_x, n_patches_y, batch_size);
|
||||
ggml_tensor * gated_2d = ggml_cont(ctx0,
|
||||
ggml_permute(ctx0, gated_3d, 2, 0, 1, 3));
|
||||
cb(gated_2d, "conv_gated_2d", il);
|
||||
|
||||
conv_out = ggml_conv_2d_dw_direct(ctx0, layer.conv2d_spatial_w, gated_2d,
|
||||
1, 1, 1, 1, 1, 1);
|
||||
cb(conv_out, "conv_spatial", il);
|
||||
|
||||
conv_out = ggml_cont(ctx0,
|
||||
ggml_permute(ctx0, conv_out, 1, 2, 0, 3));
|
||||
conv_out = ggml_reshape_2d(ctx0, conv_out, n_embd, n_pos);
|
||||
} else {
|
||||
conv_out = gated;
|
||||
}
|
||||
|
||||
cur = ggml_mul(ctx0, middle, conv_out);
|
||||
cb(cur, "conv_mul", il);
|
||||
|
||||
cur = build_mm(layer.conv2d_out_proj_w, cur);
|
||||
cb(cur, "conv_out_proj", il);
|
||||
} else {
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head_kv, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head_kv, n_pos);
|
||||
|
||||
if (layer.q_norm) {
|
||||
Qcur = build_norm(Qcur, layer.q_norm, nullptr, norm_t, eps, il);
|
||||
}
|
||||
if (layer.k_norm) {
|
||||
Kcur = build_norm(Kcur, layer.k_norm, nullptr, norm_t, eps, il);
|
||||
}
|
||||
|
||||
Qcur = ggml_rope_multi(
|
||||
ctx0, Qcur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
|
||||
32768, rope_theta, 1, 0, 1, 32, 1);
|
||||
Kcur = ggml_rope_multi(
|
||||
ctx0, Kcur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
|
||||
32768, rope_theta, 1, 0, 1, 32, 1);
|
||||
|
||||
cb(Qcur, "Qcur_rope", il);
|
||||
cb(Kcur, "Kcur_rope", il);
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b,
|
||||
Qcur, Kcur, Vcur, nullptr, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
inpL = cur;
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
|
||||
cb(cur, "ffn_inp_normed", il);
|
||||
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
// no post-layernorm for LFM3-VL
|
||||
|
||||
ggml_tensor * embeddings = inpL;
|
||||
|
||||
// projector input LayerNorm, eps is projector_layer_norm_eps (not the encoder eps)
|
||||
embeddings = build_norm(embeddings, model.mm_input_norm_w, model.mm_input_norm_b, NORM_TYPE_NORMAL, 1e-6f, -1);
|
||||
cb(embeddings, "proj_input_norm", -1);
|
||||
|
||||
// 2x2 patch interleave (same as Qwen3VL) so the reshape below merges spatial 2x2 blocks
|
||||
{
|
||||
embeddings = ggml_cont_4d(
|
||||
ctx0, embeddings,
|
||||
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
embeddings = ggml_reshape_4d(
|
||||
ctx0, embeddings,
|
||||
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
embeddings = ggml_permute(ctx0, embeddings, 0, 2, 1, 3);
|
||||
embeddings = ggml_cont_3d(
|
||||
ctx0, embeddings,
|
||||
n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
cb(embeddings, "proj_interleaved", -1);
|
||||
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
|
||||
|
||||
// projector MLP: linear_1 -> GELU (erf, as HF "gelu") -> linear_2
|
||||
embeddings = build_ffn(embeddings,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
ffn_op_type::FFN_GELU_ERF, -1);
|
||||
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_ling3vl::build() {
|
||||
// same vision tower as qwen3vl, but the merger is norm-only (no fc1/fc2) and
|
||||
// the projector MLP lives at the top level (mm.0 / mm.2)
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
GGML_ASSERT(model.mm_input_norm_w != nullptr); // merger norm (pre spatial merge)
|
||||
|
||||
const int batch_size = 1;
|
||||
const int n_pos = n_patches;
|
||||
|
||||
norm_type norm_t = NORM_TYPE_NORMAL;
|
||||
|
||||
// vision M-RoPE, same layout as qwen3vl: [row, col, row, col] quarters
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
|
||||
ggml_tensor * inp = build_inp_temporal_merge();
|
||||
|
||||
// spatial merge
|
||||
{
|
||||
inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b]
|
||||
inp = ggml_cont_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
inp = ggml_reshape_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
|
||||
inp = ggml_cont_3d(
|
||||
ctx0, inp,
|
||||
n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
|
||||
// add patch bias
|
||||
if (model.patch_bias != nullptr) {
|
||||
inp = ggml_add(ctx0, inp, model.patch_bias);
|
||||
cb(inp, "patch_bias", -1);
|
||||
}
|
||||
|
||||
// calculate absolute position embedding and apply
|
||||
ggml_tensor * learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS);
|
||||
learned_pos_embd = ggml_cont_4d(
|
||||
ctx0, learned_pos_embd,
|
||||
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
learned_pos_embd = ggml_reshape_4d(
|
||||
ctx0, learned_pos_embd,
|
||||
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);
|
||||
learned_pos_embd = ggml_cont_3d(
|
||||
ctx0, learned_pos_embd,
|
||||
n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
|
||||
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
ggml_tensor * inpL = build_vit(
|
||||
inp, n_pos, norm_t, hparams.ffn_op, learned_pos_embd,
|
||||
[&](ggml_tensor * c, const clip_layer &) {
|
||||
return ggml_rope_multi(
|
||||
ctx0, c, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
|
||||
});
|
||||
|
||||
// multimodal projection (linear_proj MLP over the merged patches)
|
||||
ggml_tensor * embeddings = inpL;
|
||||
|
||||
// per-patch merger norm, applied post-blocks before the 2x2 merge
|
||||
// (merger.norm, LayerNorm over n_embd)
|
||||
embeddings = build_norm(embeddings, model.mm_input_norm_w, model.mm_input_norm_b, norm_t, eps, -1);
|
||||
cb(embeddings, "merger_norm", -1);
|
||||
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
|
||||
|
||||
embeddings = build_ffn(embeddings,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
ffn_op_type::FFN_GELU, -1);
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_llama4::build() {
|
||||
GGML_ASSERT(model.class_embedding != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
|
||||
const int n_pos = n_patches + 1; // +1 for [CLS]
|
||||
|
||||
// 2D input positions
|
||||
ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(pos_h, "pos_h");
|
||||
ggml_set_input(pos_h);
|
||||
|
||||
ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(pos_w, "pos_w");
|
||||
ggml_set_input(pos_w);
|
||||
|
||||
ggml_tensor * inp = build_inp_raw();
|
||||
|
||||
// Llama4UnfoldConvolution
|
||||
{
|
||||
ggml_tensor * kernel = ggml_reshape_4d(ctx0, model.patch_embeddings_0,
|
||||
patch_size, patch_size, 3, n_embd);
|
||||
inp = ggml_im2col(ctx0, kernel, inp, patch_size, patch_size, 0, 0, 1, 1, true, inp->type);
|
||||
inp = build_mm(model.patch_embeddings_0, inp);
|
||||
inp = ggml_reshape_2d(ctx0, inp, n_embd, n_patches);
|
||||
cb(inp, "patch_conv", -1);
|
||||
}
|
||||
|
||||
// add CLS token
|
||||
inp = ggml_concat(ctx0, inp, model.class_embedding, 1);
|
||||
|
||||
// build ViT with 2D position embeddings
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
// first half is X axis and second half is Y axis
|
||||
// ref: https://github.com/huggingface/transformers/blob/40a493c7ed4f19f08eadb0639cf26d49bfa5e180/src/transformers/models/llama4/modeling_llama4.py#L1312
|
||||
// ref: https://github.com/Blaizzy/mlx-vlm/blob/a57156aa87b33cca6e5ee6cfc14dd4ef8f611be6/mlx_vlm/models/llama4/vision.py#L441
|
||||
return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
|
||||
};
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_pos,
|
||||
NORM_TYPE_NORMAL,
|
||||
hparams.ffn_op,
|
||||
model.position_embeddings,
|
||||
add_pos);
|
||||
|
||||
// remove CLS token
|
||||
cur = ggml_view_2d(ctx0, cur,
|
||||
n_embd, n_patches,
|
||||
ggml_row_size(cur->type, n_embd), 0);
|
||||
|
||||
// pixel shuffle
|
||||
// based on Llama4VisionPixelShuffleMLP
|
||||
// https://github.com/huggingface/transformers/blob/2932f318a20d9e54cc7aea052e040164d85de7d6/src/transformers/models/llama4/modeling_llama4.py#L1151
|
||||
{
|
||||
const int scale_factor = model.hparams.n_merge;
|
||||
const int bsz = 1; // batch size, always 1 for now since we don't support batching
|
||||
GGML_ASSERT(scale_factor > 0);
|
||||
GGML_ASSERT(n_patches_x == n_patches_y); // llama4 only supports square images
|
||||
cur = ggml_reshape_4d(ctx0, cur,
|
||||
n_embd * scale_factor,
|
||||
n_patches_x / scale_factor,
|
||||
n_patches_y,
|
||||
bsz);
|
||||
cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
|
||||
cur = ggml_cont_4d(ctx0, cur,
|
||||
n_embd * scale_factor * scale_factor,
|
||||
n_patches_x / scale_factor,
|
||||
n_patches_y / scale_factor,
|
||||
bsz);
|
||||
//cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
|
||||
// flatten to 2D
|
||||
cur = ggml_cont_2d(ctx0, cur,
|
||||
n_embd * scale_factor * scale_factor,
|
||||
n_patches / scale_factor / scale_factor);
|
||||
cb(cur, "pixel_shuffle", -1);
|
||||
}
|
||||
|
||||
// based on Llama4VisionMLP2 (always uses GELU activation, no bias)
|
||||
{
|
||||
cur = build_mm(model.mm_model_mlp_1_w, cur);
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
cur = build_mm(model.mm_model_mlp_2_w, cur);
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
cb(cur, "adapter_mlp", -1);
|
||||
}
|
||||
|
||||
// Llama4MultiModalProjector
|
||||
cur = build_mm(model.mm_model_proj, cur);
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
#include "models.h"
|
||||
|
||||
// this graph is used by llava, granite and glm
|
||||
// due to having embedding_stack (used by granite), we cannot reuse build_vit
|
||||
ggml_cgraph * clip_graph_llava::build() {
|
||||
const int batch_size = 1;
|
||||
const int n_pos = n_patches + (model.class_embedding ? 1 : 0);
|
||||
|
||||
GGML_ASSERT(n_patches_x == n_patches_y && "only square images supported");
|
||||
|
||||
// Calculate the deepest feature layer based on hparams and projector type
|
||||
int max_feature_layer = n_layer;
|
||||
{
|
||||
// Get the index of the second to last layer; this is the default for models that have a llava projector
|
||||
int il_last = hparams.n_layer - 1;
|
||||
int deepest_feature_layer = -1;
|
||||
|
||||
if (proj_type == PROJECTOR_TYPE_MINICPMV || proj_type == PROJECTOR_TYPE_GLM_EDGE) {
|
||||
il_last += 1;
|
||||
}
|
||||
|
||||
// If we set explicit vision feature layers, only go up to the deepest one
|
||||
// NOTE: only used by granite-vision models for now
|
||||
for (const auto & feature_layer : hparams.feature_layers) {
|
||||
if (feature_layer > deepest_feature_layer) {
|
||||
deepest_feature_layer = feature_layer;
|
||||
}
|
||||
}
|
||||
max_feature_layer = deepest_feature_layer < 0 ? il_last : deepest_feature_layer;
|
||||
}
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
|
||||
// concat class_embeddings and patch_embeddings
|
||||
if (model.class_embedding) {
|
||||
inp = ggml_concat(ctx0, inp, model.class_embedding, 1);
|
||||
}
|
||||
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
inp = ggml_add(ctx0, inp, ggml_get_rows(ctx0, model.position_embeddings, positions));
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
|
||||
// pre-layernorm
|
||||
if (model.pre_ln_w) {
|
||||
inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(inpL, "pre_ln", -1);
|
||||
}
|
||||
|
||||
std::vector<ggml_tensor *> embedding_stack;
|
||||
|
||||
// loop over layers
|
||||
for (int il = 0; il < max_feature_layer; il++) {
|
||||
auto & layer = model.layers[il];
|
||||
ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
|
||||
|
||||
// If this is an embedding feature layer, save the output.
|
||||
// NOTE: 0 index here refers to the input to the encoder.
|
||||
if (hparams.is_feature_layer(il)) {
|
||||
embedding_stack.push_back(cur);
|
||||
}
|
||||
|
||||
// layernorm1
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "layer_inp_normed", il);
|
||||
|
||||
// self-attention
|
||||
{
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
if (layer.q_b) {
|
||||
Qcur = ggml_add(ctx0, Qcur, layer.q_b);
|
||||
}
|
||||
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
if (layer.k_b) {
|
||||
Kcur = ggml_add(ctx0, Kcur, layer.k_b);
|
||||
}
|
||||
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
if (layer.v_b) {
|
||||
Vcur = ggml_add(ctx0, Vcur, layer.v_b);
|
||||
}
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b,
|
||||
Qcur, Kcur, Vcur, nullptr, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
// re-add the layer input, e.g., residual
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
|
||||
inpL = cur; // inpL = residual, cur = hidden_states
|
||||
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
// layernorm2
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "ffn_inp_normed", il);
|
||||
|
||||
// ffn
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
// residual 2
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
// post-layernorm
|
||||
if (model.post_ln_w) {
|
||||
inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
}
|
||||
|
||||
ggml_tensor * embeddings = inpL;
|
||||
|
||||
// process vision feature layers (used by granite)
|
||||
{
|
||||
// final layer is a vision feature layer
|
||||
if (hparams.is_feature_layer(max_feature_layer)) {
|
||||
embedding_stack.push_back(inpL);
|
||||
}
|
||||
|
||||
// If feature layers are explicitly set, stack them (if we have multiple)
|
||||
if (!embedding_stack.empty()) {
|
||||
embeddings = embedding_stack[0];
|
||||
for (size_t i = 1; i < embedding_stack.size(); i++) {
|
||||
embeddings = ggml_concat(ctx0, embeddings, embedding_stack[i], 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// llava projector (also used by granite)
|
||||
if (hparams.has_llava_projector) {
|
||||
embeddings = ggml_reshape_2d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1]);
|
||||
|
||||
ggml_tensor * patches = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(patches, "patches");
|
||||
ggml_set_input(patches);
|
||||
|
||||
// shape [1, 576, 1024]
|
||||
// ne is whcn, ne = [1024, 576, 1, 1]
|
||||
embeddings = ggml_get_rows(ctx0, embeddings, patches);
|
||||
|
||||
// print_tensor_info(embeddings, "embeddings");
|
||||
|
||||
// llava projector
|
||||
if (proj_type == PROJECTOR_TYPE_MLP) {
|
||||
embeddings = build_mm(model.mm_0_w, embeddings);
|
||||
embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);
|
||||
|
||||
embeddings = ggml_gelu(ctx0, embeddings);
|
||||
if (model.mm_2_w) {
|
||||
embeddings = build_mm(model.mm_2_w, embeddings);
|
||||
embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
|
||||
}
|
||||
}
|
||||
else if (proj_type == PROJECTOR_TYPE_MLP_NORM) {
|
||||
embeddings = build_mm(model.mm_0_w, embeddings);
|
||||
embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);
|
||||
// ggml_tensor_printf(embeddings, "mm_0_w",0,true,false);
|
||||
// First LayerNorm
|
||||
embeddings = ggml_norm(ctx0, embeddings, eps);
|
||||
embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.mm_1_w),
|
||||
model.mm_1_b);
|
||||
|
||||
// GELU activation
|
||||
embeddings = ggml_gelu(ctx0, embeddings);
|
||||
|
||||
// Second linear layer
|
||||
embeddings = build_mm(model.mm_3_w, embeddings);
|
||||
embeddings = ggml_add(ctx0, embeddings, model.mm_3_b);
|
||||
|
||||
// Second LayerNorm
|
||||
embeddings = ggml_norm(ctx0, embeddings, eps);
|
||||
embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.mm_4_w),
|
||||
model.mm_4_b);
|
||||
}
|
||||
else if (proj_type == PROJECTOR_TYPE_LDP) {
|
||||
// MobileVLM projector
|
||||
int n_patch = 24;
|
||||
ggml_tensor * mlp_1 = build_mm(model.mm_model_mlp_1_w, embeddings);
|
||||
mlp_1 = ggml_add(ctx0, mlp_1, model.mm_model_mlp_1_b);
|
||||
mlp_1 = ggml_gelu(ctx0, mlp_1);
|
||||
ggml_tensor * mlp_3 = build_mm(model.mm_model_mlp_3_w, mlp_1);
|
||||
mlp_3 = ggml_add(ctx0, mlp_3, model.mm_model_mlp_3_b);
|
||||
// mlp_3 shape = [1, 576, 2048], ne = [2048, 576, 1, 1]
|
||||
|
||||
// block 1
|
||||
ggml_tensor * block_1 = nullptr;
|
||||
{
|
||||
// transpose from [1, 576, 2048] --> [1, 2048, 576] --> [1, 2048, 24, 24]
|
||||
mlp_3 = ggml_permute(ctx0, mlp_3, 1, 0, 2, 3);
|
||||
mlp_3 = ggml_cont_4d(ctx0, mlp_3, n_patch, n_patch, mlp_3->ne[1], mlp_3->ne[2]);
|
||||
// stride = 1, padding = 1, bias is nullptr
|
||||
block_1 = ggml_conv_2d_dw(ctx0, model.mm_model_block_1_block_0_0_w, mlp_3, 1, 1, 1, 1, 1, 1);
|
||||
|
||||
// layer norm
|
||||
// // block_1 shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1]
|
||||
block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 2, 0, 3));
|
||||
// block_1 shape = [1, 24, 24, 2048], ne = [2048, 24, 24, 1]
|
||||
block_1 = ggml_norm(ctx0, block_1, eps);
|
||||
block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_1_block_0_1_w), model.mm_model_block_1_block_0_1_b);
|
||||
block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 2, 0, 1, 3));
|
||||
|
||||
// block_1 shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1]
|
||||
// hardswish
|
||||
ggml_tensor * block_1_hw = ggml_hardswish(ctx0, block_1);
|
||||
|
||||
block_1 = ggml_pool_2d(ctx0, block_1_hw, GGML_OP_POOL_AVG, block_1_hw->ne[0], block_1_hw->ne[1], block_1_hw->ne[0], block_1_hw->ne[1], 0, 0);
|
||||
// block_1 shape = [1, 2048, 1, 1], ne = [1, 1, 2048, 1]
|
||||
// pointwise conv
|
||||
block_1 = ggml_reshape_2d(ctx0, block_1, block_1->ne[0]*block_1->ne[1]*block_1->ne[2], block_1->ne[3]);
|
||||
block_1 = build_mm(model.mm_model_block_1_block_1_fc1_w, block_1);
|
||||
block_1 = ggml_add(ctx0, block_1, model.mm_model_block_1_block_1_fc1_b);
|
||||
block_1 = ggml_relu(ctx0, block_1);
|
||||
block_1 = build_mm(model.mm_model_block_1_block_1_fc2_w, block_1);
|
||||
block_1 = ggml_add(ctx0, block_1, model.mm_model_block_1_block_1_fc2_b);
|
||||
block_1 = ggml_hardsigmoid(ctx0, block_1);
|
||||
// block_1_hw shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1], block_1 shape = [1, 2048], ne = [2048, 1, 1, 1]
|
||||
block_1 = ggml_reshape_4d(ctx0, block_1, 1, 1, block_1->ne[0], block_1->ne[1]);
|
||||
block_1 = ggml_mul(ctx0, block_1_hw, block_1);
|
||||
|
||||
int w = block_1->ne[0], h = block_1->ne[1];
|
||||
block_1 = ggml_reshape_3d(ctx0, block_1, w*h, block_1->ne[2], block_1->ne[3]);
|
||||
block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 0, 2, 3));
|
||||
|
||||
// block_1 shape = [1, 24*24, 2048], ne = [24*24, 2048, 1]
|
||||
block_1 = build_mm(model.mm_model_block_1_block_2_0_w, block_1);
|
||||
block_1 = ggml_reshape_4d(ctx0, block_1, block_1->ne[0], w, h, block_1->ne[3]);
|
||||
|
||||
// block_1 shape = [1, 24, 24, 2048], ne = [2048, 24, 24, 1]
|
||||
block_1 = ggml_norm(ctx0, block_1, eps);
|
||||
block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_1_block_2_1_w), model.mm_model_block_1_block_2_1_b);
|
||||
block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 2, 0, 1, 3));
|
||||
// block1 shape = [1, 2048, 24, 24], ne = [24, 24, 2048, 1]
|
||||
// residual
|
||||
block_1 = ggml_add(ctx0, mlp_3, block_1);
|
||||
}
|
||||
|
||||
// block_2
|
||||
{
|
||||
// stride = 2
|
||||
block_1 = ggml_conv_2d_dw(ctx0, model.mm_model_block_2_block_0_0_w, block_1, 2, 2, 1, 1, 1, 1);
|
||||
|
||||
// block_1 shape = [1, 2048, 12, 12], ne = [12, 12, 2048, 1]
|
||||
// layer norm
|
||||
block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 2, 0, 3));
|
||||
// block_1 shape = [1, 12, 12, 2048], ne = [2048, 12, 12, 1]
|
||||
block_1 = ggml_norm(ctx0, block_1, eps);
|
||||
block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_2_block_0_1_w), model.mm_model_block_2_block_0_1_b);
|
||||
block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 2, 0, 1, 3));
|
||||
// block_1 shape = [1, 2048, 12, 12], ne = [12, 12, 2048, 1]
|
||||
// hardswish
|
||||
ggml_tensor * block_1_hw = ggml_hardswish(ctx0, block_1);
|
||||
|
||||
// not sure the parameters is right for globalAvgPooling
|
||||
block_1 = ggml_pool_2d(ctx0, block_1_hw, GGML_OP_POOL_AVG, block_1_hw->ne[0], block_1_hw->ne[1], block_1_hw->ne[0], block_1_hw->ne[1], 0, 0);
|
||||
// block_1 shape = [1, 2048, 1, 1], ne = [1, 1, 2048, 1]
|
||||
// pointwise conv
|
||||
block_1 = ggml_reshape_2d(ctx0, block_1, block_1->ne[0]*block_1->ne[1]*block_1->ne[2], block_1->ne[3]);
|
||||
block_1 = build_mm(model.mm_model_block_2_block_1_fc1_w, block_1);
|
||||
block_1 = ggml_add(ctx0, block_1, model.mm_model_block_2_block_1_fc1_b);
|
||||
block_1 = ggml_relu(ctx0, block_1);
|
||||
block_1 = build_mm(model.mm_model_block_2_block_1_fc2_w, block_1);
|
||||
block_1 = ggml_add(ctx0, block_1, model.mm_model_block_2_block_1_fc2_b);
|
||||
block_1 = ggml_hardsigmoid(ctx0, block_1);
|
||||
|
||||
// block_1_hw shape = [1, 2048, 12, 12], ne = [12, 12, 2048, 1], block_1 shape = [1, 2048, 1, 1], ne = [1, 1, 2048, 1]
|
||||
block_1 = ggml_reshape_4d(ctx0, block_1, 1, 1, block_1->ne[0], block_1->ne[1]);
|
||||
block_1 = ggml_mul(ctx0, block_1_hw, block_1);
|
||||
|
||||
int w = block_1->ne[0], h = block_1->ne[1];
|
||||
block_1 = ggml_reshape_3d(ctx0, block_1, w*h, block_1->ne[2], block_1->ne[3]);
|
||||
block_1 = ggml_cont(ctx0, ggml_permute(ctx0, block_1, 1, 0, 2, 3));
|
||||
// block_1 shape = [1, 24*24, 2048], ne = [24*24, 2048, 1]
|
||||
block_1 = build_mm(model.mm_model_block_2_block_2_0_w, block_1);
|
||||
block_1 = ggml_reshape_4d(ctx0, block_1, block_1->ne[0], w, h, block_1->ne[3]);
|
||||
|
||||
|
||||
// block_1 shape = [1, 12, 12, 2048], ne = [2048, 12, 12, 1]
|
||||
block_1 = ggml_norm(ctx0, block_1, eps);
|
||||
block_1 = ggml_add(ctx0, ggml_mul(ctx0, block_1, model.mm_model_block_2_block_2_1_w), model.mm_model_block_2_block_2_1_b);
|
||||
block_1 = ggml_reshape_3d(ctx0, block_1, block_1->ne[0], block_1->ne[1] * block_1->ne[2], block_1->ne[3]);
|
||||
// block_1 shape = [1, 144, 2048], ne = [2048, 144, 1]
|
||||
}
|
||||
embeddings = block_1;
|
||||
}
|
||||
else if (proj_type == PROJECTOR_TYPE_LDPV2)
|
||||
{
|
||||
int n_patch = 24;
|
||||
ggml_tensor * mlp_0 = build_mm(model.mm_model_mlp_0_w, embeddings);
|
||||
mlp_0 = ggml_add(ctx0, mlp_0, model.mm_model_mlp_0_b);
|
||||
mlp_0 = ggml_gelu(ctx0, mlp_0);
|
||||
ggml_tensor * mlp_2 = build_mm(model.mm_model_mlp_2_w, mlp_0);
|
||||
mlp_2 = ggml_add(ctx0, mlp_2, model.mm_model_mlp_2_b);
|
||||
// mlp_2 ne = [2048, 576, 1, 1]
|
||||
// // AVG Pool Layer 2*2, strides = 2
|
||||
mlp_2 = ggml_permute(ctx0, mlp_2, 1, 0, 2, 3);
|
||||
// mlp_2 ne = [576, 2048, 1, 1]
|
||||
mlp_2 = ggml_cont_4d(ctx0, mlp_2, n_patch, n_patch, mlp_2->ne[1], mlp_2->ne[2]);
|
||||
// mlp_2 ne [24, 24, 2048, 1]
|
||||
mlp_2 = ggml_pool_2d(ctx0, mlp_2, GGML_OP_POOL_AVG, 2, 2, 2, 2, 0, 0);
|
||||
// weight ne = [3, 3, 2048, 1]
|
||||
ggml_tensor * peg_0 = ggml_conv_2d_dw(ctx0, model.mm_model_peg_0_w, mlp_2, 1, 1, 1, 1, 1, 1);
|
||||
peg_0 = ggml_cont(ctx0, ggml_permute(ctx0, peg_0, 1, 2, 0, 3));
|
||||
peg_0 = ggml_add(ctx0, peg_0, model.mm_model_peg_0_b);
|
||||
mlp_2 = ggml_cont(ctx0, ggml_permute(ctx0, mlp_2, 1, 2, 0, 3));
|
||||
peg_0 = ggml_add(ctx0, peg_0, mlp_2);
|
||||
peg_0 = ggml_reshape_3d(ctx0, peg_0, peg_0->ne[0], peg_0->ne[1] * peg_0->ne[2], peg_0->ne[3]);
|
||||
embeddings = peg_0;
|
||||
}
|
||||
else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
}
|
||||
|
||||
// glm projector
|
||||
else if (proj_type == PROJECTOR_TYPE_GLM_EDGE) {
|
||||
size_t gridsz = (size_t)sqrt(embeddings->ne[1]);
|
||||
embeddings = ggml_permute(ctx0,embeddings,1,0,2,3);
|
||||
embeddings = ggml_cont_3d(ctx0, embeddings, gridsz, gridsz, embeddings->ne[1]);
|
||||
embeddings = ggml_conv_2d(ctx0, model.mm_model_adapter_conv_w, embeddings, 2, 2, 0, 0, 1, 1);
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings,embeddings->ne[0]*embeddings->ne[1] , embeddings->ne[2], batch_size);
|
||||
embeddings = ggml_cont(ctx0, ggml_permute(ctx0,embeddings, 1, 0, 2, 3));
|
||||
embeddings = ggml_add(ctx0, embeddings, model.mm_model_adapter_conv_b);
|
||||
// GLU
|
||||
{
|
||||
embeddings = build_mm(model.mm_model_mlp_0_w, embeddings);
|
||||
embeddings = ggml_norm(ctx0, embeddings, eps);
|
||||
embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.mm_model_ln_q_w), model.mm_model_ln_q_b);
|
||||
embeddings = ggml_gelu_inplace(ctx0, embeddings);
|
||||
ggml_tensor * x = embeddings;
|
||||
embeddings = build_mm(model.mm_model_mlp_2_w, embeddings);
|
||||
x = build_mm(model.mm_model_mlp_1_w,x);
|
||||
embeddings = ggml_swiglu_split(ctx0, embeddings, x);
|
||||
embeddings = build_mm(model.mm_model_mlp_3_w, embeddings);
|
||||
}
|
||||
// arrangement of BOI/EOI token embeddings
|
||||
// note: these embeddings are not present in text model, hence we cannot process them as text tokens
|
||||
// see: https://huggingface.co/THUDM/glm-edge-v-2b/blob/main/siglip.py#L53
|
||||
{
|
||||
embeddings = ggml_concat(ctx0, model.mm_boi, embeddings, 1); // BOI
|
||||
embeddings = ggml_concat(ctx0, embeddings, model.mm_eoi, 1); // EOI
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
GGML_ABORT("llava: unknown projector type");
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_mimo_audio::build() {
|
||||
ggml_tensor * inp = build_inp_raw(1); // [n_frames, n_mel, 1]
|
||||
|
||||
ggml_tensor * cur = ggml_conv_1d_ph(ctx0, model.conv1d_1_w, inp, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.conv1d_1_b);
|
||||
cur = ggml_gelu_erf(ctx0, cur);
|
||||
|
||||
cur = ggml_conv_1d_ph(ctx0, model.conv1d_2_w, cur, 2, 1);
|
||||
cur = ggml_add(ctx0, cur, model.conv1d_2_b);
|
||||
cur = ggml_gelu_erf(ctx0, cur);
|
||||
|
||||
ggml_tensor * inpL = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [n_embd, n_pos]
|
||||
const int64_t n_pos = inpL->ne[1];
|
||||
cb(inpL, "after_conv1d", -1);
|
||||
|
||||
GGML_ASSERT((int) hparams.wa_pattern_mode.size() == n_layer);
|
||||
|
||||
ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(inp_pos, "mimo_audio_positions");
|
||||
ggml_set_input(inp_pos);
|
||||
|
||||
ggml_tensor * full_mask = build_inp_attn_mask(n_pos, n_pos, 0);
|
||||
ggml_tensor * window_mask = build_inp_attn_mask(n_pos, n_pos, 1);
|
||||
|
||||
build_vit_opts opts;
|
||||
opts.attn_mask_layers.resize(n_layer);
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
opts.attn_mask_layers[il] = hparams.wa_pattern_mode[il] == -1 ? full_mask : window_mask;
|
||||
}
|
||||
// the skip connection below must be added before the post-transformer norm,
|
||||
// so build_vit must not apply that norm itself
|
||||
opts.skip_post_ln = true;
|
||||
|
||||
// encoder_skip_layer_id=3 (1-indexed) -> capture output of layer index 2
|
||||
const int skip_capture_il = 2;
|
||||
GGML_ASSERT(n_layer > skip_capture_il);
|
||||
ggml_tensor * skip_hidden = nullptr;
|
||||
opts.callback_layer_out = [&](ggml_tensor * layer_cur, int il) {
|
||||
if (il == skip_capture_il) {
|
||||
skip_hidden = layer_cur;
|
||||
}
|
||||
};
|
||||
|
||||
auto add_pos = [&](ggml_tensor * x, const clip_layer &) {
|
||||
return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head,
|
||||
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
};
|
||||
|
||||
inpL = build_vit(inpL, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, add_pos, opts);
|
||||
inpL = ggml_reshape_2d(ctx0, inpL, n_embd, n_pos); // build_vit restores a (size-1) batch dim
|
||||
|
||||
GGML_ASSERT(skip_hidden != nullptr);
|
||||
inpL = ggml_add(ctx0, inpL, skip_hidden);
|
||||
|
||||
inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(inpL, "after_transformer", -1);
|
||||
|
||||
// downsample: strided conv (no bias) + gelu + layernorm
|
||||
{
|
||||
ggml_tensor * ds = ggml_cont(ctx0, ggml_transpose(ctx0, inpL)); // [n_pos, n_embd]
|
||||
ds = ggml_conv_1d(ctx0, model.downsample_conv_w, ds, 2, 0, 1);
|
||||
ds = ggml_gelu_erf(ctx0, ds);
|
||||
ds = ggml_cont(ctx0, ggml_transpose(ctx0, ds)); // [n_embd, n_pos/2]
|
||||
ds = build_norm(ds, model.downsample_norm_w, model.downsample_norm_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
inpL = ds;
|
||||
}
|
||||
cb(inpL, "after_downsample", -1);
|
||||
|
||||
// RVQ quantize: codebook ne=[dim, max_bins, n_q]
|
||||
// quantize input vector to codes (type=I32)
|
||||
std::vector<ggml_tensor *> codes;
|
||||
{
|
||||
GGML_ASSERT(model.rvq_codebook != nullptr);
|
||||
const int64_t dim = model.rvq_codebook->ne[0];
|
||||
GGML_ASSERT(dim == inpL->ne[0]);
|
||||
GGML_ASSERT((int64_t) hparams.rvq_codebook_size.size() == model.rvq_codebook->ne[2]);
|
||||
|
||||
ggml_tensor * residual = inpL; // [dim, n_pos_ds]
|
||||
|
||||
for (size_t q = 0; q < hparams.rvq_codebook_size.size(); q++) {
|
||||
const int64_t bins = hparams.rvq_codebook_size[q];
|
||||
ggml_tensor * codebook_q = ggml_view_2d(ctx0, model.rvq_codebook, dim, bins,
|
||||
model.rvq_codebook->nb[1], q * model.rvq_codebook->nb[2]);
|
||||
codebook_q = ggml_cont(ctx0, codebook_q);
|
||||
|
||||
ggml_tensor * codebook_norm = ggml_sum_rows(ctx0, ggml_sqr(ctx0, codebook_q)); // [1, bins]
|
||||
codebook_norm = ggml_cont(ctx0, ggml_transpose(ctx0, codebook_norm)); // [bins, 1]
|
||||
|
||||
ggml_tensor * dot = ggml_mul_mat(ctx0, codebook_q, residual); // [bins, n_pos_ds]
|
||||
ggml_tensor * scores = ggml_sub(ctx0, ggml_scale(ctx0, dot, 2.0f), codebook_norm);
|
||||
|
||||
ggml_tensor * idx = ggml_argmax(ctx0, scores); // [n_pos_ds]
|
||||
codes.push_back(idx);
|
||||
|
||||
ggml_tensor * quant = ggml_get_rows(ctx0, codebook_q, idx); // [dim, n_pos_ds]
|
||||
residual = ggml_sub(ctx0, residual, quant);
|
||||
cb(idx, "rvq_code", (int) q);
|
||||
}
|
||||
}
|
||||
|
||||
// convert codes to LLM embeddings
|
||||
ggml_tensor * code_embd_sum = nullptr;
|
||||
{
|
||||
GGML_ASSERT(model.mm_a_code_embd != nullptr);
|
||||
const int64_t dim = model.mm_a_code_embd->ne[0];
|
||||
const int64_t vocab = model.mm_a_code_embd->ne[1];
|
||||
GGML_ASSERT((int64_t) codes.size() == model.mm_a_code_embd->ne[2]);
|
||||
GGML_ASSERT(dim == inpL->ne[0]);
|
||||
|
||||
for (size_t i = 0; i < codes.size(); i++) {
|
||||
ggml_tensor * table_i = ggml_view_2d(ctx0, model.mm_a_code_embd, dim, vocab,
|
||||
model.mm_a_code_embd->nb[1], i * model.mm_a_code_embd->nb[2]);
|
||||
table_i = ggml_cont(ctx0, table_i);
|
||||
|
||||
ggml_tensor * embd_i = ggml_get_rows(ctx0, table_i, codes[i]); // [dim, n_pos_ds]
|
||||
code_embd_sum = code_embd_sum ? ggml_add(ctx0, code_embd_sum, embd_i) : embd_i;
|
||||
}
|
||||
cb(code_embd_sum, "code_embd_sum", -1);
|
||||
}
|
||||
|
||||
// input_local_transformer
|
||||
// groups of `group_size` consecutive downsampled frames are processed together, attending only within their own group.
|
||||
// Implemented as a block-diagonal mask + in-group-repeating positions
|
||||
// (rather than a real batch dim) - same technique as the encoder's masks above, and as gemma4a's / deepseekocr2's chunked attention.
|
||||
|
||||
// note: hand-rolled here instead of build_vit() because this is a second, independent layer stack
|
||||
// (own layer array/count, RMSNorm instead of LN, SiLU FFN, own RoPE theta)
|
||||
|
||||
ggml_tensor * projected;
|
||||
{
|
||||
const int group_size = hparams.audio_local_group_size;
|
||||
GGML_ASSERT(group_size > 0);
|
||||
const int64_t n_pos_ds = code_embd_sum->ne[1];
|
||||
const int64_t n_groups = (n_pos_ds + group_size - 1) / group_size;
|
||||
const int64_t n_padded = n_groups * group_size;
|
||||
|
||||
ggml_tensor * cur_local = code_embd_sum;
|
||||
if (n_padded != n_pos_ds) {
|
||||
cur_local = ggml_pad(ctx0, cur_local, 0, (int) (n_padded - n_pos_ds), 0, 0);
|
||||
}
|
||||
|
||||
ggml_tensor * local_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_padded);
|
||||
ggml_set_name(local_pos, "mimo_audio_local_positions");
|
||||
ggml_set_input(local_pos);
|
||||
|
||||
ggml_tensor * local_mask = build_inp_attn_mask(n_padded, n_padded, 2);
|
||||
|
||||
const float local_rope_theta = 640000.0f; // audio_config.rope_theta (differs from the encoder's)
|
||||
auto apply_local_rope = [&](ggml_tensor * x) {
|
||||
return ggml_rope_ext(ctx0, x, local_pos, nullptr, d_head,
|
||||
GGML_ROPE_TYPE_NEOX, 0, local_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
};
|
||||
|
||||
for (int il = 0; il < hparams.audio_local_n_layer; il++) {
|
||||
auto & layer = model.mm_a_local_layers[il];
|
||||
|
||||
ggml_tensor * attn_in = build_norm(cur_local, layer.ln_1_w, nullptr, NORM_TYPE_RMS, eps, il);
|
||||
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, attn_in);
|
||||
if (layer.q_b) {
|
||||
Qcur = ggml_add(ctx0, Qcur, layer.q_b);
|
||||
}
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, attn_in);
|
||||
if (layer.k_b) {
|
||||
Kcur = ggml_add(ctx0, Kcur, layer.k_b);
|
||||
}
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, attn_in);
|
||||
if (layer.v_b) {
|
||||
Vcur = ggml_add(ctx0, Vcur, layer.v_b);
|
||||
}
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_padded);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_padded);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_padded);
|
||||
|
||||
Qcur = apply_local_rope(Qcur);
|
||||
Kcur = apply_local_rope(Kcur);
|
||||
|
||||
ggml_tensor * attn_out = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, local_mask, kq_scale, il);
|
||||
cur_local = ggml_add(ctx0, cur_local, attn_out);
|
||||
|
||||
ggml_tensor * ffn_in = build_norm(cur_local, layer.ln_2_w, nullptr, NORM_TYPE_RMS, eps, il);
|
||||
ggml_tensor * ffn_out = build_ffn(ffn_in,
|
||||
layer.ff_up_w, nullptr,
|
||||
layer.ff_gate_w, nullptr,
|
||||
layer.ff_down_w, nullptr,
|
||||
FFN_SILU, il);
|
||||
cur_local = ggml_add(ctx0, cur_local, ffn_out);
|
||||
}
|
||||
|
||||
cur_local = build_norm(cur_local, model.mm_a_local_norm_w, nullptr, NORM_TYPE_RMS, eps, -1);
|
||||
cb(cur_local, "after_local_transformer", -1);
|
||||
|
||||
// flatten each group of `group_size` frames into one (group_size*n_embd)-dim vector
|
||||
// (matching AudioProjection's flattened input)
|
||||
ggml_tensor * grouped = ggml_reshape_2d(ctx0, cur_local, n_embd * group_size, n_groups);
|
||||
|
||||
// AudioProjection: Linear (no bias) -> GELU -> Linear (no bias)
|
||||
projected = build_ffn(grouped,
|
||||
model.mm_1_w, nullptr,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, nullptr,
|
||||
FFN_GELU_ERF, -1);
|
||||
cb(projected, "after_projection", -1);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, projected);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const {
|
||||
ggml_tensor * cur = ggml_mul_mat(ctx0, w, x);
|
||||
ggml_prec_set_acc(cur, GGML_PREC_F32);
|
||||
return cur;
|
||||
}
|
||||
|
||||
// MiMoVL vision tower for MiMo-V2.5 (non-Pro). Qwen2.5-VL-shaped ViT, except:
|
||||
// 1. GQA in attention (32 Q / 8 KV heads, head_dim 64).
|
||||
// 2. Per-head attention sinks on every windowed layer. The sinks adjust
|
||||
// the softmax denominator (equivalently, a virtual extra K column with V=0),
|
||||
// so they decay attention weight without contributing to the output.
|
||||
// 3. Per-layer window-attention mode in hparams.wa_pattern_mode:
|
||||
// -1 -> full, 0 -> row-window+sinks, 1 -> col-window+sinks.
|
||||
// Col mode transposes the merge-unit grid on entry and restores
|
||||
// it on exit. Both patch and rotary orderings are pre-computed
|
||||
// host-side.
|
||||
// 4. 1D banded sliding window (|q-k| > window_size -> -inf) as a
|
||||
// single 2D mask broadcast across heads.
|
||||
// 5. Per-block MLP biases.
|
||||
ggml_cgraph * clip_graph_mimovl::build() {
|
||||
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
|
||||
GGML_ASSERT(model.patch_embeddings_1 != nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
GGML_ASSERT(hparams.n_head_kv > 0);
|
||||
GGML_ASSERT(n_head % hparams.n_head_kv == 0);
|
||||
GGML_ASSERT((int) hparams.wa_pattern_mode.size() == n_layer);
|
||||
|
||||
const int batch_size = 1;
|
||||
const int n_pos = n_patches;
|
||||
const int n_head_kv = hparams.n_head_kv;
|
||||
const int merge = hparams.n_merge > 0 ? hparams.n_merge : 2;
|
||||
const int merge_unit = merge * merge;
|
||||
const int n_units = n_pos / merge_unit;
|
||||
GGML_ASSERT(n_units * merge_unit == n_pos);
|
||||
|
||||
// MiMoVL has head_dim=64 with n_embd=1280, so n_embd is NOT n_head*head_dim
|
||||
// (the base class's d_head = n_embd/n_head = 40 is wrong here). Derive
|
||||
// head_dim from the fused QKV projection: rows = (n_head + 2*n_head_kv)*head_dim.
|
||||
GGML_ASSERT(model.layers[0].qkv_w != nullptr);
|
||||
const int qkv_rows = model.layers[0].qkv_w->ne[1];
|
||||
const int head_dim = qkv_rows / (n_head + 2 * n_head_kv);
|
||||
GGML_ASSERT(head_dim * (n_head + 2 * n_head_kv) == qkv_rows);
|
||||
const float attn_scale = 1.0f / std::sqrt((float) head_dim);
|
||||
const int rope_n_dims = head_dim / 2;
|
||||
int mrope_sections[4] = {rope_n_dims/2, rope_n_dims/2, 0, 0};
|
||||
|
||||
// Patch embed: Conv3D(kt=2) split into two Conv2D, then interleave-merge
|
||||
// along the height axis to match the merge-tile token order.
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw,
|
||||
patch_size, patch_size, 0, 0, 1, 1);
|
||||
{
|
||||
ggml_tensor * inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw,
|
||||
patch_size, patch_size, 0, 0, 1, 1);
|
||||
inp = ggml_add(ctx0, inp, inp_1);
|
||||
|
||||
GGML_ASSERT(img.nx() % (patch_size * 2) == 0);
|
||||
GGML_ASSERT(img.ny() % (patch_size * 2) == 0);
|
||||
|
||||
inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w,h,c,b] -> [c,w,h,b]
|
||||
inp = ggml_cont_4d(ctx0, inp, n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
inp = ggml_reshape_4d(ctx0, inp, n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
|
||||
inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
cb(inp, "patch_embed", -1);
|
||||
|
||||
ggml_tensor * positions_row = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos * 4);
|
||||
ggml_set_name(positions_row, "mimovl_positions_row");
|
||||
ggml_set_input(positions_row);
|
||||
|
||||
ggml_tensor * positions_col = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos * 4);
|
||||
ggml_set_name(positions_col, "mimovl_positions_col");
|
||||
ggml_set_input(positions_col);
|
||||
|
||||
// idx_col is the col-major merge-unit permutation. Take it as F32 so we can
|
||||
// derive the inverse permutation in-graph via ggml_argsort;
|
||||
// ggml_get_rows requires its index tensor to be I32, so cast back as well.
|
||||
ggml_tensor * idx_col_f = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_units);
|
||||
ggml_set_name(idx_col_f, "mimovl_idx_col");
|
||||
ggml_set_input(idx_col_f);
|
||||
ggml_tensor * idx_col = ggml_cast(ctx0, idx_col_f, GGML_TYPE_I32);
|
||||
ggml_tensor * idx_col_inv = ggml_argsort(ctx0, idx_col_f, GGML_SORT_ORDER_ASC);
|
||||
|
||||
ggml_tensor * window_mask = build_inp_attn_mask(n_pos, n_pos);
|
||||
|
||||
// Reorder helper: permute patches at merge-unit granularity. The patch
|
||||
// sequence is laid out as n_units groups of merge_unit (=4) consecutive
|
||||
// patches; the row<->col transpose only permutes whole groups. We keep
|
||||
// the per-group (h,w) ordering intact by reshaping to
|
||||
// [n_embd*merge_unit, n_units] before ggml_get_rows.
|
||||
auto reorder = [&](ggml_tensor * x, ggml_tensor * idx) {
|
||||
ggml_tensor * y = ggml_reshape_2d(ctx0, x, n_embd * merge_unit, n_units);
|
||||
y = ggml_get_rows(ctx0, y, idx);
|
||||
return ggml_reshape_3d(ctx0, y, n_embd, n_pos, batch_size);
|
||||
};
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
int prev_mode = -1;
|
||||
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
const auto & layer = model.layers[il];
|
||||
const int mode = hparams.wa_pattern_mode[il];
|
||||
const bool is_full = (mode == -1);
|
||||
const bool is_col = (mode == 1);
|
||||
|
||||
// Reorder transitions on entry/exit of a col-mode run.
|
||||
if (is_col && prev_mode != 1) {
|
||||
inpL = reorder(inpL, idx_col);
|
||||
cb(inpL, "reorder_to_col", il);
|
||||
} else if (!is_col && prev_mode == 1) {
|
||||
inpL = reorder(inpL, idx_col_inv);
|
||||
cb(inpL, "reorder_to_row", il);
|
||||
}
|
||||
|
||||
ggml_tensor * cur = inpL;
|
||||
|
||||
// Pre-attention RMSNorm.
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_RMS, eps, il);
|
||||
cb(cur, "ln1", il);
|
||||
|
||||
// Fused QKV with GQA.
|
||||
ggml_tensor * qkv = build_mm(layer.qkv_w, cur);
|
||||
qkv = ggml_add(ctx0, qkv, layer.qkv_b);
|
||||
|
||||
const size_t row = ggml_row_size(qkv->type, head_dim);
|
||||
const size_t off_k = ggml_row_size(qkv->type, n_head * head_dim);
|
||||
const size_t off_v = ggml_row_size(qkv->type, (n_head + n_head_kv) * head_dim);
|
||||
|
||||
ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, head_dim, n_head, n_pos, row, qkv->nb[1], 0);
|
||||
ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, head_dim, n_head_kv, n_pos, row, qkv->nb[1], off_k);
|
||||
ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, head_dim, n_head_kv, n_pos, row, qkv->nb[1], off_v);
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
// 2D RoPE
|
||||
ggml_tensor * pos = is_col ? positions_col : positions_row;
|
||||
Qcur = ggml_rope_multi(ctx0, Qcur, pos, nullptr, rope_n_dims, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f);
|
||||
Kcur = ggml_rope_multi(ctx0, Kcur, pos, nullptr, rope_n_dims, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f);
|
||||
cb(Qcur, "Qcur_rope", il);
|
||||
cb(Kcur, "Kcur_rope", il);
|
||||
|
||||
// Full layers: plain attention. Windowed layers: banded mask and per-head sinks.
|
||||
ggml_tensor * mask = is_full ? nullptr : window_mask;
|
||||
ggml_tensor * sinks = is_full ? nullptr : layer.attn_sinks;
|
||||
if (!is_full) {
|
||||
GGML_ASSERT(layer.attn_sinks != nullptr);
|
||||
}
|
||||
ggml_tensor * attn_out = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, mask, attn_scale, il, sinks);
|
||||
cb(attn_out, "attn_out", il);
|
||||
|
||||
// Residual 1.
|
||||
cur = ggml_add(ctx0, attn_out, inpL);
|
||||
inpL = cur;
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
// Pre-FFN RMSNorm.
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_RMS, eps, il);
|
||||
cb(cur, "ffn_inp_normed", il);
|
||||
|
||||
// SwiGLU MLP with biases
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
// Residual 2.
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
|
||||
inpL = cur;
|
||||
prev_mode = mode;
|
||||
}
|
||||
|
||||
// If the last block was col-mode, undo the transpose so the merger sees patches in row order.
|
||||
if (prev_mode == 1) {
|
||||
inpL = reorder(inpL, idx_col_inv);
|
||||
cb(inpL, "reorder_to_row_final", -1);
|
||||
}
|
||||
|
||||
// Merger: post-LayerNorm
|
||||
inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, NORM_TYPE_NORMAL, 1e-6f, n_layer);
|
||||
cb(inpL, "post_ln", -1);
|
||||
|
||||
// Spatial merge: pack each merge_unit (=4) of patches into a single
|
||||
// (n_embd*merge_unit)-wide row, then run the 2-layer MLP.
|
||||
ggml_tensor * embeddings = ggml_reshape_3d(ctx0, inpL, n_embd * merge_unit, n_units, batch_size);
|
||||
embeddings = build_ffn(embeddings,
|
||||
model.mm_0_w, nullptr,
|
||||
nullptr, nullptr,
|
||||
model.mm_1_w, nullptr,
|
||||
FFN_GELU, -1);
|
||||
cb(embeddings, "vit_out", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_minicpmv::build() {
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
const int n_pos = n_patches;
|
||||
const int n_embd_proj = n_mmproj_embd;
|
||||
|
||||
// position embeddings for the projector (not for ViT)
|
||||
// see: https://huggingface.co/openbmb/MiniCPM-o-2_6/blob/main/resampler.py#L70
|
||||
// base frequency omega
|
||||
ggml_tensor * omega = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_embd_proj / 4);
|
||||
ggml_set_name(omega, "omega");
|
||||
ggml_set_input(omega);
|
||||
|
||||
// 2D input positions (using float for sinusoidal embeddings)
|
||||
ggml_tensor * pos_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_pos);
|
||||
ggml_set_name(pos_h, "pos_h");
|
||||
ggml_set_input(pos_h);
|
||||
ggml_tensor * pos_w = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_pos);
|
||||
ggml_set_name(pos_w, "pos_w");
|
||||
ggml_set_input(pos_w);
|
||||
|
||||
// for selecting learned pos embd, used by ViT
|
||||
struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, model.position_embeddings, positions);
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * embeddings = build_vit(
|
||||
inp, n_pos,
|
||||
NORM_TYPE_NORMAL,
|
||||
hparams.ffn_op,
|
||||
learned_pos_embd,
|
||||
nullptr);
|
||||
|
||||
// resampler projector (it is just another transformer)
|
||||
|
||||
ggml_tensor * q = model.mm_model_query;
|
||||
ggml_tensor * v = build_mm(model.mm_model_kv_proj, embeddings);
|
||||
|
||||
// norm
|
||||
q = build_norm(q, model.mm_model_ln_q_w, model.mm_model_ln_q_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
v = build_norm(v, model.mm_model_ln_kv_w, model.mm_model_ln_kv_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
|
||||
// calculate sinusoidal pos embd
|
||||
ggml_tensor * pos_embed = nullptr;
|
||||
{
|
||||
// outer product
|
||||
ggml_tensor * omega_b = ggml_repeat_4d(ctx0, omega, omega->ne[0], n_pos, 1, 1); // n_pos rows
|
||||
ggml_tensor * theta_x = ggml_mul(ctx0, omega_b, pos_w);
|
||||
ggml_tensor * theta_y = ggml_mul(ctx0, omega_b, pos_h);
|
||||
// sin and cos
|
||||
ggml_tensor * pos_embd_x = ggml_concat(
|
||||
ctx0,
|
||||
ggml_sin(ctx0, theta_x),
|
||||
ggml_cos(ctx0, theta_x),
|
||||
0 // concat on first dim
|
||||
);
|
||||
ggml_tensor * pos_embd_y = ggml_concat(
|
||||
ctx0,
|
||||
ggml_sin(ctx0, theta_y),
|
||||
ggml_cos(ctx0, theta_y),
|
||||
0 // concat on first dim
|
||||
);
|
||||
pos_embed = ggml_concat(ctx0, pos_embd_x, pos_embd_y, 0);
|
||||
}
|
||||
|
||||
// k = v + pos_embed
|
||||
ggml_tensor * k = ggml_add(ctx0, v, pos_embed);
|
||||
|
||||
// attention
|
||||
{
|
||||
const int d_head = 128;
|
||||
int n_head = n_embd_proj/d_head;
|
||||
// Use actual config value if available, otherwise fall back to hardcoded values
|
||||
int num_query = hparams.minicpmv_query_num;
|
||||
ggml_tensor * Q = ggml_add(ctx0,
|
||||
build_mm(model.mm_model_attn_q_w, q),
|
||||
model.mm_model_attn_q_b);
|
||||
ggml_tensor * K = ggml_add(ctx0,
|
||||
build_mm(model.mm_model_attn_k_w, k),
|
||||
model.mm_model_attn_k_b);
|
||||
ggml_tensor * V = ggml_add(ctx0,
|
||||
build_mm(model.mm_model_attn_v_w, v),
|
||||
model.mm_model_attn_v_b);
|
||||
|
||||
Q = ggml_reshape_3d(ctx0, Q, d_head, n_head, num_query);
|
||||
K = ggml_reshape_3d(ctx0, K, d_head, n_head, n_pos);
|
||||
V = ggml_reshape_3d(ctx0, V, d_head, n_head, n_pos);
|
||||
|
||||
cb(Q, "resampler_Q", -1);
|
||||
cb(K, "resampler_K", -1);
|
||||
cb(V, "resampler_V", -1);
|
||||
|
||||
float resampler_kq_scale = 1.0f/ sqrtf(float(d_head));
|
||||
embeddings = build_attn(
|
||||
model.mm_model_attn_o_w,
|
||||
model.mm_model_attn_o_b,
|
||||
Q, K, V, nullptr, resampler_kq_scale, -1);
|
||||
cb(embeddings, "resampler_attn_out", -1);
|
||||
}
|
||||
// layernorm
|
||||
embeddings = build_norm(embeddings, model.mm_model_ln_post_w, model.mm_model_ln_post_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
|
||||
// projection
|
||||
embeddings = build_mm(model.mm_model_proj, embeddings);
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_minicpmv4_6::build() {
|
||||
const bool is_4x = hparams.n_merge == 2;
|
||||
const int n_pos = n_patches;
|
||||
const int half_h = n_patches_y / 2;
|
||||
const int half_w = n_patches_x / 2;
|
||||
const int n_ds = half_h * half_w;
|
||||
const int n_out = is_4x ? n_ds : (half_h / 2) * (half_w / 2);
|
||||
|
||||
auto add_i32_input = [&](const char * name, int n) {
|
||||
ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n);
|
||||
ggml_set_name(t, name);
|
||||
ggml_set_input(t);
|
||||
return t;
|
||||
};
|
||||
|
||||
// position indices for ViT learned positional embeddings
|
||||
ggml_tensor * positions = add_i32_input("positions", n_pos);
|
||||
ggml_tensor * learned_pos_embd = ggml_get_rows(ctx0, model.position_embeddings, positions);
|
||||
|
||||
ggml_tensor * vit_merger_window_idx = nullptr;
|
||||
ggml_tensor * vit_merger_inv_window_idx = nullptr;
|
||||
ggml_tensor * vit_merger_window_mask = nullptr;
|
||||
ggml_tensor * vit_merger_ds_idx_0 = nullptr;
|
||||
ggml_tensor * vit_merger_ds_idx_1 = nullptr;
|
||||
ggml_tensor * vit_merger_ds_idx_2 = nullptr;
|
||||
ggml_tensor * vit_merger_ds_idx_3 = nullptr;
|
||||
|
||||
if (!is_4x) {
|
||||
// ViT merger window reorder indices + block-diagonal mask
|
||||
// (mask layout follows qwen2vl: -inf except for 4x4 blocks on the diagonal,
|
||||
// so each window-major group of 4 tokens only attends to itself)
|
||||
vit_merger_window_idx = add_i32_input("vit_merger_window_idx", n_pos);
|
||||
vit_merger_inv_window_idx = add_i32_input("vit_merger_inv_window_idx", n_pos);
|
||||
vit_merger_window_mask = build_inp_attn_mask(n_pos, n_pos);
|
||||
|
||||
// ViT merger 2x2 downsample gather indices
|
||||
vit_merger_ds_idx_0 = add_i32_input("vit_merger_ds_idx_0", n_ds);
|
||||
vit_merger_ds_idx_1 = add_i32_input("vit_merger_ds_idx_1", n_ds);
|
||||
vit_merger_ds_idx_2 = add_i32_input("vit_merger_ds_idx_2", n_ds);
|
||||
vit_merger_ds_idx_3 = add_i32_input("vit_merger_ds_idx_3", n_ds);
|
||||
}
|
||||
|
||||
// final merger 2x2 downsample gather indices
|
||||
ggml_tensor * merger_ds_idx_0 = add_i32_input("merger_ds_idx_0", n_out);
|
||||
ggml_tensor * merger_ds_idx_1 = add_i32_input("merger_ds_idx_1", n_out);
|
||||
ggml_tensor * merger_ds_idx_2 = add_i32_input("merger_ds_idx_2", n_out);
|
||||
ggml_tensor * merger_ds_idx_3 = add_i32_input("merger_ds_idx_3", n_out);
|
||||
|
||||
// patch embedding + positional embedding
|
||||
ggml_tensor * inp = build_inp();
|
||||
inp = ggml_add(ctx0, inp, learned_pos_embd);
|
||||
cb(inp, "pos_embed", -1);
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
if (model.pre_ln_w) {
|
||||
inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(inpL, "pre_ln", -1);
|
||||
}
|
||||
|
||||
auto build_vit_layers = [&](ggml_tensor * input, int il_begin, int il_end, int64_t n_pos_layer) {
|
||||
for (int il = il_begin; il < il_end; il++) {
|
||||
auto & layer = model.layers[il];
|
||||
ggml_tensor * cur = input;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "layer_inp_normed", il);
|
||||
|
||||
{
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
if (layer.q_b) {
|
||||
Qcur = ggml_add(ctx0, Qcur, layer.q_b);
|
||||
}
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
if (layer.k_b) {
|
||||
Kcur = ggml_add(ctx0, Kcur, layer.k_b);
|
||||
}
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
if (layer.v_b) {
|
||||
Vcur = ggml_add(ctx0, Vcur, layer.v_b);
|
||||
}
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos_layer);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos_layer);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos_layer);
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, nullptr, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
if (layer.ls_1_w) {
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_1_w);
|
||||
cb(cur, "attn_out_scaled", il);
|
||||
}
|
||||
cur = ggml_add(ctx0, cur, input);
|
||||
input = cur;
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cb(cur, "ffn_inp_normed", il);
|
||||
|
||||
cur = build_ffn(cur, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b, hparams.ffn_op, il);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
if (layer.ls_2_w) {
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_2_w);
|
||||
cb(cur, "ffn_out_scaled", il);
|
||||
}
|
||||
input = ggml_add(ctx0, input, cur);
|
||||
cb(input, "layer_out", il);
|
||||
}
|
||||
return input;
|
||||
};
|
||||
|
||||
if (!is_4x) {
|
||||
const int insert_lid = hparams.insert_layer_id;
|
||||
|
||||
inpL = build_vit_layers(inpL, 0, insert_lid + 1, n_pos);
|
||||
|
||||
// ViT merger: window self-attention
|
||||
// Tokens are reordered to window-major (4 tokens per window are contiguous),
|
||||
// and a block-diagonal mask restricts attention to within each window. This
|
||||
// mirrors the qwen2vl windowed-attention pattern so build_attn() can pick the
|
||||
// flash-attention path when available.
|
||||
{
|
||||
ggml_tensor * residual = inpL;
|
||||
ggml_tensor * cur = build_norm(inpL,
|
||||
model.vit_merger_ln1_w, model.vit_merger_ln1_b,
|
||||
NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(cur, "vit_merger_attn_inp_normed", -1);
|
||||
|
||||
cur = ggml_get_rows(ctx0, cur, vit_merger_window_idx);
|
||||
cb(cur, "vit_merger_window_reorder", -1);
|
||||
|
||||
ggml_tensor * Qcur = build_mm(model.vit_merger_attn_q_w, cur);
|
||||
if (model.vit_merger_attn_q_b) {
|
||||
Qcur = ggml_add(ctx0, Qcur, model.vit_merger_attn_q_b);
|
||||
}
|
||||
ggml_tensor * Kcur = build_mm(model.vit_merger_attn_k_w, cur);
|
||||
if (model.vit_merger_attn_k_b) {
|
||||
Kcur = ggml_add(ctx0, Kcur, model.vit_merger_attn_k_b);
|
||||
}
|
||||
ggml_tensor * Vcur = build_mm(model.vit_merger_attn_v_w, cur);
|
||||
if (model.vit_merger_attn_v_b) {
|
||||
Vcur = ggml_add(ctx0, Vcur, model.vit_merger_attn_v_b);
|
||||
}
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
|
||||
cb(Qcur, "vit_merger_Qcur", -1);
|
||||
cb(Kcur, "vit_merger_Kcur", -1);
|
||||
cb(Vcur, "vit_merger_Vcur", -1);
|
||||
|
||||
cur = build_attn(model.vit_merger_attn_o_w, model.vit_merger_attn_o_b,
|
||||
Qcur, Kcur, Vcur, vit_merger_window_mask, kq_scale, -1);
|
||||
cb(cur, "vit_merger_attn_out", -1);
|
||||
|
||||
cur = ggml_get_rows(ctx0, cur, vit_merger_inv_window_idx);
|
||||
inpL = ggml_add(ctx0, cur, residual);
|
||||
cb(inpL, "vit_merger_attn_residual", -1);
|
||||
}
|
||||
|
||||
// ViT merger: 2x2 spatial downsample + MLP (4 tokens -> 1)
|
||||
{
|
||||
ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_0);
|
||||
ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_1);
|
||||
ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_2);
|
||||
ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, vit_merger_ds_idx_3);
|
||||
|
||||
ggml_tensor * mean_res = ggml_add(ctx0, p0, p1);
|
||||
mean_res = ggml_add(ctx0, mean_res, p2);
|
||||
mean_res = ggml_add(ctx0, mean_res, p3);
|
||||
mean_res = ggml_scale(ctx0, mean_res, 0.25f);
|
||||
cb(mean_res, "vit_merger_ds_mean_res", -1);
|
||||
|
||||
ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);
|
||||
cat = ggml_concat(ctx0, cat, p2, 0);
|
||||
cat = ggml_concat(ctx0, cat, p3, 0);
|
||||
|
||||
ggml_tensor * cur = build_norm(cat,
|
||||
model.vit_merger_ds_ln_w, model.vit_merger_ds_ln_b,
|
||||
NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(cur, "vit_merger_ds_normed", -1);
|
||||
|
||||
// ViTWindowAttentionMerger downsample MLP uses gelu_pytorch_tanh (FFN_GELU)
|
||||
cur = build_ffn(cur,
|
||||
model.vit_merger_ds_up_w, model.vit_merger_ds_up_b,
|
||||
nullptr, nullptr,
|
||||
model.vit_merger_ds_down_w, model.vit_merger_ds_down_b,
|
||||
FFN_GELU, -1);
|
||||
cb(cur, "vit_merger_ds_mlp_out", -1);
|
||||
|
||||
inpL = ggml_add(ctx0, cur, mean_res);
|
||||
cb(inpL, "vit_merger_ds_out", -1);
|
||||
}
|
||||
|
||||
inpL = build_vit_layers(inpL, insert_lid + 1, n_layer, n_ds);
|
||||
} else {
|
||||
inpL = build_vit_layers(inpL, 0, n_layer, n_pos);
|
||||
}
|
||||
|
||||
if (model.post_ln_w) {
|
||||
inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(inpL, "post_ln", -1);
|
||||
}
|
||||
|
||||
// Final Merger (DownsampleMLP): another 2x2 spatial merge -> projector embedding
|
||||
{
|
||||
ggml_tensor * p0 = ggml_get_rows(ctx0, inpL, merger_ds_idx_0);
|
||||
ggml_tensor * p1 = ggml_get_rows(ctx0, inpL, merger_ds_idx_1);
|
||||
ggml_tensor * p2 = ggml_get_rows(ctx0, inpL, merger_ds_idx_2);
|
||||
ggml_tensor * p3 = ggml_get_rows(ctx0, inpL, merger_ds_idx_3);
|
||||
|
||||
ggml_tensor * cat = ggml_concat(ctx0, p0, p1, 0);
|
||||
cat = ggml_concat(ctx0, cat, p2, 0);
|
||||
cat = ggml_concat(ctx0, cat, p3, 0);
|
||||
|
||||
ggml_tensor * cur = build_norm(cat,
|
||||
model.mm_input_norm_w, model.mm_input_norm_b,
|
||||
NORM_TYPE_NORMAL, eps, -1);
|
||||
cb(cur, "merger_normed", -1);
|
||||
|
||||
// MiniCPMV4_6DownsampleMLP uses nn.GELU() (erf-based, FFN_GELU_ERF)
|
||||
cur = build_ffn(cur,
|
||||
model.mm_ffn_up_w, model.mm_ffn_up_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_ffn_down_w, model.mm_ffn_down_b,
|
||||
FFN_GELU_ERF, -1);
|
||||
cb(cur, "merger_out", -1);
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
inpL = build_suffix(inpL);
|
||||
|
||||
ggml_build_forward_expand(gf, inpL);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_tensor * clip_graph_minimax_m3::apply_rope(
|
||||
ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) {
|
||||
const int dh = (int) x->ne[0];
|
||||
const int axd = 2 * ((2 * (dh / 2) / 3) / 2);
|
||||
|
||||
GGML_ASSERT(3 * axd <= dh);
|
||||
|
||||
const float th = hparams.rope_theta;
|
||||
|
||||
// layout of x is [t, h, w, pad]
|
||||
// t is unrotated, h and w are rotated, pad is unrotated
|
||||
x = ggml_rope_ext(ctx0, x, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
x = ggml_rope_set_offset(x, axd);
|
||||
|
||||
x = ggml_rope_ext(ctx0, x, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
x = ggml_rope_set_offset(x, 2 * axd);
|
||||
|
||||
return x;
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_minimax_m3::build() {
|
||||
GGML_ASSERT(model.patch_bias == nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1);
|
||||
GGML_ASSERT(model.mm_1_w && model.mm_2_w);
|
||||
GGML_ASSERT(model.mm_merger_fc1_w && model.mm_merger_fc2_w);
|
||||
|
||||
const int batch_size = 1;
|
||||
const int n_pos = n_patches;
|
||||
const int merge = hparams.n_merge;
|
||||
|
||||
// patch embedding
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
ggml_tensor * inp = ggml_add(ctx0,
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),
|
||||
ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
|
||||
|
||||
// spatial merge
|
||||
{
|
||||
inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);
|
||||
inp = ggml_cont_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, n_patches_y, batch_size);
|
||||
inp = ggml_reshape_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, merge, batch_size * (n_patches_y / merge));
|
||||
inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
|
||||
inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
|
||||
// t (time axis) is always 0 for now, so we leave it unrotated
|
||||
ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(pos_h, "minimax_pos_h"); ggml_set_input(pos_h);
|
||||
ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
|
||||
ggml_set_name(pos_w, "minimax_pos_w"); ggml_set_input(pos_w);
|
||||
|
||||
ggml_tensor * inpL = build_vit(
|
||||
inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr,
|
||||
[&](ggml_tensor * c, const clip_layer &) {
|
||||
return apply_rope(c, pos_h, pos_w);
|
||||
});
|
||||
|
||||
// projector
|
||||
ggml_tensor * emb = inpL;
|
||||
emb = build_ffn(emb, model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b, FFN_GELU_ERF, -1);
|
||||
|
||||
const int64_t proj = emb->ne[0];
|
||||
emb = ggml_reshape_2d(ctx0, emb, proj * merge * merge, n_pos / (merge * merge));
|
||||
|
||||
emb = build_ffn(emb, model.mm_merger_fc1_w, model.mm_merger_fc1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_merger_fc2_w, model.mm_merger_fc2_b, FFN_GELU_ERF, -1);
|
||||
|
||||
ggml_build_forward_expand(gf, emb);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
#include "models.h"
|
||||
|
||||
// Helpers for MobileNetV5 Blocks
|
||||
// RMS Norm 2D - normalizes over channels for each spatial position
|
||||
ggml_tensor * clip_graph_mobilenetv5::rms_norm_2d(ggml_tensor * inp, ggml_tensor * weight, float eps) {
|
||||
// inp: [W, H, C, B]
|
||||
|
||||
ggml_tensor * cur = ggml_permute(ctx0, inp, 2, 1, 0, 3);
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
cur = ggml_rms_norm(ctx0, cur, eps);
|
||||
|
||||
if (weight) {
|
||||
cur = ggml_mul(ctx0, cur, weight);
|
||||
}
|
||||
|
||||
cur = ggml_permute(ctx0, cur, 2, 1, 0, 3);
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
// Conv2dSame padding - asymmetric SAME padding like PyTorch/TF
|
||||
ggml_tensor* clip_graph_mobilenetv5::pad_same_2d(ggml_tensor* inp, int kernel_h, int kernel_w, int stride_h, int stride_w, int dilation_h, int dilation_w) {
|
||||
const int64_t ih = inp->ne[1]; // height
|
||||
const int64_t iw = inp->ne[0]; // width
|
||||
|
||||
// Calculate output size (ceil division)
|
||||
const int64_t oh = (ih + stride_h - 1) / stride_h;
|
||||
const int64_t ow = (iw + stride_w - 1) / stride_w;
|
||||
|
||||
// Calculate padding needed
|
||||
const int64_t pad_h = std::max((int64_t)0, (oh - 1) * stride_h + (kernel_h - 1) * dilation_h + 1 - ih);
|
||||
const int64_t pad_w = std::max((int64_t)0, (ow - 1) * stride_w + (kernel_w - 1) * dilation_w + 1 - iw);
|
||||
|
||||
// Split padding asymmetrically
|
||||
const int pad_h_top = pad_h / 2;
|
||||
const int pad_h_bottom = pad_h - pad_h_top;
|
||||
const int pad_w_left = pad_w / 2;
|
||||
const int pad_w_right = pad_w - pad_w_left;
|
||||
|
||||
// Apply padding if needed
|
||||
// ggml_pad_ext: (ctx, tensor, lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3)
|
||||
// For [W, H, C, B]: p0=width, p1=height, p2=channels, p3=batch
|
||||
if (pad_h > 0 || pad_w > 0) {
|
||||
inp = ggml_pad_ext(ctx0, inp,
|
||||
pad_w_left, pad_w_right, // width padding (dim 0)
|
||||
pad_h_top, pad_h_bottom, // height padding (dim 1)
|
||||
0, 0, // no channel padding (dim 2)
|
||||
0, 0); // no batch padding (dim 3)
|
||||
}
|
||||
|
||||
return inp;
|
||||
}
|
||||
|
||||
|
||||
// Edge Residual Block (Stage 0)
|
||||
ggml_tensor * clip_graph_mobilenetv5::build_edge_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) {
|
||||
ggml_tensor * cur = inp;
|
||||
|
||||
// 1. Expansion Conv (3x3)
|
||||
if (stride == 2) {
|
||||
// Case: Downsampling (Block 0)
|
||||
// Replicates Conv2dSame(kernel=3, stride=2)
|
||||
cur = pad_same_2d(cur, 3, 3, stride, stride);
|
||||
cur = ggml_conv_2d_direct(ctx0, block.s0_conv_exp_w, cur, stride, stride, 0, 0, 1, 1);
|
||||
} else {
|
||||
// Case: Normal 3x3 Block (Block 1, 2)
|
||||
// Replicates Conv2d(kernel=3, stride=1, padding=1)
|
||||
cur = ggml_conv_2d_direct(ctx0, block.s0_conv_exp_w, cur, stride, stride, 1, 1, 1, 1);
|
||||
}
|
||||
|
||||
// BN + Activation
|
||||
if (block.s0_bn1_w) cur = rms_norm_2d(cur, block.s0_bn1_w);
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
|
||||
// 2. Pointwise Linear Conv (1x1)
|
||||
// 1x1 Convs usually have padding=0 and stride=1
|
||||
cur = ggml_conv_2d_direct(ctx0, block.s0_conv_pwl_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
if (block.s0_bn2_w) cur = rms_norm_2d(cur, block.s0_bn2_w);
|
||||
|
||||
// 3. Residual Connection
|
||||
// Only apply residual if spatial dimensions and channels match (stride 1)
|
||||
if (stride == 1 && inp->ne[2] == cur->ne[2] && inp->ne[0] == cur->ne[0]) {
|
||||
cur = ggml_add(ctx0, cur, inp);
|
||||
}
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
// Universal Inverted Residual Block (Stage 1+)
|
||||
ggml_tensor * clip_graph_mobilenetv5::build_inverted_residual(ggml_tensor * inp, const mobilenetv5_block & block, int stride) {
|
||||
ggml_tensor * cur = inp;
|
||||
|
||||
// 1. Depthwise Start (Optional)
|
||||
// NOTE: dw_start always has stride=1 (no downsampling here)
|
||||
if (block.dw_start_w) {
|
||||
int k = block.dw_start_w->ne[0]; // 3 or 5
|
||||
int p = k / 2;
|
||||
cur = ggml_conv_2d_dw(ctx0, block.dw_start_w, cur, 1, 1, p, p, 1, 1);
|
||||
if (block.dw_start_bn_w) cur = rms_norm_2d(cur, block.dw_start_bn_w);
|
||||
}
|
||||
|
||||
// 2. Pointwise Expansion (1x1)
|
||||
if (block.pw_exp_w) {
|
||||
// Standard 1x1 conv, pad=0, stride=1
|
||||
cur = ggml_conv_2d_direct(ctx0, block.pw_exp_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
if (block.pw_exp_bn_w) cur = rms_norm_2d(cur, block.pw_exp_bn_w);
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
}
|
||||
|
||||
// 3. Depthwise Mid (Optional)
|
||||
// NOTE: dw_mid is where downsampling happens (stride=2 for first block of stage)
|
||||
if (block.dw_mid_w) {
|
||||
int k = block.dw_mid_w->ne[0]; // 3 or 5
|
||||
|
||||
if (stride > 1) {
|
||||
// Case: Stride 2 (Downsample) -> Use Asymmetric "Same" Padding
|
||||
cur = pad_same_2d(cur, k, k, stride, stride);
|
||||
cur = ggml_conv_2d_dw(ctx0, block.dw_mid_w, cur, stride, stride, 0, 0, 1, 1); // pad=0
|
||||
} else {
|
||||
// Case: Stride 1 -> Use Standard Symmetric Padding
|
||||
int p = k / 2;
|
||||
cur = ggml_conv_2d_dw(ctx0, block.dw_mid_w, cur, stride, stride, p, p, 1, 1);
|
||||
}
|
||||
|
||||
if (block.dw_mid_bn_w) cur = rms_norm_2d(cur, block.dw_mid_bn_w);
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
}
|
||||
|
||||
// 4. Pointwise Projection (1x1)
|
||||
if (block.pw_proj_w) {
|
||||
cur = ggml_conv_2d_direct(ctx0, block.pw_proj_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
if (block.pw_proj_bn_w) cur = rms_norm_2d(cur, block.pw_proj_bn_w);
|
||||
}
|
||||
|
||||
// Apply Layer Scaling if present
|
||||
if (block.layer_scale_w) {
|
||||
cur = ggml_mul(ctx0, cur, block.layer_scale_w);
|
||||
}
|
||||
|
||||
// 5. Residual Connection
|
||||
bool same_spatial = (inp->ne[0] == cur->ne[0]) && (inp->ne[1] == cur->ne[1]);
|
||||
bool same_channel = (inp->ne[2] == cur->ne[2]);
|
||||
if (same_spatial && same_channel) {
|
||||
cur = ggml_add(ctx0, cur, inp);
|
||||
}
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
// Attention Block (MQA)
|
||||
ggml_tensor * clip_graph_mobilenetv5::build_mobilenet_attn(ggml_tensor * inp, const mobilenetv5_block & block) {
|
||||
ggml_tensor * cur = inp;
|
||||
|
||||
// Norm
|
||||
if (block.attn_norm_w) {
|
||||
cur = rms_norm_2d(cur, block.attn_norm_w, 1e-6f);
|
||||
}
|
||||
|
||||
// 1. Q Calculation
|
||||
ggml_tensor * q = ggml_conv_2d_direct(ctx0, block.attn_q_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
|
||||
// 2. K Calculation (Downsampled)
|
||||
// Uses Conv2dSame(640, 640, kernel_size=(3, 3), stride=(2, 2), groups=640)
|
||||
ggml_tensor * k_inp = cur;
|
||||
if (block.attn_k_dw_w) {
|
||||
int k_size = block.attn_k_dw_w->ne[0]; // Usually 3
|
||||
k_inp = pad_same_2d(cur, k_size, k_size, 2, 2); // Apply SAME padding
|
||||
k_inp = ggml_conv_2d_dw(ctx0, block.attn_k_dw_w, k_inp, 2, 2, 0, 0, 1, 1); // padding=0
|
||||
if (block.attn_k_norm_w) {
|
||||
k_inp = rms_norm_2d(k_inp, block.attn_k_norm_w, 1e-6f);
|
||||
}
|
||||
}
|
||||
ggml_tensor * k = ggml_conv_2d_direct(ctx0, block.attn_k_w, k_inp, 1, 1, 0, 0, 1, 1);
|
||||
|
||||
// 3. V Calculation (Downsampled)
|
||||
// Uses Conv2dSame(640, 640, kernel_size=(3, 3), stride=(2, 2), groups=640)
|
||||
ggml_tensor * v_inp = cur;
|
||||
if (block.attn_v_dw_w) {
|
||||
int v_size = block.attn_v_dw_w->ne[0]; // Usually 3
|
||||
v_inp = pad_same_2d(cur, v_size, v_size, 2, 2); // Apply SAME padding
|
||||
v_inp = ggml_conv_2d_dw(ctx0, block.attn_v_dw_w, v_inp, 2, 2, 0, 0, 1, 1); // padding=0
|
||||
if (block.attn_v_norm_w) {
|
||||
v_inp = rms_norm_2d(v_inp, block.attn_v_norm_w, 1e-6f);
|
||||
}
|
||||
}
|
||||
ggml_tensor * v = ggml_conv_2d_direct(ctx0, block.attn_v_w, v_inp, 1, 1, 0, 0, 1, 1);
|
||||
|
||||
const int W = cur->ne[0]; const int H = cur->ne[1]; const int B = cur->ne[3];
|
||||
const int D = k->ne[2]; // Head dimension
|
||||
const int n_head = q->ne[2] / D;
|
||||
const int N = W * H;
|
||||
|
||||
// Process Q: [W, H, D*n_head, B] -> [D, N, n_head, B]
|
||||
q = ggml_reshape_3d(ctx0, q, N, D*n_head, B);
|
||||
q = ggml_reshape_4d(ctx0, q, N, D, n_head, B);
|
||||
q = ggml_permute(ctx0, q, 1, 0, 2, 3); // [D, N, n_head, B]
|
||||
q = ggml_cont(ctx0, q);
|
||||
|
||||
const int Wk = k->ne[0]; const int Hk = k->ne[1];
|
||||
const int M = Wk * Hk;
|
||||
|
||||
// Process K: [Wk, Hk, D, B] -> [D, M, 1, B]
|
||||
k = ggml_reshape_3d(ctx0, k, M, D, B);
|
||||
k = ggml_reshape_4d(ctx0, k, M, D, 1, B);
|
||||
k = ggml_permute(ctx0, k, 1, 0, 2, 3); // [D, M, 1, B]
|
||||
k = ggml_cont(ctx0, k);
|
||||
|
||||
// Process V: [Wk, Hk, D, B] -> [M, D, 1, B]
|
||||
v = ggml_reshape_3d(ctx0, v, M, D, B);
|
||||
v = ggml_reshape_4d(ctx0, v, M, D, 1, B);
|
||||
v = ggml_cont(ctx0, v); // [M, D, 1, B]
|
||||
|
||||
// Multi-Query Attention
|
||||
float scale = 1.0f / sqrtf((float)D);
|
||||
|
||||
// Step 1: Compute Q @ K.T
|
||||
ggml_tensor * scores = ggml_mul_mat(ctx0, k, q);
|
||||
|
||||
scores = ggml_scale(ctx0, scores, scale);
|
||||
|
||||
scores = ggml_soft_max(ctx0, scores);
|
||||
|
||||
ggml_tensor * kqv = ggml_mul_mat(ctx0, v, scores);
|
||||
|
||||
kqv = ggml_permute(ctx0, kqv, 1, 0, 2, 3);
|
||||
kqv = ggml_cont(ctx0, kqv);
|
||||
|
||||
|
||||
kqv = ggml_reshape_3d(ctx0, kqv, N, D * n_head, B);
|
||||
kqv = ggml_reshape_4d(ctx0, kqv, W, H, D * n_head, B);
|
||||
kqv = ggml_cont(ctx0, kqv);
|
||||
|
||||
// Output projection
|
||||
cur = ggml_conv_2d_direct(ctx0, block.attn_o_w, kqv, 1, 1, 0, 0, 1, 1);
|
||||
|
||||
// Residual & Layer Scale
|
||||
if (inp->ne[0] == cur->ne[0] && inp->ne[2] == cur->ne[2]) {
|
||||
if (block.layer_scale_w) {
|
||||
cur = ggml_mul(ctx0, cur, block.layer_scale_w);
|
||||
}
|
||||
cur = ggml_add(ctx0, cur, inp);
|
||||
}
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_mobilenetv5::build() {
|
||||
ggml_tensor * inp = build_inp_raw();
|
||||
|
||||
// 1. Stem - Conv2dSame(3, 64, kernel_size=(3, 3), stride=(2, 2))
|
||||
ggml_tensor * cur = pad_same_2d(inp, 3, 3, 2, 2); // Apply SAME padding
|
||||
|
||||
cur = ggml_conv_2d_direct(ctx0, model.mobilenet_stem_conv_w, cur, 2, 2, 0, 0, 1, 1); // padding=0
|
||||
if (model.mobilenet_stem_conv_b) {
|
||||
cur = ggml_add(ctx0, cur, model.mobilenet_stem_conv_b);
|
||||
}
|
||||
if (model.mobilenet_stem_norm_w) cur = rms_norm_2d(cur, model.mobilenet_stem_norm_w);
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
|
||||
|
||||
// 2. Blocks
|
||||
std::vector<ggml_tensor*> intermediate_features;
|
||||
const int total_blocks = model.mobilenet_blocks.size();
|
||||
|
||||
auto is_stage_start = [&](int i) {
|
||||
if (i == 0) return true;
|
||||
for (int end_idx : model.mobilenet_stage_ends) {
|
||||
if (i == end_idx + 1) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
auto is_fusion_point = [&](int i) {
|
||||
if (model.mobilenet_stage_ends.size() >= 4) {
|
||||
if (i == model.mobilenet_stage_ends[2]) return true; // End of Stage 2
|
||||
if (i == model.mobilenet_stage_ends[3]) return true; // End of Stage 3
|
||||
} else {
|
||||
if (i == total_blocks - 1) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for (int i = 0; i < total_blocks; i++) {
|
||||
const auto & block = model.mobilenet_blocks[i];
|
||||
int stride = is_stage_start(i) ? 2 : 1;
|
||||
|
||||
if (block.s0_conv_exp_w) cur = build_edge_residual(cur, block, stride);
|
||||
else if (block.attn_q_w) cur = build_mobilenet_attn(cur, block);
|
||||
else cur = build_inverted_residual(cur, block, stride);
|
||||
|
||||
if (is_fusion_point(i)) {
|
||||
|
||||
intermediate_features.push_back(cur);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Multi-Scale Fusion Adapter (MSFA)
|
||||
if (!intermediate_features.empty()) {
|
||||
|
||||
// A. Reference Resolution: PyTorch implementation uses inputs[0]
|
||||
// We assume intermediate_features[0] is the "High Resolution" target.
|
||||
// In MobileNet designs, this is typically the feature map with the smallest stride (e.g. 32x32).
|
||||
ggml_tensor* target_feat = intermediate_features[0];
|
||||
int high_res_w = target_feat->ne[0];
|
||||
int high_res_h = target_feat->ne[1];
|
||||
|
||||
std::vector<ggml_tensor*> resized_feats;
|
||||
|
||||
// B. Resize inputs to match inputs[0] (High Resolution)
|
||||
for (auto feat : intermediate_features) {
|
||||
int feat_w = feat->ne[0];
|
||||
int feat_h = feat->ne[1];
|
||||
|
||||
// PyTorch: if feat_size < high_resolution: interpolate
|
||||
if (feat_w < high_res_w || feat_h < high_res_h) {
|
||||
// Calculate scale factor.
|
||||
// Note: PyTorch 'nearest' works on arbitrary float scales.
|
||||
// ggml_upscale generally takes integer factors or target sizes depending on helper.
|
||||
// Assuming standard power-of-2 scaling (e.g. 16 -> 32 means scale=2).
|
||||
int scale_w = high_res_w / feat_w;
|
||||
// int scale_h = high_res_h / feat_h;
|
||||
|
||||
// Safety check for non-integer scaling if strictly replicating
|
||||
GGML_ASSERT(high_res_w % feat_w == 0);
|
||||
|
||||
// Upsample (Nearest Neighbor)
|
||||
// 2 is the scale factor
|
||||
feat = ggml_upscale(ctx0, feat, scale_w, ggml_scale_mode::GGML_SCALE_MODE_NEAREST);
|
||||
}
|
||||
resized_feats.push_back(feat);
|
||||
}
|
||||
|
||||
// C. Concatenate at High Resolution (Channel Dim = 2 in ggml)
|
||||
cur = resized_feats[0];
|
||||
for (size_t k = 1; k < resized_feats.size(); ++k) {
|
||||
cur = ggml_concat(ctx0, cur, resized_feats[k], 2);
|
||||
}
|
||||
|
||||
// D. FFN (UniversalInvertedResidual)
|
||||
// Structure: Expand Conv -> Norm -> GELU -> Project Conv -> Norm
|
||||
|
||||
// 1. Expansion
|
||||
if (model.msfa_ffn_expand_w) {
|
||||
// 1x1 Conv
|
||||
cur = ggml_conv_2d_direct(ctx0, model.msfa_ffn_expand_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
|
||||
if (model.msfa_ffn_expand_bn) {
|
||||
cur = rms_norm_2d(cur, model.msfa_ffn_expand_bn);
|
||||
}
|
||||
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
|
||||
}
|
||||
|
||||
// 2. Projection (No DW because kernel_size=0)
|
||||
if (model.msfa_ffn_project_w) {
|
||||
// 1x1 Conv
|
||||
cur = ggml_conv_2d_direct(ctx0, model.msfa_ffn_project_w, cur, 1, 1, 0, 0, 1, 1);
|
||||
|
||||
// UniversalInvertedResidual typically has a norm after projection
|
||||
if (model.msfa_ffn_project_bn) {
|
||||
cur = rms_norm_2d(cur, model.msfa_ffn_project_bn);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// E. Final Downsample to Target Resolution (Output Resolution)
|
||||
// PyTorch: matches self.output_resolution (e.g. 16x16)
|
||||
const int target_out_res = 16;
|
||||
int current_w = cur->ne[0];
|
||||
|
||||
if (current_w > target_out_res) {
|
||||
int s = current_w / target_out_res;
|
||||
|
||||
GGML_ASSERT(current_w % target_out_res == 0);
|
||||
|
||||
// Avg Pool: Kernel=s, Stride=s
|
||||
cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG, s, s, s, s, 0, 0);
|
||||
|
||||
}
|
||||
|
||||
// F. Final Norm
|
||||
if (model.msfa_concat_norm_w) {
|
||||
cur = rms_norm_2d(cur, model.msfa_concat_norm_w);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Gemma 3n Multimodal Projection (Embedder)
|
||||
// Input: 'cur' is [Width, Height, Channels, Batch]
|
||||
int W = cur->ne[0];
|
||||
int H = cur->ne[1];
|
||||
int C = cur->ne[2];
|
||||
int B = cur->ne[3];
|
||||
|
||||
GGML_ASSERT(C == hparams.n_embd);
|
||||
|
||||
// 1. Permute and Flatten to [Channels, Tokens, Batch]
|
||||
// PyTorch expects (Batch, Seq, Hidden), GGML usually processes (Hidden, Seq, Batch)
|
||||
cur = ggml_permute(ctx0, cur, 2, 1, 0, 3); // -> [C, H, W, B]
|
||||
cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); // -> [C, W, H, B]
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
cur = ggml_reshape_3d(ctx0, cur, C, W*H, B);
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
|
||||
// 2. FEATURE SCALING
|
||||
// PyTorch: vision_outputs *= self.config.vision_config.hidden_size**0.5
|
||||
const float scale_factor = sqrtf((float)C);
|
||||
cur = ggml_scale(ctx0, cur, scale_factor);
|
||||
|
||||
|
||||
// 3. SOFT EMBEDDING NORM
|
||||
// PyTorch: self._norm(x) * self.weight
|
||||
// We must normalize regardless, then multiply if weight exists.
|
||||
{
|
||||
const float eps = 1e-6f; // Gemma3n uses 1e-6
|
||||
cur = ggml_rms_norm(ctx0, cur, eps);
|
||||
|
||||
if (model.mm_soft_emb_norm_w) {
|
||||
// Weight shape is (2048,) -> Element-wise broadcast multiply
|
||||
cur = ggml_mul(ctx0, cur, model.mm_soft_emb_norm_w);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// 4. PROJECTION
|
||||
// PyTorch: embedding_projection = nn.Linear(vision_hidden, text_hidden, bias=False)
|
||||
// Weight stored as [out_features, in_features] = [text_hidden_size, vision_hidden_size]
|
||||
if (model.mm_input_proj_w) {
|
||||
cur = build_mm(model.mm_input_proj_w, cur);
|
||||
}
|
||||
|
||||
// 5. POST PROJECTION NORM
|
||||
// PyTorch: embedding_post_projection_norm = Gemma3nRMSNorm(..., with_scale=False)
|
||||
// with_scale=False means weight is registered as buffer with value 1.0
|
||||
// So output = rms_norm(x) * 1.0 = rms_norm(x), magnitude ~1
|
||||
{
|
||||
const float eps = 1e-6f;
|
||||
cur = ggml_rms_norm(ctx0, cur, eps);
|
||||
|
||||
if (model.mm_post_proj_norm_w) {
|
||||
// If weight is loaded, multiply (should be ~1.0 anyway)
|
||||
cur = ggml_mul(ctx0, cur, model.mm_post_proj_norm_w);
|
||||
}
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
#pragma once
|
||||
|
||||
#include "../clip-graph.h"
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
/*
|
||||
* IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated.
|
||||
* We encourage human contributors to ensure the quality and reliability of the codebase.
|
||||
*/
|
||||
|
||||
struct clip_graph_siglip : clip_graph {
|
||||
clip_graph_siglip(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_gemma4v : clip_graph {
|
||||
clip_graph_gemma4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
|
||||
bool support_batch() const override { return true; }
|
||||
};
|
||||
|
||||
struct clip_graph_gemma4uv : clip_graph {
|
||||
clip_graph_gemma4uv(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_pixtral : clip_graph {
|
||||
clip_graph_pixtral(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_deepseek4v : clip_graph {
|
||||
clip_graph_deepseek4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_qwen2vl : clip_graph {
|
||||
clip_graph_qwen2vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_qwen3vl : clip_graph_qwen2vl {
|
||||
clip_graph_qwen3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen2vl(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_sensenova_u1 : clip_graph {
|
||||
clip_graph_sensenova_u1(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_lfm3vl : clip_graph {
|
||||
clip_graph_lfm3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_ling3vl : clip_graph_qwen3vl {
|
||||
clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_minimax_m3 : clip_graph {
|
||||
clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w);
|
||||
};
|
||||
|
||||
struct clip_graph_mimovl : clip_graph {
|
||||
clip_graph_mimovl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
// Force F32 mat-mul accumulation to avoid F16 overflow in the FFN down-proj
|
||||
// when the mmproj is stored in F16 (the source weights are BF16; downcasting
|
||||
// to F16 reduces dynamic range below the SwiGLU output magnitude on the last few layers).
|
||||
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
|
||||
};
|
||||
|
||||
struct clip_graph_step3vl : clip_graph {
|
||||
clip_graph_step3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_youtuvl : clip_graph {
|
||||
clip_graph_youtuvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_yasa2 : clip_graph {
|
||||
clip_graph_yasa2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
ggml_tensor * layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps = 1e-6f);
|
||||
ggml_tensor * convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b);
|
||||
};
|
||||
|
||||
struct clip_graph_minicpmv : clip_graph {
|
||||
clip_graph_minicpmv(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_minicpmv4_6 : clip_graph {
|
||||
clip_graph_minicpmv4_6(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_internvl : clip_graph {
|
||||
clip_graph_internvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
bool support_batch() const override { return true; }
|
||||
};
|
||||
|
||||
struct clip_graph_nemotron_v2_vl : clip_graph {
|
||||
clip_graph_nemotron_v2_vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_nemotron_omni : clip_graph {
|
||||
clip_graph_nemotron_omni(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
ggml_tensor * build_inp_with_temporal_merge();
|
||||
};
|
||||
|
||||
struct clip_graph_llama4 : clip_graph {
|
||||
clip_graph_llama4(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_kimivl : clip_graph {
|
||||
clip_graph_kimivl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_paddleocr : clip_graph {
|
||||
clip_graph_paddleocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_dotsocr : clip_graph {
|
||||
clip_graph_dotsocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_dots3note_a : clip_graph {
|
||||
clip_graph_dots3note_a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_cogvlm : clip_graph {
|
||||
clip_graph_cogvlm(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_llava : clip_graph {
|
||||
clip_graph_llava(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_whisper_enc : clip_graph {
|
||||
clip_graph_whisper_enc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
// causal whisper-like encoder with a sliding window KV cache (voxtral realtime)
|
||||
struct clip_graph_voxtral_rt : clip_graph {
|
||||
clip_enc_state * state; // nullptr during warmup, the cache is then a graph input
|
||||
clip_graph_voxtral_rt(clip_ctx * ctx, const clip_image_f32 & img, clip_enc_state * state) : clip_graph(ctx, img), state(state) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_deepseekocr : clip_graph {
|
||||
clip_graph_deepseekocr(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
ggml_tensor * build_sam(ggml_tensor * inp); // build the SAM model
|
||||
bool support_batch() const override { return true; }
|
||||
};
|
||||
|
||||
struct clip_graph_deepseekocr2 : clip_graph_deepseekocr {
|
||||
clip_graph_deepseekocr2(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_deepseekocr(ctx, img) {}
|
||||
ggml_cgraph * build() override; // reuses build_sam() from base
|
||||
bool support_batch() const override { return true; }
|
||||
};
|
||||
|
||||
struct clip_graph_conformer : clip_graph {
|
||||
clip_graph_conformer(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_granite_speech : clip_graph {
|
||||
clip_graph_granite_speech(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_muse_spark_audio : clip_graph {
|
||||
clip_graph_muse_spark_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_gemma4a : clip_graph {
|
||||
clip_graph_gemma4a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const override;
|
||||
};
|
||||
|
||||
struct clip_graph_gemma4ua : clip_graph {
|
||||
clip_graph_gemma4ua(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_glm4v : clip_graph {
|
||||
clip_graph_glm4v(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_hunyuanvl : clip_graph {
|
||||
clip_graph_hunyuanvl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_mobilenetv5 : clip_graph {
|
||||
clip_graph_mobilenetv5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
ggml_tensor * rms_norm_2d(
|
||||
ggml_tensor * inp,
|
||||
ggml_tensor * weight,
|
||||
float eps = 1e-6f);
|
||||
|
||||
ggml_tensor* pad_same_2d(
|
||||
ggml_tensor* inp,
|
||||
int kernel_h,
|
||||
int kernel_w,
|
||||
int stride_h,
|
||||
int stride_w,
|
||||
int dilation_h = 1,
|
||||
int dilation_w = 1);
|
||||
|
||||
ggml_tensor * build_edge_residual(
|
||||
ggml_tensor * inp,
|
||||
const mobilenetv5_block & block,
|
||||
int stride);
|
||||
|
||||
ggml_tensor * build_inverted_residual(
|
||||
ggml_tensor * inp,
|
||||
const mobilenetv5_block & block,
|
||||
int stride);
|
||||
|
||||
ggml_tensor * build_mobilenet_attn(
|
||||
ggml_tensor * inp,
|
||||
const mobilenetv5_block & block);
|
||||
};
|
||||
|
||||
struct clip_graph_qwen3a : clip_graph {
|
||||
clip_graph_qwen3a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_mimo_audio : clip_graph {
|
||||
clip_graph_mimo_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_qwen3tts_spkenc : clip_graph {
|
||||
clip_graph_qwen3tts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
ggml_tensor * conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
|
||||
ggml_tensor * res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const;
|
||||
ggml_tensor * se_block(ggml_tensor * x, const clip_layer & layer) const;
|
||||
ggml_tensor * se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const;
|
||||
ggml_tensor * attentive_stats_pool(ggml_tensor * x) const;
|
||||
};
|
||||
|
||||
struct clip_graph_qwen3tts_gen : clip_graph {
|
||||
clip_graph_qwen3tts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int top_k, float top_p)
|
||||
: clip_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
// which sub-graph build() constructs, fixed at graph-build time
|
||||
clip_gen_process_type gen_process;
|
||||
|
||||
// sampling params, fixed at graph-build time (GEN_CODE only)
|
||||
int top_k;
|
||||
float top_p;
|
||||
|
||||
//
|
||||
// code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes
|
||||
// MTP-style code predictor, one token per codebook
|
||||
//
|
||||
struct code_gen : clip_graph {
|
||||
code_gen(const clip_graph & parent, int top_k, float top_p)
|
||||
: clip_graph(parent), top_k(top_k), top_p(top_p) {}
|
||||
ggml_cgraph * build() override { GGML_ABORT("call prefill()/step() instead"); }
|
||||
|
||||
int top_k;
|
||||
float top_p;
|
||||
|
||||
ggml_tensor * cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const;
|
||||
ggml_tensor * do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const;
|
||||
|
||||
ggml_tensor * const_i32(ggml_tensor * anchor, float value) const;
|
||||
ggml_tensor * causal_mask_row(int64_t n_kv_pad, int pos) const;
|
||||
ggml_tensor * project_in(ggml_tensor * cur) const;
|
||||
|
||||
ggml_tensor * layer_forward(
|
||||
ggml_tensor * cur,
|
||||
const clip_layer & layer,
|
||||
ggml_tensor * inp_pos,
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor *& k_cache_layer,
|
||||
ggml_tensor *& v_cache_layer,
|
||||
int64_t n_kv_pad,
|
||||
int pos,
|
||||
int il) const;
|
||||
|
||||
void prefill(
|
||||
std::vector<ggml_tensor *> & k_cache,
|
||||
std::vector<ggml_tensor *> & v_cache,
|
||||
ggml_tensor *& out_code_cache,
|
||||
ggml_tensor * h_state,
|
||||
ggml_tensor * code0_embd,
|
||||
ggml_tensor * inp_rand) const;
|
||||
|
||||
ggml_tensor * step(
|
||||
std::vector<ggml_tensor *> & k_cache,
|
||||
std::vector<ggml_tensor *> & v_cache,
|
||||
ggml_tensor * out_code_cache,
|
||||
ggml_tensor * inp_rand,
|
||||
int step_idx) const;
|
||||
};
|
||||
|
||||
//
|
||||
// code2wav: RVQ codes -> raw PCM (quantizer + pre_conv + pre_transformer + upsample + DAC).
|
||||
//
|
||||
struct code2wav : clip_graph {
|
||||
code2wav(const clip_graph & parent) : clip_graph(parent) {}
|
||||
ggml_cgraph * build() override { GGML_ABORT("call decode() instead"); }
|
||||
|
||||
// state_in: previous call's persisted state, by slot name (see list_c2w_state_slots())
|
||||
std::map<std::string, ggml_tensor *> state_in;
|
||||
// state_out: this call's state to persist, added to the graph outputs by build()
|
||||
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
|
||||
|
||||
// stateful conv ops: read/update their state via state_in/state_out[state_name]
|
||||
ggml_tensor * causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const;
|
||||
ggml_tensor * causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const;
|
||||
ggml_tensor * causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const;
|
||||
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
|
||||
|
||||
ggml_tensor * quant_decode(ggml_tensor * inp_codes) const;
|
||||
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const;
|
||||
ggml_tensor * convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const;
|
||||
ggml_tensor * dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const;
|
||||
|
||||
// inp_codes [1, n_codes] I32 -> this frame's audio samples [n_samples] F32, clamped to [-1, 1]
|
||||
ggml_tensor * decode(ggml_tensor * inp_codes) const;
|
||||
};
|
||||
};
|
||||
|
||||
//
|
||||
// pocket-tts: SEANet convolution stack, shared by the voice encoder and the mimi decoder.
|
||||
// stateless unless state_in is populated: convs then pad instead of carrying left-context.
|
||||
//
|
||||
struct clip_graph_pockettts_seanet : clip_graph {
|
||||
clip_graph_pockettts_seanet(const clip_graph & parent) : clip_graph(parent) {}
|
||||
ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); }
|
||||
|
||||
// per-call streaming state, keyed by slot name (see list_pockettts_state_slots)
|
||||
std::map<std::string, ggml_tensor *> state_in;
|
||||
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
|
||||
|
||||
ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation,
|
||||
bool pad_replicate = false, const std::string & state_name = "") const;
|
||||
ggml_tensor * conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride,
|
||||
const std::string & state_name = "") const;
|
||||
ggml_tensor * res_unit(ggml_tensor * x, const clip_seanet::stage & stage, int dilation,
|
||||
const std::string & state_prefix = "") const;
|
||||
|
||||
// x: [T, C] -> [T / hop, dim]
|
||||
ggml_tensor * encode(ggml_tensor * x) const;
|
||||
// x: [T, dim] -> [T * hop, 1], streams when state_in is populated
|
||||
ggml_tensor * decode(ggml_tensor * x) const;
|
||||
};
|
||||
|
||||
// mimi encoder + speaker_proj: reference waveform -> voice conditioning rows
|
||||
struct clip_graph_pockettts_spkenc : clip_graph {
|
||||
clip_graph_pockettts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const;
|
||||
};
|
||||
|
||||
//
|
||||
// pocket-tts generation:
|
||||
// GEN_CODE = flow-matching decoder + end-of-speech head, one latent per call
|
||||
// GEN_WAV = mimi decoder, a window of latents -> PCM
|
||||
//
|
||||
struct clip_graph_pockettts_gen : clip_graph {
|
||||
clip_graph_pockettts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int n_step, int n_frames)
|
||||
: clip_graph(ctx, img), gen_process(gen_process), n_step(n_step), n_frames(n_frames) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
clip_gen_process_type gen_process;
|
||||
int n_step; // lsd_decode steps, fixed at graph-build time
|
||||
int n_frames; // GEN_WAV only: number of latents to decode
|
||||
|
||||
// AdaLN modulation: x * (1 + scale) + shift
|
||||
ggml_tensor * modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const;
|
||||
ggml_tensor * time_embed(const clip_flow_net::time_embd & te, float t) const;
|
||||
ggml_tensor * flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const;
|
||||
};
|
||||
|
||||
// one persisted state buffer used by code2wav, see qwen3tts-gen.cpp
|
||||
struct c2w_state_slot {
|
||||
std::string name;
|
||||
int64_t ne0;
|
||||
int64_t ne1;
|
||||
};
|
||||
std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model);
|
||||
|
||||
// same, for the streaming mimi decoder (pocket-tts GEN_WAV)
|
||||
std::vector<c2w_state_slot> list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model);
|
||||
|
||||
struct clip_graph_soprano : clip_graph {
|
||||
clip_graph_soprano(clip_ctx * ctx, const clip_image_f32 & img, int n_frames) : clip_graph(ctx, img), n_frames(n_frames) {}
|
||||
ggml_cgraph * build() override;
|
||||
static void decode_spectrum(const clip_model & model, const std::vector<float> & spectrum, std::vector<float> & pcm);
|
||||
// SopranoDecoder default
|
||||
static constexpr int upscale = 4;
|
||||
int n_frames;
|
||||
};
|
||||
|
||||
struct clip_graph_kimik25 : clip_graph {
|
||||
clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode);
|
||||
};
|
||||
|
||||
struct clip_graph_parakeet : clip_graph {
|
||||
clip_graph_parakeet(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_exaone4_5 : clip_graph {
|
||||
clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_granite4_vision : clip_graph {
|
||||
clip_graph_granite4_vision(clip_ctx * ctx, const clip_image_f32 & img)
|
||||
: clip_graph(ctx, img),
|
||||
anyres(img.anyres),
|
||||
n_tiles(img.ny() / img.nx()),
|
||||
tile_side(img.nx() / patch_size) {}
|
||||
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
private:
|
||||
// the input image is a stack of tiles on the Y axis: [overview, tile(0,0), tile(0,1), ...]
|
||||
const clip_image_f32::anyres_info anyres;
|
||||
const int n_tiles;
|
||||
const int tile_side; // patches per tile side
|
||||
|
||||
ggml_tensor * build_tile_inp();
|
||||
ggml_tensor * gather(ggml_tensor * src, const std::string & name, int idx_len);
|
||||
ggml_tensor * interp_down(ggml_tensor * src, int side, int new_side);
|
||||
ggml_tensor * build_block(const qf_block & blk, ggml_tensor * h, int bid,
|
||||
int spatial_offset, int image_side, int window_side,
|
||||
int query_side, float qformer_eps);
|
||||
|
||||
ggml_tensor * build_newline_row(ggml_context * ctx0);
|
||||
ggml_tensor * build_anyres_assembly(ggml_tensor * cur, int out_side);
|
||||
};
|
||||
|
||||
struct clip_graph_muse_glimmer : clip_graph {
|
||||
clip_graph_muse_glimmer(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
@@ -0,0 +1,86 @@
|
||||
#include "models.h"
|
||||
|
||||
// MuseGlimmer vision encoder: 50-layer ViT with 2D RoPE, sparse block-diagonal
|
||||
// window attention (every 4th + last layer global), pixel-shuffle downsample, then
|
||||
// adapter MLP + LLM's vision_projection.
|
||||
//
|
||||
// Several quantities are precomputed on host and fed as named graph inputs (filled in
|
||||
// clip.cpp set_input, PROJECTOR_TYPE_MUSE_GLIMMER branch):
|
||||
// muse_glimmer_pos_w/_h [n_tok] i32 : 1-indexed RoPE positions (sparse-permuted order)
|
||||
// muse_glimmer_sp_perm [n_tok] i32 : window grouping permutation (applied after ln_pre)
|
||||
// muse_glimmer_inv_perm [n_tok] i32 : inverse of sp_perm (applied after blocks)
|
||||
// muse_glimmer_ds_perm [n_tok] i32 : pixel-shuffle gather (original order)
|
||||
// attn_mask [n_tok, n_tok] f32 (f16 with flash attn) : block-diagonal window mask (sparse layers)
|
||||
ggml_cgraph * clip_graph_muse_glimmer::build() {
|
||||
const int ds = hparams.n_merge; // downsample factor (2)
|
||||
const int sf = hparams.muse_glimmer_sparse_factor; // 4
|
||||
const int n_tok = n_patches;
|
||||
const int n_out = (n_patches_x / ds) * (n_patches_y / ds);
|
||||
const float rope_base = hparams.rope_theta; // 10000
|
||||
|
||||
auto inp_i32 = [&](const char * name, int64_t n) {
|
||||
ggml_tensor * t = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n);
|
||||
ggml_set_name(t, name);
|
||||
ggml_set_input(t);
|
||||
return t;
|
||||
};
|
||||
|
||||
ggml_tensor * pos_w = inp_i32("muse_glimmer_pos_w", n_tok);
|
||||
ggml_tensor * pos_h = inp_i32("muse_glimmer_pos_h", n_tok);
|
||||
ggml_tensor * sp_perm = inp_i32("muse_glimmer_sp_perm", n_tok);
|
||||
ggml_tensor * inv_perm = inp_i32("muse_glimmer_inv_perm", n_tok);
|
||||
ggml_tensor * ds_perm = inp_i32("muse_glimmer_ds_perm", n_tok);
|
||||
|
||||
ggml_tensor * sp_mask = build_inp_attn_mask(n_tok, n_tok);
|
||||
|
||||
// patchify via build_inp (conv2d over raw pixels) + bilinear-resized learned pos-emb
|
||||
ggml_tensor * x = build_inp(); // [n_embd, n_tok, 1]
|
||||
x = ggml_add(ctx0, x, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));
|
||||
cb(x, "after_posemb", -1);
|
||||
|
||||
// group patches into pgrid x pgrid windows (sparse attention order)
|
||||
x = ggml_get_rows(ctx0, x, sp_perm);
|
||||
cb(x, "after_sp_perm", -1);
|
||||
|
||||
// per-layer mask: sparse layers get sp_mask, global layers (every sf-th and last) get none
|
||||
std::vector<ggml_tensor *> attn_mask_layers(n_layer);
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const bool is_global = (il == n_layer - 1) || ((il + 1) % sf == 0);
|
||||
attn_mask_layers[il] = is_global ? nullptr : sp_mask;
|
||||
}
|
||||
|
||||
// 2D RoPE: first half of head_dim uses width pos, second half uses height pos
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return build_rope_2d(ctx0, cur, pos_w, pos_h, rope_base, false);
|
||||
};
|
||||
|
||||
build_vit_opts opts;
|
||||
opts.attn_mask_layers = std::move(attn_mask_layers);
|
||||
|
||||
// pre_ln, per-layer transformer, post_ln (all inside build_vit); reference uses exact (erf) GELU
|
||||
x = build_vit(x, n_tok, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, add_pos, opts);
|
||||
|
||||
// un-permute back to original grid order
|
||||
x = ggml_get_rows(ctx0, x, inv_perm);
|
||||
cb(x, "after_inv_perm", -1);
|
||||
|
||||
// pixel-shuffle downsample: gather f*f spatial neighbors then concat channel-outer.
|
||||
// out[c*(ds*ds)+s, o] = x[ds_perm gathered][o*(ds*ds)+s, c]
|
||||
x = ggml_get_rows(ctx0, x, ds_perm); // [n_embd, n_tok], grouped
|
||||
x = ggml_reshape_3d(ctx0, x, n_embd, ds * ds, n_out);// [c, s, o]
|
||||
x = ggml_permute(ctx0, x, 1, 0, 2, 3); // [s, c, o]
|
||||
x = ggml_cont(ctx0, x);
|
||||
x = ggml_reshape_2d(ctx0, x, n_embd * ds * ds, n_out); // [6144, n_out]
|
||||
cb(x, "encoder_out", -1);
|
||||
|
||||
// adapter (6144->4096->4096, exact GELU each) + LLM vision_projection (4096->6656)
|
||||
x = build_mm(model.mm_0_w, x);
|
||||
x = ggml_gelu_erf(ctx0, x);
|
||||
x = build_mm(model.mm_1_w, x);
|
||||
x = ggml_gelu_erf(ctx0, x);
|
||||
x = build_mm(model.mm_2_w, x); // [6656, n_out]
|
||||
cb(x, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, x);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
#include "models.h"
|
||||
|
||||
// Muse Spark audio tower: a causal conv pyramid over the raw waveform, no attention and no spectrogram
|
||||
// each stage folds `patch` samples into the channel dim, projects, then runs `depth` residual blocks (depthwise causal conv, RMS norm, GELU MLP)
|
||||
ggml_cgraph * clip_graph_muse_spark_audio::build() {
|
||||
const int kernel = hparams.audio_conv_kernel_size;
|
||||
const int n_stage = (int) hparams.audio_patch_sizes.size();
|
||||
|
||||
// shape [1, n_samples], so the first stage folds the same way as the others
|
||||
ggml_tensor * cur = build_inp_raw(1);
|
||||
cur = ggml_reshape_2d(ctx0, cur, 1, img.nx());
|
||||
cb(cur, "inp_raw", -1);
|
||||
|
||||
int bid = 0;
|
||||
for (int stage = 0; stage < n_stage; ++stage) {
|
||||
const int patch = hparams.audio_patch_sizes[stage];
|
||||
const auto & pe = model.a_patches[stage];
|
||||
|
||||
// fold `patch` positions into the channel dim
|
||||
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0] * patch, cur->ne[1] / patch);
|
||||
cb(cur, "patch_folded", stage);
|
||||
|
||||
GGML_ASSERT(pe.proj_w->ne[1] == hparams.audio_hidden_sizes[stage]);
|
||||
cur = ggml_mul_mat(ctx0, pe.proj_w, cur);
|
||||
cur = ggml_add(ctx0, cur, pe.proj_b);
|
||||
cur = build_norm(cur, pe.norm_w, nullptr, NORM_TYPE_RMS, hparams.eps, stage);
|
||||
cb(cur, "patch_embd", stage);
|
||||
|
||||
for (int i = 0; i < hparams.audio_depths[stage]; ++i, ++bid) {
|
||||
const auto & layer = model.layers[bid];
|
||||
ggml_tensor * residual = cur;
|
||||
|
||||
// depthwise conv over time, pad on the left so it is causal
|
||||
ggml_tensor * conv = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
conv = ggml_pad_ext(ctx0, conv, kernel - 1, 0, 0, 0, 0, 0, 0, 0);
|
||||
conv = ggml_conv_1d_dw(ctx0, layer.conv_w, conv, 1, 0, 1);
|
||||
conv = ggml_cont(ctx0, ggml_transpose(ctx0, conv));
|
||||
conv = ggml_add(ctx0, conv, layer.conv_b);
|
||||
cb(conv, "conv", bid);
|
||||
|
||||
cur = build_norm(conv, layer.conv_norm_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
|
||||
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
nullptr, nullptr,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
FFN_GELU_ERF, bid);
|
||||
cb(cur, "ffn_out", bid);
|
||||
|
||||
cur = ggml_add(ctx0, cur, residual);
|
||||
cb(cur, "block_out", bid);
|
||||
}
|
||||
}
|
||||
|
||||
cur = build_norm(cur, model.pre_ln_w, nullptr, NORM_TYPE_RMS, hparams.eps, -1);
|
||||
cb(cur, "enc_norm", -1);
|
||||
|
||||
// straight into the text hidden size, no adapter
|
||||
cur = ggml_mul_mat(ctx0, model.mm_fc_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_fc_b);
|
||||
cur = build_norm(cur, model.post_ln_w, nullptr, NORM_TYPE_RMS, hparams.eps, -1);
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_tensor * clip_graph_nemotron_omni::build_inp_with_temporal_merge() {
|
||||
if (n_batch == 1 && !img_batch->is_video) {
|
||||
return build_inp();
|
||||
}
|
||||
|
||||
GGML_ASSERT(n_batch == 1 || n_batch == 2);
|
||||
GGML_ASSERT(model.patch_embeddings_1 != nullptr);
|
||||
GGML_ASSERT(model.patch_embeddings_1->ne[2] == 6);
|
||||
|
||||
ggml_tensor * inp_raw = build_inp_raw();
|
||||
if (n_batch == 1) {
|
||||
// pad the last video frame inside the graph
|
||||
inp_raw = ggml_concat(ctx0, inp_raw, inp_raw, 2);
|
||||
} else {
|
||||
inp_raw = ggml_reshape_4d(ctx0, inp_raw, img.nx(), img.ny(), 6, 1);
|
||||
}
|
||||
|
||||
ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
|
||||
inp = ggml_reshape_3d(ctx0, inp, n_patches, n_embd, 1);
|
||||
return ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_nemotron_omni::build() {
|
||||
GGML_ASSERT(model.class_embedding != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
|
||||
const int n_registers = model.class_embedding->ne[1];
|
||||
const int n_pos = n_patches + n_registers;
|
||||
|
||||
ggml_tensor * inp = build_inp_with_temporal_merge();
|
||||
|
||||
inp = ggml_add(ctx0, inp, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));
|
||||
cb(inp, "inp_pos", -1);
|
||||
|
||||
inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
|
||||
|
||||
ggml_tensor * cur = build_vit(inp, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, nullptr);
|
||||
|
||||
cur = ggml_view_2d(ctx0, cur,
|
||||
n_embd, n_patches,
|
||||
ggml_row_size(cur->type, n_embd),
|
||||
n_registers * ggml_row_size(cur->type, n_embd));
|
||||
|
||||
cur = build_patch_merge_permute(cur, model.hparams.n_merge);
|
||||
|
||||
{
|
||||
cur = build_norm(cur, model.mm_0_w, nullptr, NORM_TYPE_RMS, 1e-5, -1);
|
||||
cur = build_ffn(cur, model.mm_1_w, nullptr, nullptr, nullptr, model.mm_3_w, nullptr, FFN_RELU_SQR, -1);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_nemotron_v2_vl::build() {
|
||||
GGML_ASSERT(model.class_embedding != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
|
||||
const int n_registers = model.class_embedding->ne[1];
|
||||
const int n_pos = n_patches + n_registers;
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
|
||||
// add position embeddings (pre-downsampled during GGUF conversion for fixed 512x512 input)
|
||||
inp = ggml_add(ctx0, inp, model.position_embeddings);
|
||||
cb(inp, "inp_pos", -1);
|
||||
|
||||
inp = ggml_concat(ctx0, model.class_embedding, inp, 1);
|
||||
|
||||
ggml_tensor * cur = build_vit(inp, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, nullptr);
|
||||
|
||||
cur = ggml_view_2d(ctx0, cur,
|
||||
n_embd, n_patches,
|
||||
ggml_row_size(cur->type, n_embd),
|
||||
n_registers * ggml_row_size(cur->type, n_embd));
|
||||
|
||||
cur = build_patch_merge_permute(cur, model.hparams.n_merge);
|
||||
|
||||
{
|
||||
cur = build_norm(cur, model.mm_0_w, nullptr, NORM_TYPE_RMS, 1e-6, -1);
|
||||
cur = build_ffn(cur, model.mm_1_w, nullptr, nullptr, nullptr, model.mm_3_w, nullptr, FFN_RELU_SQR, -1);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_paddleocr::build() {
|
||||
const int n_pos = n_patches;
|
||||
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
|
||||
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return ggml_rope_multi(
|
||||
ctx0, cur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
|
||||
32768, 10000, 1, 0, 1, 32, 1);
|
||||
};
|
||||
|
||||
ggml_tensor * learned_pos_embd = resize_position_embeddings();
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
NORM_TYPE_NORMAL,
|
||||
hparams.ffn_op,
|
||||
learned_pos_embd,
|
||||
add_pos);
|
||||
|
||||
cb(cur, "vit_out", -1);
|
||||
|
||||
{
|
||||
// mlp_AR paddleocr projector
|
||||
float proj_norm_eps = 1e-5;
|
||||
cur = build_norm(cur,
|
||||
model.mm_input_norm_w, model.mm_input_norm_b,
|
||||
NORM_TYPE_NORMAL, proj_norm_eps, -1);
|
||||
|
||||
const int scale_factor = model.hparams.n_merge;
|
||||
cur = build_patch_merge_permute(cur, scale_factor);
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
hparams.ffn_op, -1);
|
||||
cb(cur, "mlp_out", -1);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
#include "models.h"
|
||||
|
||||
static constexpr int PARAKEET_LOCAL_ATTN_THRESHOLD = 8192;
|
||||
static constexpr int PARAKEET_LOCAL_ATTN_WINDOW = 128;
|
||||
|
||||
// conv subsampling + conformer encoder
|
||||
ggml_cgraph * clip_graph_parakeet::build() {
|
||||
|
||||
// Conv subsampling
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
ggml_tensor * cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[0], inp, 2, 2, 1, 1, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[0]);
|
||||
cb(cur, "pre_conv_0", -1);
|
||||
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cb(cur, "pre_conv_0_relu", -1);
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[2], cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[2]);
|
||||
cb(cur, "pre_conv_2", -1);
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[3], cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[3]);
|
||||
cb(cur, "pre_conv_3", -1);
|
||||
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cb(cur, "pre_conv_3_relu", -1);
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[5], cur, 2, 2, 1, 1, 1, 1);
|
||||
cb(cur, "pre_conv_5_direct", -1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[5]);
|
||||
cb(cur, "pre_conv_5", -1);
|
||||
|
||||
// [freq, time, channels, batch]
|
||||
cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[6], cur, 1, 1, 0, 0, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[6]);
|
||||
cb(cur, "pre_conv_6", -1);
|
||||
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cb(cur, "pre_conv_6_relu", -1);
|
||||
|
||||
// [freq, time, chan]
|
||||
cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
|
||||
// [freq, chan, time]
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
const int n_freq = cur->ne[0];
|
||||
const int n_chan = cur->ne[1];
|
||||
const int n_frames = cur->ne[2];
|
||||
|
||||
// [freq, time, chan, batch] -> [(freq * chan), time]
|
||||
cur = ggml_reshape_2d(ctx0, cur, n_freq * n_chan, n_frames);
|
||||
|
||||
cur = build_mm(model.pre_encode_out_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.pre_encode_out_b);
|
||||
|
||||
ggml_set_name(cur, "pre_enc_out");
|
||||
|
||||
// Encoder
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_state = hparams.n_embd;
|
||||
const float fc_factor = 0.5f;
|
||||
|
||||
const int n_time = cur->ne[1];
|
||||
const bool local_attn = n_time > PARAKEET_LOCAL_ATTN_THRESHOLD;
|
||||
const int att_left = local_attn ? PARAKEET_LOCAL_ATTN_WINDOW : n_time - 1;
|
||||
const int att_right = local_attn ? PARAKEET_LOCAL_ATTN_WINDOW : n_time - 1;
|
||||
const int window_size = local_attn ? att_left + att_right + 1 : 2 * n_time - 1;
|
||||
const int d_half = n_state / 2;
|
||||
const int mask_dim = local_attn ? window_size : n_time;
|
||||
|
||||
// mask [key, n_time]
|
||||
struct ggml_tensor * attn_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, mask_dim, n_time);
|
||||
ggml_set_name(attn_mask, "attn_mask");
|
||||
ggml_set_input(attn_mask);
|
||||
|
||||
struct ggml_tensor * local_mask = nullptr;
|
||||
if (local_attn) {
|
||||
const int chunk = att_left + att_right;
|
||||
local_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, chunk + window_size - 1, chunk);
|
||||
ggml_set_name(local_mask, "local_mask");
|
||||
ggml_set_input(local_mask);
|
||||
}
|
||||
|
||||
struct ggml_tensor * pos_freqs = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, d_half);
|
||||
ggml_set_name(pos_freqs, "pos_freqs");
|
||||
ggml_set_input(pos_freqs);
|
||||
|
||||
struct ggml_tensor * rel_positions = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, window_size);
|
||||
ggml_set_name(rel_positions, "rel_positions");
|
||||
ggml_set_input(rel_positions);
|
||||
|
||||
struct ggml_tensor * freqs = ggml_repeat_4d(ctx0, pos_freqs, d_half, window_size, 1, 1);
|
||||
struct ggml_tensor * theta = ggml_mul(ctx0, freqs, rel_positions);
|
||||
|
||||
struct ggml_tensor * sin = ggml_reshape_3d(ctx0, ggml_sin(ctx0, theta), 1, d_half, window_size);
|
||||
struct ggml_tensor * cos = ggml_reshape_3d(ctx0, ggml_cos(ctx0, theta), 1, d_half, window_size);
|
||||
struct ggml_tensor * pos_emb = ggml_reshape_2d(ctx0, ggml_cont(ctx0, ggml_concat(ctx0, sin, cos, 0)), n_state, window_size);
|
||||
ggml_set_name(pos_emb, "pos_emb");
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const auto & layer = model.layers[il];
|
||||
// FFN1
|
||||
{
|
||||
struct ggml_tensor * residual = cur;
|
||||
ggml_format_name(cur, "enc_%d_res", il);
|
||||
|
||||
// norm
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ff_norm_w), layer.ff_norm_b);
|
||||
ggml_format_name(cur, "enc_%d_ffn_norm_1", il);
|
||||
|
||||
cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_SILU, il);
|
||||
ggml_format_name(cur, "enc_%d_ffn_1", il);
|
||||
|
||||
cur = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, fc_factor));
|
||||
ggml_format_name(cur, "enc_%d_res_ffn", il);
|
||||
}
|
||||
|
||||
// self attention block using relative positional encoding from model.position_embedding.
|
||||
{
|
||||
// [feat, time_frames, 1, 1]
|
||||
struct ggml_tensor * residual = cur;
|
||||
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ln_1_w), layer.ln_1_b);
|
||||
ggml_format_name(cur, "enc_%d_attn_norm", il);
|
||||
|
||||
const int n_head = hparams.n_head;
|
||||
const int d_head = n_state / n_head;
|
||||
|
||||
// [feat, time_frames, 1, 1]
|
||||
struct ggml_tensor * Q_cur = build_mm(layer.q_w, cur);
|
||||
struct ggml_tensor * K_cur = build_mm(layer.k_w, cur);
|
||||
struct ggml_tensor * V_cur = build_mm(layer.v_w, cur);
|
||||
|
||||
// [d_head, n_heads, n_time, 1]
|
||||
Q_cur = ggml_reshape_3d(ctx0, Q_cur, d_head, n_head, n_time);
|
||||
K_cur = ggml_reshape_3d(ctx0, K_cur, d_head, n_head, n_time);
|
||||
V_cur = ggml_reshape_3d(ctx0, V_cur, d_head, n_head, n_time);
|
||||
|
||||
// [n_state, window_size]
|
||||
struct ggml_tensor * pos = build_mm(layer.linear_pos_w, pos_emb);
|
||||
// [feat, head, window_size, 1]
|
||||
pos = ggml_reshape_3d(ctx0, pos, d_head, n_head, pos_emb->ne[1]);
|
||||
// [feat, window_size, head, 1]
|
||||
pos = ggml_cont(ctx0, ggml_permute(ctx0, pos, 0, 2, 1, 3));
|
||||
ggml_format_name(pos, "enc_%d_attn_pos", il);
|
||||
|
||||
if (local_attn) {
|
||||
const int chunk = att_left + att_right;
|
||||
const int n_group = (n_time + chunk - 1) / chunk;
|
||||
const int n_time_padded = n_group * chunk;
|
||||
const int n_kv_chunk = chunk + window_size - 1;
|
||||
const int n_kv_dense = n_kv_chunk * n_group;
|
||||
const bool need_padding = n_time_padded > n_time;
|
||||
|
||||
Q_cur = ggml_cont(ctx0, ggml_permute(ctx0, Q_cur, 0, 2, 1, 3));
|
||||
K_cur = ggml_cont(ctx0, ggml_permute(ctx0, K_cur, 0, 2, 1, 3));
|
||||
V_cur = ggml_cont(ctx0, ggml_permute(ctx0, V_cur, 0, 2, 1, 3));
|
||||
|
||||
// content bias
|
||||
struct ggml_tensor * bias_u = ggml_reshape_3d(ctx0, layer.pos_bias_u, d_head, 1, n_head);
|
||||
struct ggml_tensor * Q_u = ggml_add(ctx0, Q_cur, bias_u);
|
||||
|
||||
// position bias
|
||||
struct ggml_tensor * bias_v = ggml_reshape_3d(ctx0, layer.pos_bias_v, d_head, 1, n_head);
|
||||
struct ggml_tensor * Q_v = ggml_add(ctx0, Q_cur, bias_v);
|
||||
|
||||
// right pad the time dimension
|
||||
struct ggml_tensor * Q_u_padded = need_padding ?
|
||||
ggml_pad_ext(ctx0, Q_u, 0, 0, 0, n_time_padded - n_time, 0, 0, 0, 0) : Q_u;
|
||||
Q_u_padded = ggml_reshape_4d(ctx0, Q_u_padded, d_head, chunk, n_group, n_head);
|
||||
|
||||
// pad front and back for the first and last time frames
|
||||
struct ggml_tensor * K_padded = ggml_pad_ext(ctx0, K_cur, 0, 0, att_left, att_right, 0, 0, 0, 0);
|
||||
if (n_kv_dense > K_padded->ne[1]) {
|
||||
K_padded = ggml_pad_ext(ctx0, K_padded, 0, 0, 0, n_kv_dense - K_padded->ne[1], 0, 0, 0, 0);
|
||||
}
|
||||
|
||||
// sliding window view: each group spans n_kv_chunk keys but steps by chunk
|
||||
struct ggml_tensor * K_chunk = ggml_view_4d(ctx0, K_padded,
|
||||
d_head, n_kv_chunk, n_group, n_head,
|
||||
K_padded->nb[1],
|
||||
(size_t) chunk * K_padded->nb[1],
|
||||
K_padded->nb[2],
|
||||
0);
|
||||
K_chunk = ggml_cont(ctx0, K_chunk);
|
||||
|
||||
struct ggml_tensor * content_scores = ggml_mul_mat(ctx0, K_chunk, Q_u_padded);
|
||||
|
||||
// trim the dense output down to window_size scores per query
|
||||
content_scores = ggml_view_4d(ctx0, content_scores,
|
||||
window_size, chunk, n_group, n_head,
|
||||
(size_t) (chunk + window_size) * content_scores->nb[0],
|
||||
content_scores->nb[2],
|
||||
content_scores->nb[3],
|
||||
0);
|
||||
content_scores = ggml_cont(ctx0, content_scores);
|
||||
|
||||
// ungroup: [window_size, n_time_padded, n_head]
|
||||
content_scores = ggml_reshape_3d(ctx0, content_scores, window_size, n_time_padded, n_head);
|
||||
if (need_padding) {
|
||||
content_scores = ggml_view_3d(ctx0, content_scores,
|
||||
window_size, n_time, n_head,
|
||||
content_scores->nb[1],
|
||||
content_scores->nb[2],
|
||||
0);
|
||||
}
|
||||
|
||||
// Q_v: [d_head, time, head]
|
||||
Q_v = ggml_cont(ctx0, ggml_permute(ctx0, Q_v, 0, 2, 1, 3));
|
||||
struct ggml_tensor * rel_pos_scores = ggml_mul_mat(ctx0, pos, Q_v);
|
||||
|
||||
struct ggml_tensor * attn_scores = ggml_add(ctx0, content_scores, rel_pos_scores);
|
||||
attn_scores = ggml_soft_max_ext(ctx0, attn_scores, attn_mask, 1.0f / std::sqrt(d_head), 0.0f);
|
||||
ggml_format_name(attn_scores, "enc_%d_attn_probs", il);
|
||||
|
||||
// expand probs back to n_kv_chunk width for the V matmul
|
||||
struct ggml_tensor * probs_padded = need_padding ?
|
||||
ggml_pad_ext(ctx0, attn_scores, 0, 0, 0, n_time_padded - n_time, 0, 0, 0, 0) : attn_scores;
|
||||
|
||||
probs_padded = ggml_reshape_4d(ctx0, probs_padded, window_size, chunk, n_group, n_head);
|
||||
probs_padded = ggml_pad_ext(ctx0, probs_padded, 0, chunk, 0, 0, 0, 0, 0, 0);
|
||||
probs_padded = ggml_view_4d(ctx0, probs_padded,
|
||||
n_kv_chunk, chunk, n_group, n_head,
|
||||
(size_t) n_kv_chunk * probs_padded->nb[0],
|
||||
probs_padded->nb[2],
|
||||
probs_padded->nb[3],
|
||||
0);
|
||||
probs_padded = ggml_cont(ctx0, probs_padded);
|
||||
probs_padded = ggml_mul(ctx0, probs_padded, local_mask);
|
||||
|
||||
struct ggml_tensor * V_padded = ggml_pad_ext(ctx0, V_cur, 0, 0, att_left, att_right, 0, 0, 0, 0);
|
||||
if (n_kv_dense > V_padded->ne[1]) {
|
||||
V_padded = ggml_pad_ext(ctx0, V_padded, 0, 0, 0, n_kv_dense - V_padded->ne[1], 0, 0, 0, 0);
|
||||
}
|
||||
V_padded = ggml_cont(ctx0, ggml_transpose(ctx0, V_padded));
|
||||
|
||||
struct ggml_tensor * V_chunk = ggml_view_4d(ctx0, V_padded,
|
||||
n_kv_chunk, d_head, n_group, n_head,
|
||||
V_padded->nb[1],
|
||||
(size_t) chunk * V_padded->nb[0],
|
||||
V_padded->nb[2],
|
||||
0);
|
||||
V_chunk = ggml_cont(ctx0, V_chunk);
|
||||
|
||||
cur = ggml_mul_mat(ctx0, V_chunk, probs_padded);
|
||||
cur = ggml_reshape_3d(ctx0, cur, d_head, n_time_padded, n_head);
|
||||
if (need_padding) {
|
||||
cur = ggml_view_3d(ctx0, cur, d_head, n_time, n_head, cur->nb[1], cur->nb[2], 0);
|
||||
}
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3));
|
||||
cur = ggml_reshape_2d(ctx0, cur, n_state, n_time);
|
||||
cur = build_mm(layer.o_w, cur);
|
||||
} else {
|
||||
// full attention
|
||||
struct ggml_tensor * Q_u = ggml_add(ctx0, Q_cur, layer.pos_bias_u);
|
||||
ggml_format_name(Q_u, "enc_%d_attn_q_u", il);
|
||||
|
||||
struct ggml_tensor * K_prep = ggml_permute(ctx0, K_cur, 0, 2, 1, 3);
|
||||
struct ggml_tensor * Q_prep = ggml_permute(ctx0, Q_u, 0, 2, 1, 3);
|
||||
struct ggml_tensor * content_scores = ggml_mul_mat(ctx0, K_prep, Q_prep);
|
||||
ggml_format_name(content_scores, "enc_%d_attn_content_scores", il);
|
||||
|
||||
struct ggml_tensor * Q_v = ggml_add(ctx0, Q_cur, layer.pos_bias_v);
|
||||
ggml_format_name(Q_v, "enc_%d_attn_q_v", il);
|
||||
|
||||
Q_v = ggml_permute(ctx0, Q_v, 0, 2, 1, 3);
|
||||
Q_v = ggml_cont(ctx0, Q_v);
|
||||
ggml_format_name(Q_v, "enc_%d_attn_q_v_perm", il);
|
||||
|
||||
struct ggml_tensor * rel_pos_scores = ggml_mul_mat(ctx0, pos, Q_v);
|
||||
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos", il);
|
||||
|
||||
// Relative positional shift
|
||||
{
|
||||
const auto pos_window = rel_pos_scores->ne[0];
|
||||
const auto n_frame = rel_pos_scores->ne[1];
|
||||
const auto n_head = rel_pos_scores->ne[2];
|
||||
|
||||
rel_pos_scores = ggml_pad_ext(ctx0, rel_pos_scores, 1, 0, 0, 0, 0, 0, 0, 0);
|
||||
|
||||
rel_pos_scores = ggml_reshape_3d(ctx0, rel_pos_scores, n_frame, pos_window + 1, n_head);
|
||||
rel_pos_scores = ggml_cont(ctx0, rel_pos_scores);
|
||||
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_reshaped", il);
|
||||
|
||||
int center = pos_window / 2;
|
||||
size_t offset = rel_pos_scores->nb[0] * (center+1);
|
||||
|
||||
rel_pos_scores = ggml_view_3d(ctx0, rel_pos_scores,
|
||||
n_frame, pos_window, n_head,
|
||||
(pos_window) * 4,
|
||||
rel_pos_scores->nb[2],
|
||||
offset);
|
||||
rel_pos_scores = ggml_cont(ctx0, rel_pos_scores);
|
||||
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_shifted", il);
|
||||
|
||||
rel_pos_scores = ggml_view_3d(ctx0, rel_pos_scores,
|
||||
content_scores->ne[0],
|
||||
content_scores->ne[1],
|
||||
rel_pos_scores->ne[2],
|
||||
rel_pos_scores->nb[1],
|
||||
rel_pos_scores->nb[2],
|
||||
0);
|
||||
rel_pos_scores = ggml_cont(ctx0, rel_pos_scores);
|
||||
ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_shifted_view", il);
|
||||
}
|
||||
|
||||
struct ggml_tensor * attn_scores = ggml_add(ctx0, content_scores, rel_pos_scores);
|
||||
ggml_format_name(attn_scores, "enc_%d_attn_scores", il);
|
||||
attn_scores = ggml_scale(ctx0, attn_scores, 1.0f / std::sqrt(d_head));
|
||||
attn_scores = ggml_add(ctx0, attn_scores, attn_mask);
|
||||
ggml_format_name(attn_scores, "enc_%d_attn_scores_scaled", il);
|
||||
|
||||
struct ggml_tensor * probs = ggml_soft_max(ctx0, attn_scores);
|
||||
ggml_format_name(probs, "enc_%d_attn_probs", il);
|
||||
|
||||
V_cur = ggml_cont(ctx0, ggml_permute(ctx0, V_cur, 1, 2, 0, 3));
|
||||
ggml_format_name(V_cur, "enc_%d_attn_v_cur", il);
|
||||
cur = ggml_mul_mat(ctx0, probs, V_cur);
|
||||
ggml_format_name(cur, "enc_%d_attn_inp", il);
|
||||
|
||||
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3);
|
||||
cur = ggml_cont_2d(ctx0, cur, n_state, n_time);
|
||||
cur = build_mm(layer.o_w, cur);
|
||||
}
|
||||
ggml_format_name(cur, "enc_%d_attn_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, residual, cur);
|
||||
ggml_format_name(cur, "enc_%d_attn_res", il);
|
||||
}
|
||||
|
||||
// Convolution
|
||||
{
|
||||
struct ggml_tensor * residual = cur;
|
||||
ggml_format_name(cur, "enc_%d_residual_conv", il);
|
||||
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.norm_conv_w), layer.norm_conv_b);
|
||||
ggml_format_name(cur, "enc_%d_norm_conv", il);
|
||||
|
||||
// pointwise 1d convolution:
|
||||
cur = build_mm(layer.conv_pw1_w, cur);
|
||||
ggml_format_name(cur, "enc_%d_conv_pw1", il);
|
||||
|
||||
{
|
||||
int64_t d = cur->ne[0] / 2;
|
||||
struct ggml_tensor * signal = ggml_view_2d(ctx0, cur, d, cur->ne[1], cur->nb[1], 0);
|
||||
struct ggml_tensor * gate = ggml_view_2d(ctx0, cur, d, cur->ne[1], cur->nb[1], d * cur->nb[0]);
|
||||
|
||||
cur = ggml_mul(ctx0, signal, ggml_sigmoid(ctx0, gate));
|
||||
ggml_format_name(cur, "enc_%d_conv_glu", il);
|
||||
}
|
||||
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
|
||||
// use ggml_ssm_conv for f32 precision
|
||||
const int dw_pad = (hparams.audio_conv_kernel_size - 1) / 2;
|
||||
cur = ggml_pad_ext(ctx0, cur, dw_pad, dw_pad, 0, 0, 0, 0, 0, 0);
|
||||
ggml_format_name(cur, "enc_%d_conv_dw_pad", il);
|
||||
|
||||
cur = ggml_ssm_conv(ctx0, cur, layer.conv_dw_w);
|
||||
ggml_format_name(cur, "enc_%d_conv_1d_dw", il);
|
||||
|
||||
cur = ggml_sub(ctx0, cur, layer.conv_norm_mean);
|
||||
struct ggml_tensor * std = ggml_sqrt(ctx0, layer.conv_norm_var);
|
||||
cur = ggml_div(ctx0, cur, std);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.conv_norm_w), layer.conv_norm_b);
|
||||
ggml_format_name(cur, "enc_%d_conv_bn", il);
|
||||
|
||||
cur = ggml_silu(ctx0, cur);
|
||||
ggml_format_name(cur, "enc_%d_conv_silu", il);
|
||||
|
||||
cur = build_mm(layer.conv_pw2_w, cur);
|
||||
ggml_format_name(cur, "enc_%d_conv_pw2", il);
|
||||
|
||||
cur = ggml_add(ctx0, residual, cur);
|
||||
ggml_format_name(cur, "enc_%d_conv_res", il);
|
||||
}
|
||||
|
||||
// FFN2
|
||||
{
|
||||
struct ggml_tensor * residual = cur;
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ff_norm_1_w), layer.ff_norm_1_b);
|
||||
ggml_format_name(cur, "enc_%d_ffn_norm_2", il);
|
||||
|
||||
cur = build_ffn(cur, layer.ff_up_1_w, nullptr, nullptr, nullptr, layer.ff_down_1_w, nullptr, FFN_SILU, il);
|
||||
cur = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, 0.5));
|
||||
ggml_format_name(cur, "enc_%d_ffn_res", il);
|
||||
}
|
||||
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ln_2_w), layer.ln_2_b);
|
||||
}
|
||||
|
||||
cb(cur, "encoder_out", -1);
|
||||
|
||||
cur = ggml_rms_norm(ctx0, cur, 1e-6);
|
||||
cur = ggml_mul(ctx0, cur, model.mm_norm_pre_w);
|
||||
cb(cur, "sound_projection.norm", -1);
|
||||
|
||||
cur = build_ffn(cur, model.mm_0_w, model.mm_0_b, nullptr, nullptr, model.mm_1_w, model.mm_1_b, FFN_RELU_SQR, -1);
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_pixtral::build() {
|
||||
const int n_merge = hparams.n_merge;
|
||||
|
||||
// 2D input positions
|
||||
ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_h, "pos_h");
|
||||
ggml_set_input(pos_h);
|
||||
|
||||
ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_w, "pos_w");
|
||||
ggml_set_input(pos_w);
|
||||
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return build_rope_2d(ctx0, cur, pos_h, pos_w, hparams.rope_theta, true);
|
||||
};
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
NORM_TYPE_RMS,
|
||||
hparams.ffn_op,
|
||||
nullptr, // no learned pos embd
|
||||
add_pos);
|
||||
|
||||
// mistral small 3.1 patch merger
|
||||
// ref: https://github.com/huggingface/transformers/blob/7a3e208892c06a5e278144eaf38c8599a42f53e7/src/transformers/models/mistral3/modeling_mistral3.py#L67
|
||||
if (model.mm_patch_merger_w) {
|
||||
GGML_ASSERT(hparams.n_merge > 0);
|
||||
|
||||
cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, eps), model.mm_input_norm_w);
|
||||
|
||||
// reshape image tokens to 2D grid
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_embd, n_patches_x, n_patches_y);
|
||||
cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); // [x, y, n_embd]
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
// torch.nn.functional.unfold is just an im2col under the hood
|
||||
// we just need a dummy kernel to make it work
|
||||
ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2], 0, 0, 0);
|
||||
cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type);
|
||||
|
||||
// project to n_embd
|
||||
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
|
||||
cur = build_mm(model.mm_patch_merger_w, cur);
|
||||
}
|
||||
|
||||
// LlavaMultiModalProjector (always using GELU activation)
|
||||
{
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU,
|
||||
-1);
|
||||
}
|
||||
|
||||
// arrangement of the [IMG_BREAK] token
|
||||
if (model.token_embd_img_break) {
|
||||
// not efficient, but works
|
||||
// the trick is to view the embeddings as a 3D tensor with shape [n_embd, n_patches_per_row, n_rows]
|
||||
// and then concatenate the [IMG_BREAK] token to the end of each row, aka n_patches_per_row dimension
|
||||
// after the concatenation, we have a tensor with shape [n_embd, n_patches_per_row + 1, n_rows]
|
||||
|
||||
const int p_y = n_patches_y / n_merge;
|
||||
const int p_x = n_patches_x / n_merge;
|
||||
const int p_total = p_x * p_y;
|
||||
const int n_embd_text = cur->ne[0];
|
||||
const int n_tokens_output = p_total + p_y - 1; // one [IMG_BREAK] per row, except the last row
|
||||
|
||||
ggml_tensor * tmp = ggml_reshape_3d(ctx0, cur, n_embd_text, p_x, p_y);
|
||||
ggml_tensor * tok = ggml_new_tensor_3d(ctx0, tmp->type, n_embd_text, 1, p_y);
|
||||
tok = ggml_scale(ctx0, tok, 0.0); // clear the tensor
|
||||
tok = ggml_add(ctx0, tok, model.token_embd_img_break);
|
||||
tmp = ggml_concat(ctx0, tmp, tok, 1);
|
||||
cur = ggml_view_2d(ctx0, tmp,
|
||||
n_embd_text, n_tokens_output,
|
||||
ggml_row_size(tmp->type, n_embd_text), 0);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <cmath>
|
||||
|
||||
// pocket-tts generation stages
|
||||
//
|
||||
// GEN_CODE: backbone hidden state -> next 32-d latent (flow matching) + end-of-speech score
|
||||
// GEN_WAV : a window of latents -> PCM, through the mimi decoder
|
||||
//
|
||||
// there is no codebook anywhere, "codes" in the mtmd API are continuous features here
|
||||
|
||||
ggml_tensor * clip_graph_pockettts_gen::modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const {
|
||||
ggml_tensor * cur = ggml_mul(ctx0, x, ggml_scale_bias(ctx0, scale, 1.0f, 1.0f));
|
||||
return ggml_add(ctx0, cur, shift);
|
||||
}
|
||||
|
||||
// see TimestepEmbedder in the reference
|
||||
ggml_tensor * clip_graph_pockettts_gen::time_embed(const clip_flow_net::time_embd & te, float t) const {
|
||||
// t is a graph-build constant, so the cos/sin table can be folded into a scaled copy
|
||||
ggml_tensor * args = ggml_scale(ctx0, te.freqs, t);
|
||||
ggml_tensor * emb = ggml_concat(ctx0, ggml_cos(ctx0, args), ggml_sin(ctx0, args), 0);
|
||||
|
||||
ggml_tensor * cur = build_mm(te.up_w, emb);
|
||||
cur = ggml_add(ctx0, cur, te.up_b);
|
||||
cur = ggml_silu(ctx0, cur);
|
||||
cur = build_mm(te.down_w, cur);
|
||||
cur = ggml_add(ctx0, cur, te.down_b);
|
||||
|
||||
// this "RMSNorm" divides by the unbiased variance, not the mean square
|
||||
// it also rescales the input, not the centered value, see _rms_norm() in mlp.py
|
||||
{
|
||||
const int64_t n = cur->ne[0];
|
||||
ggml_tensor * mean = ggml_mean(ctx0, cur);
|
||||
ggml_tensor * dev = ggml_sub(ctx0, cur, mean);
|
||||
ggml_tensor * var = ggml_mean(ctx0, ggml_sqr(ctx0, dev));
|
||||
var = ggml_scale_bias(ctx0, var, (float) n / (float) (n - 1), 1e-5f);
|
||||
cur = ggml_div(ctx0, cur, ggml_sqrt(ctx0, var));
|
||||
cur = ggml_mul(ctx0, cur, te.norm);
|
||||
}
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
// one velocity evaluation: v(cond, s, t, x)
|
||||
ggml_tensor * clip_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const {
|
||||
const auto & flow = model.flow;
|
||||
|
||||
ggml_tensor * cur = build_mm(flow.input_proj_w, x);
|
||||
cur = ggml_add(ctx0, cur, flow.input_proj_b);
|
||||
|
||||
// the two time conditions are averaged, then added to the projected backbone state
|
||||
ggml_tensor * ts = ggml_add(ctx0, time_embed(flow.time[0], s), time_embed(flow.time[1], t));
|
||||
ts = ggml_scale(ctx0, ts, 1.0f / (float) flow.time.size());
|
||||
|
||||
ggml_tensor * c = build_mm(flow.cond_embd_w, cond);
|
||||
c = ggml_add(ctx0, c, flow.cond_embd_b);
|
||||
|
||||
ggml_tensor * y = ggml_add(ctx0, ts, c);
|
||||
cb(y, "flow_cond", -1);
|
||||
|
||||
const int64_t n_ch = flow.blocks.empty() ? 0 : flow.blocks[0].norm_w->ne[0];
|
||||
|
||||
for (size_t il = 0; il < flow.blocks.size(); il++) {
|
||||
const auto & blk = flow.blocks[il];
|
||||
|
||||
ggml_tensor * mod = build_mm(blk.ada_w, ggml_silu(ctx0, y));
|
||||
mod = ggml_add(ctx0, mod, blk.ada_b);
|
||||
|
||||
ggml_tensor * shift = ggml_view_1d(ctx0, mod, n_ch, 0);
|
||||
ggml_tensor * scale = ggml_view_1d(ctx0, mod, n_ch, (size_t) n_ch * mod->nb[0]);
|
||||
ggml_tensor * gate = ggml_view_1d(ctx0, mod, n_ch, (size_t) 2 * n_ch * mod->nb[0]);
|
||||
|
||||
ggml_tensor * h = build_norm(cur, blk.norm_w, blk.norm_b, NORM_TYPE_NORMAL, 1e-6f, (int) il);
|
||||
h = modulate(h, shift, scale);
|
||||
h = build_mm(blk.up_w, h);
|
||||
h = ggml_add(ctx0, h, blk.up_b);
|
||||
h = ggml_silu(ctx0, h);
|
||||
h = build_mm(blk.down_w, h);
|
||||
h = ggml_add(ctx0, h, blk.down_b);
|
||||
|
||||
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, gate, h));
|
||||
cb(cur, "flow_blk", (int) il);
|
||||
}
|
||||
|
||||
// final layer: the norm has no weights, only the AdaLN modulation
|
||||
ggml_tensor * mod = build_mm(flow.final_ada_w, ggml_silu(ctx0, y));
|
||||
mod = ggml_add(ctx0, mod, flow.final_ada_b);
|
||||
|
||||
ggml_tensor * shift = ggml_view_1d(ctx0, mod, n_ch, 0);
|
||||
ggml_tensor * scale = ggml_view_1d(ctx0, mod, n_ch, (size_t) n_ch * mod->nb[0]);
|
||||
|
||||
cur = build_norm(cur, nullptr, nullptr, NORM_TYPE_NORMAL, 1e-6f, -1);
|
||||
cur = modulate(cur, shift, scale);
|
||||
cur = build_mm(flow.final_proj_w, cur);
|
||||
cur = ggml_add(ctx0, cur, flow.final_proj_b);
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
// state carried between GEN_WAV calls: rope offset, per-layer KV window, conv left context
|
||||
// and the transposed-conv overlap tails
|
||||
std::vector<c2w_state_slot> list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model) {
|
||||
std::vector<c2w_state_slot> slots;
|
||||
if (model.gen_upsample_w == nullptr) {
|
||||
return slots; // not a pocket-tts decoder
|
||||
}
|
||||
const auto & seanet = model.seanet;
|
||||
|
||||
// the slots below are sized from these
|
||||
GGML_ASSERT(!model.gen_tfm_layers.empty());
|
||||
GGML_ASSERT((int) seanet.stages.size() >= hparams.seanet_n_stage);
|
||||
GGML_ASSERT((int) hparams.seanet_ratios.size() >= hparams.seanet_n_stage);
|
||||
GGML_ASSERT(hparams.mimi_tfm_context > 1 && hparams.mimi_downsample > 0);
|
||||
|
||||
slots.push_back({"tfm_pos", 1, 1});
|
||||
|
||||
const int64_t n_embd_a = model.gen_tfm_layers[0].q_w->ne[1];
|
||||
const int64_t prefix = hparams.mimi_tfm_context - 1;
|
||||
for (size_t il = 0; il < model.gen_tfm_layers.size(); il++) {
|
||||
slots.push_back({"tfm_k_" + std::to_string(il), n_embd_a, prefix});
|
||||
slots.push_back({"tfm_v_" + std::to_string(il), n_embd_a, prefix});
|
||||
}
|
||||
|
||||
// upsample is depthwise, its output channel count is the input one
|
||||
slots.push_back({"up", model.gen_upsample_w->ne[0] - hparams.mimi_downsample, model.gen_upsample_w->ne[2]});
|
||||
|
||||
slots.push_back({"dec_in", seanet.conv_in_w->ne[0] - 1, seanet.conv_in_w->ne[1]});
|
||||
for (int i = 0; i < hparams.seanet_n_stage; i++) {
|
||||
const auto & stage = seanet.stages[i];
|
||||
const int stride = hparams.seanet_ratios[hparams.seanet_n_stage - 1 - i];
|
||||
slots.push_back({"dec_up_" + std::to_string(i), stage.scale_conv_w->ne[0] - stride, stage.scale_conv_w->ne[1]});
|
||||
slots.push_back({"dec_res_" + std::to_string(i), stage.res_conv1_w->ne[0] - 1, stage.res_conv1_w->ne[1]});
|
||||
}
|
||||
slots.push_back({"dec_out", seanet.conv_out_w->ne[0] - 1, seanet.conv_out_w->ne[1]});
|
||||
|
||||
return slots;
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_pockettts_gen::build() {
|
||||
if (gen_process == CLIP_GEN_PROCESS_GEN_CODE) {
|
||||
// the backbone hidden state arrives as the single batch entry
|
||||
ggml_tensor * h_state = build_inp_raw(1);
|
||||
h_state = ggml_reshape_2d(ctx0, h_state, n_mmproj_embd, 1);
|
||||
|
||||
// end-of-speech probe, thresholded on the host side
|
||||
ggml_tensor * eos = build_mm(model.gen_out_eos_w, h_state);
|
||||
eos = ggml_add(ctx0, eos, model.gen_out_eos_b);
|
||||
ggml_set_name(eos, "out_eos_score");
|
||||
ggml_set_output(eos);
|
||||
ggml_build_forward_expand(gf, eos);
|
||||
|
||||
const int64_t n_latent = model.gen_input_lin_w->ne[0];
|
||||
|
||||
ggml_tensor * noise = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, 1);
|
||||
ggml_set_name(noise, "inp_noise");
|
||||
ggml_set_input(noise);
|
||||
|
||||
// lsd_decode: integrate the velocity field from the noise sample
|
||||
ggml_tensor * cur = noise;
|
||||
for (int i = 0; i < n_step; i++) {
|
||||
const float s = (float) i / (float) n_step;
|
||||
const float t = (float) (i + 1) / (float) n_step;
|
||||
ggml_tensor * v = flow_forward(h_state, cur, s, t);
|
||||
cur = ggml_add(ctx0, cur, ggml_scale(ctx0, v, 1.0f / (float) n_step));
|
||||
}
|
||||
cb(cur, "flow_latent", -1);
|
||||
|
||||
ggml_set_name(cur, "out_feats");
|
||||
ggml_set_output(cur);
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
// the same latent, projected into the backbone's input space for the next step
|
||||
ggml_tensor * embd = build_mm(model.gen_input_lin_w, cur);
|
||||
cb(embd, "gen_embd", -1);
|
||||
ggml_build_forward_expand(gf, embd);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
// GEN_WAV: [32, n_frames] latents -> PCM
|
||||
ggml_tensor * feats = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32,
|
||||
model.gen_input_lin_w->ne[0], n_frames);
|
||||
ggml_set_name(feats, "inp_feats");
|
||||
ggml_set_input(feats);
|
||||
|
||||
// denormalize, then the DummyQuantizer up-projection
|
||||
ggml_tensor * cur = ggml_add(ctx0, ggml_mul(ctx0, feats, model.gen_emb_std), model.gen_emb_mean);
|
||||
cur = build_mm(model.gen_quant_out_w, cur);
|
||||
cb(cur, "quant_out", -1);
|
||||
|
||||
clip_graph_pockettts_seanet seanet(*this);
|
||||
for (const auto & slot : list_pockettts_state_slots(hparams, model)) {
|
||||
ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
|
||||
ggml_set_name(t, ("state_in_" + slot.name).c_str());
|
||||
ggml_set_input(t);
|
||||
seanet.state_in[slot.name] = t;
|
||||
}
|
||||
|
||||
// model frame rate -> encoder frame rate, depthwise transposed conv
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = seanet.conv_transpose1d(cur, model.gen_upsample_w, nullptr, hparams.mimi_downsample, "up");
|
||||
cb(cur, "mimi_upsample", -1);
|
||||
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
|
||||
// positions continue across calls, the counter lives in the state
|
||||
const int64_t n_pos = cur->ne[1];
|
||||
const int64_t prefix = hparams.mimi_tfm_context - 1;
|
||||
const int64_t n_kv = prefix + n_pos;
|
||||
|
||||
ggml_tensor * base = ggml_reshape_1d(ctx0, seanet.state_in.at("tfm_pos"), 1);
|
||||
ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_add(ctx0, ggml_arange(ctx0, 0.0f, (float) n_pos, 1.0f), base),
|
||||
GGML_TYPE_I32);
|
||||
seanet.state_out.push_back({"tfm_pos", ggml_scale_bias(ctx0, seanet.state_in.at("tfm_pos"), 1.0f, (float) n_pos)});
|
||||
|
||||
// banded causal mask over [cached prefix | this chunk]
|
||||
// the last factor masks out cache rows that hold no real frame yet
|
||||
ggml_tensor * pos_k = ggml_reshape_2d(ctx0, ggml_arange(ctx0, 0.0f, (float) n_kv, 1.0f), n_kv, 1);
|
||||
ggml_tensor * pos_q = ggml_reshape_2d(ctx0, ggml_arange(ctx0, (float) prefix, (float) (prefix + n_pos), 1.0f), 1, n_pos);
|
||||
ggml_tensor * diff = ggml_sub(ctx0, ggml_repeat_4d(ctx0, pos_q, n_kv, n_pos, 1, 1), pos_k);
|
||||
|
||||
ggml_tensor * keep = ggml_mul(ctx0,
|
||||
ggml_step(ctx0, ggml_scale_bias(ctx0, diff, 1.0f, 0.5f)), // delta >= 0
|
||||
ggml_step(ctx0, ggml_scale_bias(ctx0, diff, -1.0f, (float) hparams.mimi_tfm_context - 0.5f))); // delta < context
|
||||
keep = ggml_mul(ctx0, keep,
|
||||
ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix)));
|
||||
ggml_tensor * kq_mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, n_pos, 1, 1);
|
||||
if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
|
||||
kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
|
||||
}
|
||||
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
const auto & layer = model.gen_tfm_layers[il];
|
||||
ggml_tensor * inp = cur;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
|
||||
|
||||
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
|
||||
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
|
||||
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
|
||||
// prepend the cached window, then keep this chunk's tail for the next call
|
||||
const std::string k_name = "tfm_k_" + std::to_string(il);
|
||||
const std::string v_name = "tfm_v_" + std::to_string(il);
|
||||
ggml_tensor * k_full = ggml_concat(ctx0, seanet.state_in.at(k_name),
|
||||
ggml_reshape_2d(ctx0, Kcur, d_head * n_head, n_pos), 1);
|
||||
ggml_tensor * v_full = ggml_concat(ctx0, seanet.state_in.at(v_name), Vcur, 1);
|
||||
seanet.state_out.push_back({k_name, ggml_cont(ctx0, ggml_view_2d(ctx0, k_full, k_full->ne[0], prefix,
|
||||
k_full->nb[1], (size_t) n_pos * k_full->nb[1]))});
|
||||
seanet.state_out.push_back({v_name, ggml_cont(ctx0, ggml_view_2d(ctx0, v_full, v_full->ne[0], prefix,
|
||||
v_full->nb[1], (size_t) n_pos * v_full->nb[1]))});
|
||||
|
||||
ggml_tensor * q_cur = ggml_reshape_4d(ctx0, Qcur, d_head, n_head, n_pos, 1);
|
||||
ggml_tensor * k_cur = ggml_reshape_4d(ctx0, k_full, d_head, n_head, n_kv, 1);
|
||||
ggml_tensor * v_cur = ggml_reshape_4d(ctx0, v_full, d_head, n_head, n_kv, 1);
|
||||
|
||||
cur = build_attn(layer.o_w, nullptr, q_cur, k_cur, v_cur, kq_mask, kq_scale, il);
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_1_w);
|
||||
cur = ggml_add(ctx0, cur, inp);
|
||||
|
||||
inp = cur;
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_GELU, il);
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_2_w);
|
||||
cur = ggml_add(ctx0, cur, inp);
|
||||
}
|
||||
cb(cur, "mimi_dec_tfm", -1);
|
||||
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = seanet.decode(cur);
|
||||
|
||||
for (const auto & s : seanet.state_out) {
|
||||
ggml_set_name(s.second, ("state_out_" + s.first).c_str());
|
||||
ggml_set_output(s.second);
|
||||
ggml_build_forward_expand(gf, s.second);
|
||||
}
|
||||
|
||||
// [n_samples, 1] -> [n_samples], clamped like the reference output
|
||||
cur = ggml_reshape_1d(ctx0, cur, cur->ne[0]);
|
||||
cur = ggml_clamp(ctx0, cur, -1.0f, 1.0f);
|
||||
ggml_set_name(cur, "out_audio");
|
||||
ggml_set_output(cur);
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
#include "models.h"
|
||||
|
||||
// SEANet convolution stack of the mimi codec, see pocket_tts/modules/seanet.py
|
||||
//
|
||||
// tensors are T-first here: [T, C]
|
||||
// the convs are causal: left context comes from a state slot, or from padding on a cold start
|
||||
|
||||
static int64_t div_ceil(int64_t a, int64_t b) {
|
||||
return a / b + (a % b ? 1 : 0);
|
||||
}
|
||||
|
||||
// x: [T, IC], w: [K, IC, OC] -> [T / stride, OC]
|
||||
// the convs are causal, so the whole K - stride padding goes on the left
|
||||
ggml_tensor * clip_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation,
|
||||
bool pad_replicate, const std::string & state_name) const {
|
||||
const int64_t k_size = (w->ne[0] - 1) * dilation + 1;
|
||||
const int64_t p_total = k_size - stride;
|
||||
|
||||
// trailing padding so the last frame is not dropped, see pad_for_conv1d() in conv.py
|
||||
const int64_t n_frames = div_ceil(x->ne[0] - k_size + p_total, stride);
|
||||
const int64_t ideal_len = n_frames * stride + k_size - p_total;
|
||||
const int64_t p_extra = ideal_len - x->ne[0];
|
||||
|
||||
if (!state_name.empty() && p_total > 0) {
|
||||
// streaming: the left context is the tail of the previous call
|
||||
ggml_tensor * left = state_in.at(state_name); // [p_total, IC]
|
||||
x = ggml_concat(ctx0, left, x, 0);
|
||||
state_out.push_back({state_name,
|
||||
ggml_cont(ctx0, ggml_view_2d(ctx0, x, p_total, x->ne[1], x->nb[1],
|
||||
(size_t) (x->ne[0] - p_total) * x->nb[0]))});
|
||||
} else if (pad_replicate && p_total > 0) {
|
||||
// the resamplers repeat the first frame instead of zero-padding
|
||||
ggml_tensor * first = ggml_view_2d(ctx0, x, 1, x->ne[1], x->nb[1], 0);
|
||||
ggml_tensor * left = ggml_repeat_4d(ctx0, first, p_total, x->ne[1], 1, 1);
|
||||
x = ggml_concat(ctx0, left, x, 0);
|
||||
x = ggml_pad_ext(ctx0, x, 0, p_extra, 0, 0, 0, 0, 0, 0);
|
||||
} else {
|
||||
x = ggml_pad_ext(ctx0, x, p_total, p_extra, 0, 0, 0, 0, 0, 0);
|
||||
}
|
||||
|
||||
ggml_tensor * y = ggml_conv_1d(ctx0, w, x, stride, 0, dilation);
|
||||
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
|
||||
if (b) {
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
return y;
|
||||
}
|
||||
|
||||
// x: [T, IC], w: [K, OC/groups, IC] -> [T * stride, OC]
|
||||
// the K - stride overlap tail belongs to the next call: added to its head when streaming, else dropped
|
||||
ggml_tensor * clip_graph_pockettts_seanet::conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride,
|
||||
const std::string & state_name) const {
|
||||
const int64_t K = w->ne[0];
|
||||
const int64_t T = x->ne[0];
|
||||
const int64_t p_total = K - stride;
|
||||
const bool depthwise = w->ne[1] == 1 && w->ne[2] > 1;
|
||||
const int64_t OC = depthwise ? w->ne[2] : w->ne[1];
|
||||
const int64_t emit_len = T * stride;
|
||||
|
||||
// one column per input step, holding the [K, OC] window that col2im scatter-adds at t * stride
|
||||
ggml_tensor * col;
|
||||
if (depthwise) {
|
||||
// one group per channel: a batched matmul over the channels scales the kernel by each step
|
||||
ggml_tensor * krn = ggml_reshape_3d(ctx0, w, 1, K, OC); // [1, K, OC]
|
||||
ggml_tensor * xs = ggml_reshape_3d(ctx0, x, 1, T, OC); // [1, T, OC]
|
||||
col = ggml_mul_mat(ctx0, krn, xs); // [K, T, OC]
|
||||
col = ggml_cont(ctx0, ggml_permute(ctx0, col, 0, 2, 1, 3)); // [K, OC, T]
|
||||
col = ggml_reshape_2d(ctx0, col, K * OC, T);
|
||||
} else {
|
||||
ggml_tensor * w2 = ggml_reshape_2d(ctx0, w, K * OC, w->ne[2]);
|
||||
w2 = ggml_cont(ctx0, ggml_transpose(ctx0, w2)); // [IC, K * OC]
|
||||
ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [IC, T]
|
||||
col = ggml_mul_mat(ctx0, w2, xt);
|
||||
}
|
||||
ggml_tensor * full = ggml_col2im_1d(ctx0, col, stride, OC, 0); // [emit_len + p_total, OC]
|
||||
|
||||
ggml_tensor * out;
|
||||
if (state_name.empty() || p_total == 0) {
|
||||
out = ggml_cont(ctx0, ggml_view_2d(ctx0, full, emit_len, full->ne[1], full->nb[1], 0));
|
||||
} else {
|
||||
// overlap-add the tail the previous call held back
|
||||
ggml_tensor * prev = state_in.at(state_name); // [p_total, OC]
|
||||
ggml_tensor * head = ggml_add(ctx0, ggml_view_2d(ctx0, full, p_total, full->ne[1], full->nb[1], 0), prev);
|
||||
if (emit_len > p_total) {
|
||||
ggml_tensor * rest = ggml_view_2d(ctx0, full, emit_len - p_total, full->ne[1], full->nb[1],
|
||||
(size_t) p_total * full->nb[0]);
|
||||
out = ggml_concat(ctx0, head, rest, 0);
|
||||
} else {
|
||||
out = head;
|
||||
}
|
||||
state_out.push_back({state_name,
|
||||
ggml_cont(ctx0, ggml_view_2d(ctx0, full, p_total, full->ne[1], full->nb[1],
|
||||
(size_t) emit_len * full->nb[0]))});
|
||||
}
|
||||
|
||||
if (b) {
|
||||
out = ggml_add(ctx0, out, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_pockettts_seanet::res_unit(ggml_tensor * x, const clip_seanet::stage & stage, int dilation,
|
||||
const std::string & state_prefix) const {
|
||||
ggml_tensor * h = ggml_elu(ctx0, x);
|
||||
h = conv1d(h, stage.res_conv1_w, stage.res_conv1_b, 1, dilation, false, state_prefix);
|
||||
h = ggml_elu(ctx0, h);
|
||||
// the second conv is pointwise, it needs no left context
|
||||
h = conv1d(h, stage.res_conv2_w, stage.res_conv2_b, 1, 1);
|
||||
return ggml_add(ctx0, x, h);
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_pockettts_seanet::encode(ggml_tensor * x) const {
|
||||
const auto & seanet = model.seanet;
|
||||
|
||||
ggml_tensor * cur = conv1d(x, seanet.conv_in_w, seanet.conv_in_b, 1, 1);
|
||||
cb(cur, "seanet_enc_in", -1);
|
||||
|
||||
for (int i = 0; i < hparams.seanet_n_stage; i++) {
|
||||
const auto & stage = seanet.stages[i];
|
||||
const int stride = hparams.seanet_ratios[i];
|
||||
|
||||
cur = res_unit(cur, stage, 1);
|
||||
cur = ggml_elu(ctx0, cur);
|
||||
cur = conv1d(cur, stage.scale_conv_w, stage.scale_conv_b, stride, 1);
|
||||
cb(cur, "seanet_enc_stage", i);
|
||||
}
|
||||
|
||||
cur = ggml_elu(ctx0, cur);
|
||||
cur = conv1d(cur, seanet.conv_out_w, seanet.conv_out_b, 1, 1);
|
||||
cb(cur, "seanet_enc_out", -1);
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_pockettts_seanet::decode(ggml_tensor * x) const {
|
||||
const auto & seanet = model.seanet;
|
||||
const bool stream = !state_in.empty();
|
||||
|
||||
ggml_tensor * cur = conv1d(x, seanet.conv_in_w, seanet.conv_in_b, 1, 1, false,
|
||||
stream ? "dec_in" : "");
|
||||
cb(cur, "seanet_dec_in", -1);
|
||||
|
||||
for (int i = 0; i < hparams.seanet_n_stage; i++) {
|
||||
const auto & stage = seanet.stages[i];
|
||||
// the decoder mirrors the encoder, so the ratios are walked backwards
|
||||
const int stride = hparams.seanet_ratios[hparams.seanet_n_stage - 1 - i];
|
||||
const std::string id = std::to_string(i);
|
||||
|
||||
cur = ggml_elu(ctx0, cur);
|
||||
cur = conv_transpose1d(cur, stage.scale_conv_w, stage.scale_conv_b, stride,
|
||||
stream ? "dec_up_" + id : "");
|
||||
cur = res_unit(cur, stage, 1, stream ? "dec_res_" + id : "");
|
||||
cb(cur, "seanet_dec_stage", i);
|
||||
}
|
||||
|
||||
cur = ggml_elu(ctx0, cur);
|
||||
cur = conv1d(cur, seanet.conv_out_w, seanet.conv_out_b, 1, 1, false,
|
||||
stream ? "dec_out" : "");
|
||||
cb(cur, "seanet_dec_out", -1);
|
||||
|
||||
return cur;
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
#include "models.h"
|
||||
|
||||
// voice-prompt encoder: raw 24kHz waveform -> one conditioning row per 12.5Hz frame
|
||||
// mimi encoder (SEANet + transformer + downsample), then flow_lm.speaker_proj_weight
|
||||
|
||||
// pre-norm block with layer scale on both residual paths, see mimi_transformer.py
|
||||
ggml_tensor * clip_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const {
|
||||
ggml_tensor * inp = cur;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
|
||||
const int64_t n_pos = cur->ne[1];
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
|
||||
|
||||
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
|
||||
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
|
||||
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
|
||||
cur = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, kq_mask, kq_scale, il);
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_1_w);
|
||||
cur = ggml_add(ctx0, cur, inp);
|
||||
|
||||
inp = cur;
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_GELU, il);
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_2_w);
|
||||
cur = ggml_add(ctx0, cur, inp);
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_pockettts_spkenc::build() {
|
||||
// the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1]
|
||||
ggml_tensor * inp_raw = build_inp_raw(1);
|
||||
ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
|
||||
|
||||
clip_graph_pockettts_seanet seanet(*this);
|
||||
cur = seanet.encode(cur);
|
||||
cb(cur, "mimi_enc", -1);
|
||||
|
||||
// [T, 512] -> transformer works on [512, T]
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
|
||||
ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, cur->ne[1]);
|
||||
ggml_set_name(inp_pos, "inp_pos");
|
||||
ggml_set_input(inp_pos);
|
||||
|
||||
// the mimi transformer is causal with a sliding window, see _build_attention_mask()
|
||||
ggml_tensor * kq_mask = build_inp_attn_mask(cur->ne[1], cur->ne[1]);
|
||||
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
cur = tfm_layer_forward(cur, model.layers[il], inp_pos, kq_mask, il);
|
||||
}
|
||||
cb(cur, "mimi_enc_tfm", -1);
|
||||
|
||||
// downsample to the model frame rate, [512, T] -> [T, 512] -> [T / 16, 32]
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = seanet.conv1d(cur, model.downsample_w, nullptr, hparams.mimi_downsample, 1, true);
|
||||
cb(cur, "mimi_downsample", -1);
|
||||
|
||||
// voice latent -> backbone embd
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = build_mm(model.spk_proj_w, cur);
|
||||
cb(cur, "spk_proj", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_qwen2vl::build() {
|
||||
GGML_ASSERT(model.patch_bias == nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
|
||||
const int batch_size = 1;
|
||||
const bool use_window_attn = hparams.n_wa_pattern > 0;
|
||||
const int n_wa_pattern = hparams.n_wa_pattern;
|
||||
const int n_pos = n_patches;
|
||||
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
|
||||
|
||||
norm_type norm_t = proj_type == PROJECTOR_TYPE_QWEN25VL
|
||||
? NORM_TYPE_RMS // qwen 2.5 vl
|
||||
: NORM_TYPE_NORMAL; // qwen 2 vl
|
||||
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
|
||||
ggml_tensor * inp = build_inp_temporal_merge();
|
||||
|
||||
// second conv dimension
|
||||
{
|
||||
inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b]
|
||||
inp = ggml_cont_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
inp = ggml_reshape_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
|
||||
inp = ggml_cont_3d(
|
||||
ctx0, inp,
|
||||
n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
ggml_tensor * window_mask = nullptr;
|
||||
ggml_tensor * window_idx = nullptr;
|
||||
ggml_tensor * inv_window_idx = nullptr;
|
||||
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
// pre-layernorm
|
||||
if (model.pre_ln_w) {
|
||||
inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1);
|
||||
}
|
||||
|
||||
if (use_window_attn) {
|
||||
// handle window attention inputs
|
||||
inv_window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);
|
||||
ggml_set_name(inv_window_idx, "inv_window_idx");
|
||||
ggml_set_input(inv_window_idx);
|
||||
// mask for window attention
|
||||
window_mask = build_inp_attn_mask(n_pos, n_pos);
|
||||
|
||||
// inpL shape: [n_embd, n_patches_x * n_patches_y, batch_size]
|
||||
GGML_ASSERT(batch_size == 1);
|
||||
inpL = ggml_reshape_2d(ctx0, inpL, n_embd * 4, n_patches_x * n_patches_y * batch_size / 4);
|
||||
inpL = ggml_get_rows(ctx0, inpL, inv_window_idx);
|
||||
inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
|
||||
// loop over layers
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
const auto & layer = model.layers[il];
|
||||
const bool full_attn = use_window_attn ? (il + 1) % n_wa_pattern == 0 : true;
|
||||
|
||||
ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
|
||||
|
||||
// layernorm1
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
|
||||
cb(cur, "ln1", il);
|
||||
|
||||
// self-attention
|
||||
{
|
||||
ggml_tensor * Qcur = ggml_add(ctx0,
|
||||
build_mm(layer.q_w, cur), layer.q_b);
|
||||
ggml_tensor * Kcur = ggml_add(ctx0,
|
||||
build_mm(layer.k_w, cur), layer.k_b);
|
||||
ggml_tensor * Vcur = ggml_add(ctx0,
|
||||
build_mm(layer.v_w, cur), layer.v_b);
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_patches);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_patches);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_patches);
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
// apply M-RoPE
|
||||
Qcur = ggml_rope_multi(
|
||||
ctx0, Qcur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
|
||||
Kcur = ggml_rope_multi(
|
||||
ctx0, Kcur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
|
||||
|
||||
cb(Qcur, "Qcur_rope", il);
|
||||
cb(Kcur, "Kcur_rope", il);
|
||||
|
||||
ggml_tensor * attn_mask = full_attn ? nullptr : window_mask;
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b,
|
||||
Qcur, Kcur, Vcur, attn_mask, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
// re-add the layer input, e.g., residual
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
|
||||
inpL = cur; // inpL = residual, cur = hidden_states
|
||||
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
// layernorm2
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
|
||||
cb(cur, "ffn_inp_normed", il);
|
||||
|
||||
// ffn
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
// residual 2
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
// post-layernorm
|
||||
if (model.post_ln_w) {
|
||||
inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, n_layer);
|
||||
}
|
||||
|
||||
// multimodal projection
|
||||
ggml_tensor * embeddings = inpL;
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
|
||||
embeddings = build_ffn(embeddings,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
FFN_GELU,
|
||||
-1);
|
||||
|
||||
if (use_window_attn) {
|
||||
window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);
|
||||
ggml_set_name(window_idx, "window_idx");
|
||||
ggml_set_input(window_idx);
|
||||
|
||||
// embeddings shape: [n_embd, n_patches_x * n_patches_y, batch_size]
|
||||
GGML_ASSERT(batch_size == 1);
|
||||
embeddings = ggml_reshape_2d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4);
|
||||
embeddings = ggml_get_rows(ctx0, embeddings, window_idx);
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4, batch_size);
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_qwen3a::build() {
|
||||
// Ref implementation: https://github.com/QwenLM/Qwen3-ASR/blob/main/qwen_asr/core/transformers_backend/modeling_qwen3_asr.py
|
||||
|
||||
// inp_raw: [n_frames, n_mel, 1] (nx=n_frames, ny=n_mel)
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
|
||||
const int64_t n_frames = inp->ne[0]; // total frames, padded to multiple of chunk_size
|
||||
const int64_t n_mel = inp->ne[1]; // 128
|
||||
const int64_t chunk_size = 100; // n_window * 2 (n_window=50 from model config)
|
||||
const int64_t n_chunks = n_frames / chunk_size;
|
||||
|
||||
GGML_ASSERT(n_frames % chunk_size == 0); // preprocessor should already pad the input
|
||||
GGML_ASSERT(inp->type == GGML_TYPE_F32);
|
||||
|
||||
// View mel spectrogram as batched 100-frame chunks: [chunk_size, n_mel, 1, n_chunks]
|
||||
inp = ggml_view_4d(ctx0, inp,
|
||||
chunk_size, n_mel, 1, n_chunks,
|
||||
n_frames * (int64_t)sizeof(float), // nb[1]: stride over mel bins
|
||||
chunk_size * (int64_t)sizeof(float), // nb[2]: stride for C=1 (unused)
|
||||
chunk_size * (int64_t)sizeof(float), // nb[3]: stride over chunks
|
||||
0);
|
||||
inp = ggml_cont(ctx0, inp);
|
||||
cb(inp, "inp_chunks", -1);
|
||||
|
||||
// 3 x conv2d + gelu
|
||||
{
|
||||
// conv output [OW, OH, C_out, n_chunks]
|
||||
auto conv_block = [&](ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) {
|
||||
x = ggml_conv_2d(ctx0, w, x, 2, 2, 1, 1, 1, 1);
|
||||
if (b) {
|
||||
x = ggml_add(ctx0, x, ggml_reshape_4d(ctx0, b, 1, 1, x->ne[2], 1));
|
||||
}
|
||||
return ggml_gelu_erf(ctx0, x);
|
||||
};
|
||||
|
||||
inp = conv_block(inp, model.conv2d_1_w, model.conv2d_1_b);
|
||||
inp = conv_block(inp, model.conv2d_2_w, model.conv2d_2_b);
|
||||
inp = conv_block(inp, model.conv2d_3_w, model.conv2d_3_b);
|
||||
// inp: [OW=13, OH=16, OC=480, n_chunks]
|
||||
cb(inp, "after_conv_blocks", -1);
|
||||
}
|
||||
|
||||
// permute [OW=25, OH=16, OC=480, n_chunks] -> [OH=16, OC=480, OW=25, n_chunks]
|
||||
// reshape to [OH*OC=7680, OW*n_chunks]
|
||||
// feature index h+16*c = c*16+f (matches python code)
|
||||
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
|
||||
inp = ggml_reshape_2d(ctx0, inp, inp->ne[0] * inp->ne[1], inp->ne[2] * inp->ne[3]);
|
||||
|
||||
// Project to d_model: [d_model, 25*n_chunks]
|
||||
inp = ggml_mul_mat(ctx0, model.conv_out_w, inp);
|
||||
if (model.conv_out_b) {
|
||||
inp = ggml_add(ctx0, inp, model.conv_out_b);
|
||||
}
|
||||
cb(inp, "after_conv_out", -1);
|
||||
|
||||
const int64_t n_pos = inp->ne[1]; // 25 * n_chunks
|
||||
|
||||
// Per-chunk positional embeddings: repeat pos[0:13] for each chunk
|
||||
// (position indices reset 0..12 per chunk, not sequential across chunks)
|
||||
{
|
||||
const int64_t tokens_per_chunk = n_pos / n_chunks; // 13
|
||||
ggml_tensor * pos_tmp = ggml_view_2d(ctx0, model.position_embeddings,
|
||||
model.position_embeddings->ne[0], tokens_per_chunk,
|
||||
model.position_embeddings->nb[1], 0);
|
||||
ggml_tensor * tgt = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32,
|
||||
model.position_embeddings->ne[0], n_pos);
|
||||
inp = ggml_add(ctx0, inp, ggml_repeat(ctx0, pos_tmp, tgt));
|
||||
}
|
||||
|
||||
ggml_tensor * cur = build_vit(inp, n_pos,
|
||||
NORM_TYPE_NORMAL, hparams.ffn_op,
|
||||
nullptr, // pos embd already added above
|
||||
nullptr);
|
||||
cb(cur, "after_transformer", -1);
|
||||
|
||||
// MLP projector
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU_ERF, -1);
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,777 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
// on-device sampling: top-k, top-p, then a random draw
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code_gen::do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const {
|
||||
logits = ggml_reshape_1d(ctx0, logits, ggml_nelements(logits));
|
||||
const int64_t n_vocab = logits->ne[0];
|
||||
|
||||
// sort a's rows by idx
|
||||
auto sort_by = [this](ggml_tensor * a, ggml_tensor * idx) {
|
||||
ggml_tensor * a2d = ggml_reshape_2d(ctx0, a, 1, a->ne[0]);
|
||||
return ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, a2d, idx), idx->ne[0]);
|
||||
};
|
||||
|
||||
ggml_tensor * cur = logits;
|
||||
ggml_tensor * candidates = nullptr; // maps row index back to vocab id
|
||||
|
||||
if (top_k > 0 && top_k < n_vocab) {
|
||||
ggml_tensor * idx = ggml_top_k(ctx0, cur, top_k);
|
||||
candidates = idx;
|
||||
cur = sort_by(cur, idx);
|
||||
cb(cur, "sample_top_k_logits", -1);
|
||||
}
|
||||
|
||||
if (top_p < 1.0f) {
|
||||
ggml_tensor * sorted_idx = ggml_argsort(ctx0, cur, GGML_SORT_ORDER_DESC);
|
||||
ggml_tensor * sorted_logits = sort_by(cur, sorted_idx);
|
||||
candidates = candidates ? sort_by(candidates, sorted_idx) : sorted_idx;
|
||||
|
||||
ggml_tensor * probs = ggml_soft_max(ctx0, sorted_logits);
|
||||
ggml_tensor * cdf = ggml_cumsum(ctx0, probs);
|
||||
|
||||
// keep_mask[i] = 1 once cdf[i] crosses top_p
|
||||
ggml_tensor * cdf_scaled = ggml_scale_bias(ctx0, cdf, -1.0f, top_p);
|
||||
ggml_tensor * keep_mask = ggml_step(ctx0, cdf_scaled);
|
||||
ggml_tensor * idxf = ggml_sum(ctx0, keep_mask);
|
||||
idxf = ggml_clamp(ctx0, idxf, 0.0f, (float) keep_mask->ne[0] - 1);
|
||||
ggml_tensor * ones = ggml_scale_bias(ctx0, idxf, 0.0f, 1.0f);
|
||||
|
||||
// top-p must include the crossing element, so force it to 1
|
||||
ggml_tensor * keep_mask_2d = ggml_reshape_2d(ctx0, keep_mask, 1, keep_mask->ne[0]);
|
||||
keep_mask_2d = ggml_set_rows(ctx0, keep_mask_2d, ones, ggml_cast(ctx0, idxf, GGML_TYPE_I32));
|
||||
keep_mask = ggml_reshape_1d(ctx0, keep_mask_2d, keep_mask->ne[0]);
|
||||
|
||||
// log(1) = 0 (keep), log(0) = -inf (drop)
|
||||
ggml_tensor * bias = ggml_log(ctx0, keep_mask);
|
||||
cur = ggml_add(ctx0, sorted_logits, bias);
|
||||
cb(cur, "sample_top_p_logits", -1);
|
||||
}
|
||||
|
||||
// draw one token: find where the cdf crosses inp_rand
|
||||
ggml_tensor * probs = ggml_soft_max(ctx0, cur);
|
||||
ggml_tensor * cumsum = ggml_cumsum(ctx0, probs);
|
||||
|
||||
ggml_tensor * diff = ggml_sub(ctx0, cumsum, inp_rand);
|
||||
ggml_tensor * cross_mask = ggml_step(ctx0, diff);
|
||||
ggml_tensor * idxf = ggml_sum(ctx0, cross_mask);
|
||||
ggml_tensor * idx = ggml_cast(ctx0, ggml_scale_bias(ctx0, idxf, -1.0f, (float) cross_mask->ne[0]), GGML_TYPE_I32);
|
||||
|
||||
if (candidates) {
|
||||
ggml_tensor * cand_2d = ggml_reshape_2d(ctx0, candidates, 1, candidates->ne[0]);
|
||||
idx = ggml_get_rows(ctx0, cand_2d, idx);
|
||||
}
|
||||
cb(idx, "sample_token_id", -1);
|
||||
|
||||
return idx;
|
||||
}
|
||||
|
||||
// returns a new cache with row row_idx set to value
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code_gen::cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const {
|
||||
const int64_t n_embd = cache->ne[0];
|
||||
const int64_t n_cache = cache->ne[1];
|
||||
GGML_ASSERT(row_idx >= 0 && row_idx < n_cache);
|
||||
|
||||
// append value as the last row, then gather it back into place
|
||||
ggml_tensor * value_2d = ggml_reshape_2d(ctx0, value, n_embd, 1);
|
||||
ggml_tensor * cache_ext = ggml_concat(ctx0, cache, value_2d, 1); // [n_embd, n_cache + 1]
|
||||
|
||||
// gather indices [0..row_idx-1, n_cache, row_idx+1..n_cache-1]
|
||||
// built via concat, since ggml_set_rows needs F32/F16 values, not an I32 index array
|
||||
ggml_tensor * idx = const_i32(cache, (float) n_cache);
|
||||
if (row_idx > 0) {
|
||||
ggml_tensor * prefix = ggml_cast(ctx0, ggml_arange(ctx0, 0.0f, (float) row_idx, 1.0f), GGML_TYPE_I32);
|
||||
idx = ggml_concat(ctx0, prefix, idx, 0);
|
||||
}
|
||||
if (row_idx < n_cache - 1) {
|
||||
ggml_tensor * suffix = ggml_cast(ctx0, ggml_arange(ctx0, (float) (row_idx + 1), (float) n_cache, 1.0f), GGML_TYPE_I32);
|
||||
idx = ggml_concat(ctx0, idx, suffix, 0);
|
||||
}
|
||||
|
||||
ggml_tensor * result = ggml_get_rows(ctx0, cache_ext, idx);
|
||||
cb(result, "cache_set_out", -1);
|
||||
return result;
|
||||
}
|
||||
|
||||
// builds a const i32 with no host upload: view a tensor, zero it via scale, add value, cast to i32
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code_gen::const_i32(ggml_tensor * anchor, float value) const {
|
||||
ggml_tensor * v = ggml_view_1d(ctx0, anchor, 1, 0);
|
||||
if (v->type != GGML_TYPE_F32) {
|
||||
v = ggml_cast(ctx0, v, GGML_TYPE_F32);
|
||||
}
|
||||
return ggml_cast(ctx0, ggml_scale_bias(ctx0, v, 0.0f, value), GGML_TYPE_I32);
|
||||
}
|
||||
|
||||
// causal keep-mask row for a query at position pos, window size n_kv_pad
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code_gen::causal_mask_row(int64_t n_kv_pad, int pos) const {
|
||||
ggml_tensor * ones = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv_pad, n_kv_pad), 1.0f);
|
||||
ggml_tensor * keep = ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG);
|
||||
ggml_tensor * row = ggml_view_1d(ctx0, keep, n_kv_pad, (size_t) pos * keep->nb[1]);
|
||||
ggml_tensor * mask = ggml_log(ctx0, row); // 0 = keep, -inf = masked
|
||||
mask = ggml_reshape_4d(ctx0, mask, n_kv_pad, 1, 1, 1);
|
||||
if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
|
||||
mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
|
||||
}
|
||||
return mask;
|
||||
}
|
||||
|
||||
// talker hidden size -> predictor hidden size (small_to_mtp_projection)
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code_gen::project_in(ggml_tensor * cur) const {
|
||||
if (!model.gen_code_proj_in_w) {
|
||||
return cur;
|
||||
}
|
||||
cur = ggml_mul_mat(ctx0, model.gen_code_proj_in_w, cur);
|
||||
if (model.gen_code_proj_in_b) {
|
||||
cur = ggml_add(ctx0, cur, model.gen_code_proj_in_b);
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
// one transformer layer at position pos; writes k/v into k_cache_layer/v_cache_layer at row pos
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code_gen::layer_forward(
|
||||
ggml_tensor * cur,
|
||||
const clip_layer & layer,
|
||||
ggml_tensor * inp_pos,
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor *& k_cache_layer,
|
||||
ggml_tensor *& v_cache_layer,
|
||||
int64_t n_kv_pad,
|
||||
int pos,
|
||||
int il) const {
|
||||
const int n_head = hparams.n_head;
|
||||
const int n_head_kv = hparams.n_head_kv;
|
||||
const int64_t d_head = layer.q_w->ne[1] / n_head; // real head_dim, not n_embd / n_head
|
||||
const float kq_scale = 1.0f / sqrtf((float) d_head);
|
||||
|
||||
ggml_tensor * residual = cur;
|
||||
|
||||
ggml_tensor * h = ggml_rms_norm(ctx0, cur, hparams.eps);
|
||||
h = ggml_mul(ctx0, h, layer.ln_1_w);
|
||||
|
||||
ggml_tensor * q = ggml_mul_mat(ctx0, layer.q_w, h);
|
||||
ggml_tensor * k = ggml_mul_mat(ctx0, layer.k_w, h);
|
||||
ggml_tensor * v = ggml_mul_mat(ctx0, layer.v_w, h);
|
||||
|
||||
q = ggml_reshape_3d(ctx0, q, d_head, n_head, 1);
|
||||
k = ggml_reshape_3d(ctx0, k, d_head, n_head_kv, 1);
|
||||
|
||||
q = ggml_rms_norm(ctx0, q, hparams.eps);
|
||||
q = ggml_mul(ctx0, q, layer.q_norm);
|
||||
k = ggml_rms_norm(ctx0, k, hparams.eps);
|
||||
k = ggml_mul(ctx0, k, layer.k_norm);
|
||||
|
||||
q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0,
|
||||
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0,
|
||||
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
|
||||
// write k/v into the cache at row pos, flat layout
|
||||
ggml_tensor * k_flat = ggml_reshape_1d(ctx0, k, d_head * n_head_kv);
|
||||
k_cache_layer = cache_set(k_cache_layer, pos, k_flat);
|
||||
v_cache_layer = cache_set(v_cache_layer, pos, v);
|
||||
|
||||
ggml_tensor * q_cur = ggml_reshape_4d(ctx0, q, d_head, n_head, 1, 1);
|
||||
ggml_tensor * k_cur = ggml_reshape_4d(ctx0, k_cache_layer, d_head, n_head_kv, n_kv_pad, 1);
|
||||
ggml_tensor * v_cur = ggml_reshape_4d(ctx0, v_cache_layer, d_head, n_head_kv, n_kv_pad, 1);
|
||||
|
||||
ggml_tensor * attn_out = build_attn(layer.o_w, layer.o_b, q_cur, k_cur, v_cur, kq_mask, kq_scale, il);
|
||||
|
||||
cur = ggml_add(ctx0, residual, attn_out);
|
||||
|
||||
ggml_tensor * h2 = ggml_rms_norm(ctx0, cur, hparams.eps);
|
||||
h2 = ggml_mul(ctx0, h2, layer.ln_2_w);
|
||||
|
||||
ggml_tensor * gate = ggml_mul_mat(ctx0, layer.ff_gate_w, h2);
|
||||
ggml_tensor * up = ggml_mul_mat(ctx0, layer.ff_up_w, h2);
|
||||
ggml_tensor * gu = ggml_swiglu_split(ctx0, gate, up);
|
||||
ggml_tensor * down = ggml_mul_mat(ctx0, layer.ff_down_w, gu);
|
||||
|
||||
return ggml_add(ctx0, cur, down);
|
||||
}
|
||||
|
||||
// position 0: hidden bridge, seeds the k/v cache, no sampling
|
||||
// position 1: embed(code0), sample with lm_head[0], write out_code_cache[1]
|
||||
void clip_graph_qwen3tts_gen::code_gen::prefill(
|
||||
std::vector<ggml_tensor *> & k_cache,
|
||||
std::vector<ggml_tensor *> & v_cache,
|
||||
ggml_tensor *& out_code_cache,
|
||||
ggml_tensor * h_state,
|
||||
ggml_tensor * code0_embd,
|
||||
ggml_tensor * inp_rand) const {
|
||||
const int64_t n_kv_pad = k_cache[0]->ne[1];
|
||||
|
||||
{
|
||||
ggml_tensor * cur = project_in(h_state);
|
||||
ggml_tensor * kq_mask = causal_mask_row(n_kv_pad, 0);
|
||||
ggml_tensor * inp_pos = const_i32(k_cache[0], 0.0f);
|
||||
for (size_t il = 0; il < model.layers.size(); il++) {
|
||||
cur = layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_kv_pad, 0, (int) il);
|
||||
}
|
||||
// position 0's output is unused, it only seeded the cache
|
||||
}
|
||||
|
||||
{
|
||||
ggml_tensor * cur = project_in(code0_embd);
|
||||
ggml_tensor * kq_mask = causal_mask_row(n_kv_pad, 1);
|
||||
ggml_tensor * inp_pos = const_i32(k_cache[0], 1.0f);
|
||||
for (size_t il = 0; il < model.layers.size(); il++) {
|
||||
cur = layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_kv_pad, 1, (int) il);
|
||||
}
|
||||
|
||||
cur = ggml_rms_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_mul(ctx0, cur, model.gen_code_norm_w);
|
||||
|
||||
ggml_tensor * head_w = model.gen_code_head_w;
|
||||
ggml_tensor * head_g = ggml_view_2d(ctx0, head_w, head_w->ne[0], head_w->ne[1], head_w->nb[1], 0); // lm_head[0]
|
||||
ggml_tensor * logits = ggml_mul_mat(ctx0, head_g, cur);
|
||||
|
||||
ggml_tensor * sampled = do_sampling(logits, inp_rand);
|
||||
out_code_cache = cache_set(out_code_cache, 1, sampled);
|
||||
}
|
||||
}
|
||||
|
||||
// one decode step of code_predictor
|
||||
// at step_idx g:
|
||||
// - read code from out_code_cache[g], then embed it with codebook table g-1
|
||||
// - write new kv at cache row g+1, sample with lm_head[g]
|
||||
// - write result to out_code_cache[g+1]
|
||||
// step_idx must be in [1, n_acoustic - 1]
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code_gen::step(
|
||||
std::vector<ggml_tensor *> & k_cache,
|
||||
std::vector<ggml_tensor *> & v_cache,
|
||||
ggml_tensor * out_code_cache,
|
||||
ggml_tensor * inp_rand,
|
||||
int step_idx) const {
|
||||
const int64_t n_acoustic = model.gen_code_head_w->ne[2];
|
||||
GGML_ASSERT(step_idx >= 1 && step_idx < n_acoustic);
|
||||
GGML_ASSERT(k_cache.size() == model.layers.size());
|
||||
GGML_ASSERT(v_cache.size() == model.layers.size());
|
||||
|
||||
const int64_t n_kv_pad = k_cache[0]->ne[1];
|
||||
const int pos = step_idx + 1; // new cache row and RoPE position
|
||||
|
||||
// embed the previous code via this step's codebook table (rows are already scalars)
|
||||
ggml_tensor * code_in = ggml_view_1d(ctx0, out_code_cache, 1, (size_t) step_idx * out_code_cache->nb[1]);
|
||||
|
||||
ggml_tensor * embd_w = model.gen_code_embd_w; // [n_embd_talker, vocab, n_acoustic]
|
||||
ggml_tensor * embd_g = ggml_view_2d(ctx0, embd_w, embd_w->ne[0], embd_w->ne[1], embd_w->nb[1],
|
||||
(size_t) (step_idx - 1) * embd_w->nb[2]);
|
||||
ggml_tensor * cur = ggml_get_rows(ctx0, embd_g, code_in);
|
||||
cur = ggml_reshape_1d(ctx0, cur, cur->ne[0]);
|
||||
cb(cur, "step_embd_in", step_idx);
|
||||
|
||||
cur = project_in(cur);
|
||||
cb(cur, "step_proj_in", step_idx);
|
||||
|
||||
ggml_tensor * kq_mask = causal_mask_row(n_kv_pad, pos);
|
||||
ggml_tensor * inp_pos = const_i32(k_cache[0], (float) pos);
|
||||
|
||||
for (size_t il = 0; il < model.layers.size(); il++) {
|
||||
cur = layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_kv_pad, pos, (int) il);
|
||||
cb(cur, "step_layer_out", (int) il);
|
||||
}
|
||||
|
||||
// final norm, this step's lm_head, sample, write the result
|
||||
cur = ggml_rms_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_mul(ctx0, cur, model.gen_code_norm_w);
|
||||
|
||||
ggml_tensor * head_w = model.gen_code_head_w; // [n_embd_pred, vocab, n_acoustic]
|
||||
ggml_tensor * head_g = ggml_view_2d(ctx0, head_w, head_w->ne[0], head_w->ne[1], head_w->nb[1],
|
||||
(size_t) step_idx * head_w->nb[2]);
|
||||
ggml_tensor * logits = ggml_mul_mat(ctx0, head_g, cur);
|
||||
cb(logits, "step_logits", step_idx);
|
||||
|
||||
ggml_tensor * sampled = do_sampling(logits, inp_rand);
|
||||
cb(sampled, "step_sampled", step_idx);
|
||||
|
||||
return cache_set(out_code_cache, pos, sampled);
|
||||
}
|
||||
|
||||
// causal conv1d, stride 1: prepend persisted left-context instead of zero-padding, then a plain conv
|
||||
// x: [T, IC] (T-first). w: [K, IC, OC]. state_name empty means K == 1 (no left-context). returns [T, OC]
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const {
|
||||
const int K = (int) w->ne[0];
|
||||
const int pad = (K - 1) * dilation;
|
||||
|
||||
ggml_tensor * x_full = x;
|
||||
if (pad > 0) {
|
||||
ggml_tensor * left = state_in.at(state_name); // [pad, IC]
|
||||
x_full = ggml_concat(ctx0, left, x, 0);
|
||||
}
|
||||
ggml_tensor * y = ggml_conv_1d(ctx0, w, x_full, 1, 0, dilation); // [T, OC, 1]
|
||||
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
|
||||
if (b) {
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
if (pad > 0) {
|
||||
ggml_tensor * new_left = ggml_cont(ctx0, ggml_view_2d(ctx0, x_full, pad, x_full->ne[1], x_full->nb[1],
|
||||
(size_t) (x_full->ne[0] - pad) * x_full->nb[0]));
|
||||
state_out.push_back({state_name, new_left});
|
||||
}
|
||||
return y;
|
||||
}
|
||||
|
||||
// causal depthwise conv1d, stride 1, dilation 1, kernel from w's shape.
|
||||
// x: [T, C]. w: [K, 1, C]. returns [T, C]. see causal_conv1d for the state contract.
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const {
|
||||
const int K = (int) w->ne[0];
|
||||
const int pad = K - 1;
|
||||
|
||||
ggml_tensor * x_full = x;
|
||||
if (pad > 0) {
|
||||
ggml_tensor * left = state_in.at(state_name); // [pad, C]
|
||||
x_full = ggml_concat(ctx0, left, x, 0);
|
||||
}
|
||||
ggml_tensor * y = ggml_conv_1d_dw(ctx0, w, x_full, 1, 0, 1); // [T, C, 1]
|
||||
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
|
||||
if (b) {
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
if (pad > 0) {
|
||||
ggml_tensor * new_left = ggml_cont(ctx0, ggml_view_2d(ctx0, x_full, pad, x_full->ne[1], x_full->nb[1],
|
||||
(size_t) (x_full->ne[0] - pad) * x_full->nb[0]));
|
||||
state_out.push_back({state_name, new_left});
|
||||
}
|
||||
return y;
|
||||
}
|
||||
|
||||
// causal ConvTranspose1d, the (kernel - stride) overlap tail is kept as state for the next call
|
||||
// x: [T, IC], w: [K, OC, IC]. state_name empty means K == stride (no overlap). returns [T * stride, OC]
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const {
|
||||
const int K = (int) w->ne[0];
|
||||
const int OC = (int) w->ne[1];
|
||||
const int trim = K - stride;
|
||||
const int64_t emit_len = x->ne[0] * stride;
|
||||
|
||||
// transposed conv as GEMM + col2im scatter-add, y: [emit_len + trim, OC]
|
||||
ggml_tensor * w2 = ggml_reshape_2d(ctx0, w, (int64_t) K * OC, w->ne[2]);
|
||||
w2 = ggml_cont(ctx0, ggml_transpose(ctx0, w2));
|
||||
ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x));
|
||||
ggml_tensor * col = ggml_mul_mat(ctx0, w2, xt);
|
||||
ggml_tensor * y = ggml_col2im_1d(ctx0, col, stride, OC, 0);
|
||||
|
||||
ggml_tensor * out = y;
|
||||
if (trim > 0) {
|
||||
ggml_tensor * tail = state_in.at(state_name); // [trim, OC]
|
||||
ggml_tensor * head = ggml_add(ctx0, ggml_view_2d(ctx0, y, trim, y->ne[1], y->nb[1], 0), tail);
|
||||
if (emit_len > trim) {
|
||||
ggml_tensor * middle = ggml_view_2d(ctx0, y, emit_len - trim, y->ne[1], y->nb[1], (size_t) trim * y->nb[0]);
|
||||
out = ggml_concat(ctx0, head, middle, 0);
|
||||
} else {
|
||||
out = head;
|
||||
}
|
||||
ggml_tensor * new_tail = ggml_cont(ctx0, ggml_view_2d(ctx0, y, trim, y->ne[1], y->nb[1], (size_t) emit_len * y->nb[0]));
|
||||
state_out.push_back({state_name, new_tail});
|
||||
}
|
||||
if (b) {
|
||||
out = ggml_add(ctx0, out, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// SnakeBeta activation: y = x + sin(alpha*x)^2 * inv_beta (alpha/inv_beta folded via exp/reciprocal at conversion time)
|
||||
// x: [T, C]. alpha/beta: [C], broadcasts over T
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const {
|
||||
ggml_tensor * a = ggml_reshape_2d(ctx0, alpha, 1, alpha->ne[0]);
|
||||
ggml_tensor * b = ggml_reshape_2d(ctx0, beta, 1, beta->ne[0]);
|
||||
|
||||
// expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them
|
||||
ggml_build_forward_expand(gf, a);
|
||||
ggml_build_forward_expand(gf, b);
|
||||
|
||||
ggml_tensor * s = ggml_sin(ctx0, ggml_mul(ctx0, x, a));
|
||||
s = ggml_sqr(ctx0, s);
|
||||
s = ggml_mul(ctx0, s, b);
|
||||
return ggml_add(ctx0, x, s);
|
||||
}
|
||||
|
||||
// RVQ codebook decode: T frames of 16 codes -> 512-dim hidden (C-first, [512, T])
|
||||
// codebook 0 (semantic) and 1..15 (acoustic) sum within their group, project separately, then add
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_codes) const {
|
||||
const auto & c2w = model.c2w;
|
||||
const int64_t T = inp_codes->ne[0];
|
||||
|
||||
// ids for codebook group g over all T frames, [T] I32
|
||||
auto group_ids = [&](int g) {
|
||||
return ggml_view_1d(ctx0, inp_codes, T, (size_t) g * inp_codes->nb[1]);
|
||||
};
|
||||
|
||||
ggml_tensor * sem = ggml_get_rows(ctx0, c2w.quant_first_cb_w, group_ids(0)); // [256, T]
|
||||
ggml_tensor * sem_out = ggml_mul_mat(ctx0, c2w.quant_first_out_w, sem); // [512, T]
|
||||
|
||||
ggml_tensor * acc = nullptr;
|
||||
const int64_t n_acoustic = c2w.quant_rest_cb_w->ne[2];
|
||||
for (int g = 1; g <= n_acoustic; g++) {
|
||||
ggml_tensor * cb_g = ggml_view_2d(ctx0, c2w.quant_rest_cb_w, c2w.quant_rest_cb_w->ne[0], c2w.quant_rest_cb_w->ne[1],
|
||||
c2w.quant_rest_cb_w->nb[1], (size_t) (g - 1) * c2w.quant_rest_cb_w->nb[2]);
|
||||
ggml_tensor * embd = ggml_get_rows(ctx0, cb_g, group_ids(g)); // [256, T]
|
||||
acc = acc ? ggml_add(ctx0, acc, embd) : embd;
|
||||
}
|
||||
ggml_tensor * ac_out = ggml_mul_mat(ctx0, c2w.quant_rest_out_w, acc); // [512, T]
|
||||
|
||||
ggml_tensor * hidden = ggml_add(ctx0, sem_out, ac_out);
|
||||
cb(hidden, "wav_quant_hidden", -1);
|
||||
return hidden;
|
||||
}
|
||||
|
||||
// one pre_transformer layer over a batch of N = sliding_window new frames
|
||||
// attention runs over [(W-1)-frame prefix from the last batch] + [N new frames]
|
||||
// RoPE positions come from a persisted counter, so phases line up across batches
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const {
|
||||
const int n_head = hparams.wav_tfm_n_head;
|
||||
const int n_head_kv = hparams.wav_tfm_n_head_kv;
|
||||
const int64_t d_head = layer.q_w->ne[1] / n_head;
|
||||
const float kq_scale = 1.0f / sqrtf((float) d_head);
|
||||
const int64_t W = hparams.wav_tfm_swa; // == N, frames per batch
|
||||
const int64_t N = cur->ne[1];
|
||||
const int64_t prefix = W - 1;
|
||||
const int64_t total_kv = prefix + N;
|
||||
|
||||
ggml_tensor * residual = cur;
|
||||
ggml_tensor * h = ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps);
|
||||
h = ggml_mul(ctx0, h, layer.ln_1_w);
|
||||
|
||||
ggml_tensor * q = ggml_mul_mat(ctx0, layer.q_w, h); // [n_head*d_head, N]
|
||||
ggml_tensor * k = ggml_mul_mat(ctx0, layer.k_w, h); // [n_head_kv*d_head, N]
|
||||
ggml_tensor * v = ggml_mul_mat(ctx0, layer.v_w, h); // [n_head_kv*d_head, N]
|
||||
|
||||
q = ggml_reshape_3d(ctx0, q, d_head, n_head, N);
|
||||
k = ggml_reshape_3d(ctx0, k, d_head, n_head_kv, N);
|
||||
|
||||
// real, ever-increasing positions: base (persisted) .. base+N-1
|
||||
ggml_tensor * base = ggml_reshape_1d(ctx0, state_in.at("tfm_pos"), 1);
|
||||
ggml_tensor * offset = ggml_arange(ctx0, 0.0f, (float) N, 1.0f);
|
||||
ggml_tensor * pos = ggml_cast(ctx0, ggml_add(ctx0, offset, base), GGML_TYPE_I32);
|
||||
|
||||
q = ggml_rope_ext(ctx0, q, pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0,
|
||||
hparams.wav_tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
k = ggml_rope_ext(ctx0, k, pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0,
|
||||
hparams.wav_tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
|
||||
// the position counter is the same for all layers, push it once from layer 0
|
||||
if (il == 0) {
|
||||
state_out.push_back({"tfm_pos", ggml_scale_bias(ctx0, state_in.at("tfm_pos"), 1.0f, (float) N)});
|
||||
}
|
||||
|
||||
ggml_tensor * k_new = ggml_reshape_2d(ctx0, k, d_head * n_head_kv, N);
|
||||
ggml_tensor * v_new = ggml_reshape_2d(ctx0, v, d_head * n_head_kv, N);
|
||||
|
||||
ggml_tensor * old_k = state_in.at("tfm_k_" + std::to_string(il)); // [d_head*n_head_kv, W-1]
|
||||
ggml_tensor * old_v = state_in.at("tfm_v_" + std::to_string(il));
|
||||
|
||||
ggml_tensor * k_full = ggml_concat(ctx0, old_k, k_new, 1); // [.., prefix+N]
|
||||
ggml_tensor * v_full = ggml_concat(ctx0, old_v, v_new, 1);
|
||||
|
||||
// next batch's prefix: the last (W-1) frames of this batch
|
||||
state_out.push_back({"tfm_k_" + std::to_string(il),
|
||||
ggml_cont(ctx0, ggml_view_2d(ctx0, k_full, k_full->ne[0], prefix, k_full->nb[1], (size_t) N * k_full->nb[1]))});
|
||||
state_out.push_back({"tfm_v_" + std::to_string(il),
|
||||
ggml_cont(ctx0, ggml_view_2d(ctx0, v_full, v_full->ne[0], prefix, v_full->nb[1], (size_t) N * v_full->nb[1]))});
|
||||
|
||||
// banded causal mask: key j is visible to query i iff 0 <= (prefix+i) - j < W
|
||||
ggml_tensor * pos_k = ggml_reshape_2d(ctx0, ggml_arange(ctx0, 0.0f, (float) total_kv, 1.0f), total_kv, 1);
|
||||
ggml_tensor * pos_q = ggml_reshape_2d(ctx0, ggml_arange(ctx0, (float) prefix, (float) (prefix + N), 1.0f), 1, N);
|
||||
ggml_tensor * pos_q_grid = ggml_repeat_4d(ctx0, pos_q, total_kv, N, 1, 1);
|
||||
ggml_tensor * diff = ggml_sub(ctx0, pos_q_grid, pos_k); // [total_kv, N]
|
||||
|
||||
ggml_tensor * causal_keep = ggml_step(ctx0, ggml_scale_bias(ctx0, diff, 1.0f, 0.5f)); // diff >= 0
|
||||
ggml_tensor * in_window = ggml_step(ctx0, ggml_scale_bias(ctx0, diff, -1.0f, (float) W - 0.5f)); // diff < W
|
||||
ggml_tensor * keep = ggml_mul(ctx0, causal_keep, in_window);
|
||||
|
||||
// on a cold start, key j is real state only when j >= prefix - tfm_pos, mask out the rest
|
||||
ggml_tensor * warm = ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base),
|
||||
1.0f, 0.5f - (float) prefix)); // j + pos > prefix - 0.5
|
||||
keep = ggml_mul(ctx0, keep, warm);
|
||||
|
||||
ggml_tensor * mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), total_kv, N, 1, 1); // 0 = keep, -inf = masked
|
||||
if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
|
||||
mask = ggml_cast(ctx0, mask, GGML_TYPE_F16);
|
||||
}
|
||||
|
||||
ggml_tensor * q_cur = ggml_reshape_4d(ctx0, q, d_head, n_head, N, 1);
|
||||
ggml_tensor * k_cur = ggml_reshape_4d(ctx0, k_full, d_head, n_head_kv, total_kv, 1);
|
||||
ggml_tensor * v_cur = ggml_reshape_4d(ctx0, v_full, d_head, n_head_kv, total_kv, 1);
|
||||
|
||||
ggml_tensor * attn_out = build_attn(layer.o_w, layer.o_b, q_cur, k_cur, v_cur, mask, kq_scale, il);
|
||||
if (layer.ls_1_w) {
|
||||
attn_out = ggml_mul(ctx0, attn_out, layer.ls_1_w);
|
||||
}
|
||||
cur = ggml_add(ctx0, residual, attn_out);
|
||||
|
||||
ggml_tensor * residual2 = cur;
|
||||
ggml_tensor * h2 = ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps);
|
||||
h2 = ggml_mul(ctx0, h2, layer.ln_2_w);
|
||||
|
||||
ggml_tensor * gate = ggml_mul_mat(ctx0, layer.ff_gate_w, h2);
|
||||
ggml_tensor * up = ggml_mul_mat(ctx0, layer.ff_up_w, h2);
|
||||
ggml_tensor * gu = ggml_swiglu_split(ctx0, gate, up);
|
||||
ggml_tensor * down = ggml_mul_mat(ctx0, layer.ff_down_w, gu);
|
||||
if (layer.ls_2_w) {
|
||||
down = ggml_mul(ctx0, down, layer.ls_2_w);
|
||||
}
|
||||
return ggml_add(ctx0, residual2, down);
|
||||
}
|
||||
|
||||
// dwconv -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual
|
||||
// x: [T, C] T-first; LayerNorm/pwconv need C on ne0, so this transposes in and back out
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const {
|
||||
ggml_tensor * residual = x;
|
||||
|
||||
ggml_tensor * h = causal_conv1d_dw(x, blk.dwconv_w, blk.dwconv_b, state_prefix + "_dwconv"); // [T, C]
|
||||
ggml_tensor * hc = ggml_cont(ctx0, ggml_transpose(ctx0, h)); // [C, T]
|
||||
|
||||
hc = ggml_norm(ctx0, hc, 1e-6f);
|
||||
hc = ggml_mul(ctx0, hc, blk.norm_w);
|
||||
hc = ggml_add(ctx0, hc, blk.norm_b);
|
||||
|
||||
ggml_tensor * g = ggml_mul_mat(ctx0, blk.pw1_w, hc);
|
||||
g = ggml_add(ctx0, g, blk.pw1_b);
|
||||
g = ggml_gelu(ctx0, g);
|
||||
g = ggml_mul_mat(ctx0, blk.pw2_w, g);
|
||||
g = ggml_add(ctx0, g, blk.pw2_b);
|
||||
g = ggml_mul(ctx0, g, blk.gamma);
|
||||
|
||||
ggml_tensor * g_t = ggml_cont(ctx0, ggml_transpose(ctx0, g)); // back to [T, C]
|
||||
return ggml_add(ctx0, residual, g_t);
|
||||
}
|
||||
|
||||
// SnakeBeta -> dilated causal conv (k=7) -> SnakeBeta -> pointwise causal conv (k=1) -> residual.
|
||||
// x: [T, C]. returns [T, C].
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const {
|
||||
ggml_tensor * residual = x;
|
||||
ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta);
|
||||
h = causal_conv1d(h, res.conv1_w, res.conv1_b, dilation, state_name);
|
||||
h = snake(h, res.act2_alpha, res.act2_beta);
|
||||
h = causal_conv1d(h, res.conv2_w, res.conv2_b, 1, ""); // k=1, no left-context needed
|
||||
return ggml_add(ctx0, residual, h);
|
||||
}
|
||||
|
||||
// RVQ codes -> raw PCM for a batch of N = sliding_window frames
|
||||
ggml_tensor * clip_graph_qwen3tts_gen::code2wav::decode(ggml_tensor * inp_codes) const {
|
||||
const auto & c2w = model.c2w;
|
||||
|
||||
// 1. quantizer decode: N frames of 16 codes -> [512, N] (C-first)
|
||||
ggml_tensor * hidden = quant_decode(inp_codes);
|
||||
|
||||
// 2. pre_conv: [512, N] -> T-first [N, 512] -> causal conv k=3 -> [N, 1024]
|
||||
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, hidden)); // [N, 512]
|
||||
x = causal_conv1d(x, c2w.pre_conv_w, c2w.pre_conv_b, 1, "pre_conv"); // [N, 1024]
|
||||
cb(x, "wav_pre_conv_out", -1);
|
||||
|
||||
// 3. pre_transformer: back to C-first [1024, N], project down, run the layers, project back up
|
||||
ggml_tensor * cur = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [1024, N]
|
||||
cur = ggml_mul_mat(ctx0, c2w.tfm_in_proj_w, cur);
|
||||
cur = ggml_add(ctx0, cur, c2w.tfm_in_proj_b); // [512 (tfm hidden), N]
|
||||
|
||||
for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
|
||||
cur = tfm_layer_forward(cur, c2w.tfm_layers[il], il);
|
||||
}
|
||||
|
||||
cur = ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps);
|
||||
cur = ggml_mul(ctx0, cur, c2w.tfm_output_norm_w);
|
||||
cur = ggml_mul_mat(ctx0, c2w.tfm_out_proj_w, cur);
|
||||
cur = ggml_add(ctx0, cur, c2w.tfm_out_proj_b); // [1024, N]
|
||||
cb(cur, "wav_tfm_out", -1);
|
||||
|
||||
// 4. upsample: 2x (causal ConvTranspose1d, stride 2 + ConvNeXt block), back to T-first
|
||||
// kernel == stride here, so there is no overlap tail to persist
|
||||
x = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [N, 1024]
|
||||
for (size_t il = 0; il < c2w.upsample.size(); il++) {
|
||||
const auto & up = c2w.upsample[il];
|
||||
x = causal_conv_transpose1d(x, up.conv_w, up.conv_b, 2, "");
|
||||
x = convnext_block(x, up, "up" + std::to_string(il));
|
||||
cb(x, "wav_upsample_out", (int) il);
|
||||
}
|
||||
|
||||
// 5. DAC decoder: conv_pre -> n blocks (SnakeBeta -> ConvTranspose1d -> 3 res units) -> conv_post
|
||||
static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 };
|
||||
|
||||
x = causal_conv1d(x, c2w.dac_entry_w, c2w.dac_entry_b, 1, "dac_entry");
|
||||
cb(x, "wav_dac_entry_out", -1);
|
||||
|
||||
for (size_t il = 0; il < c2w.dac.size(); il++) {
|
||||
const auto & blk = c2w.dac[il];
|
||||
const int stride = (int) (blk.conv_w->ne[0] / 2); // kernel == 2*stride for all 4 blocks
|
||||
const std::string blk_name = "dac" + std::to_string(il);
|
||||
x = snake(x, blk.snake_alpha, blk.snake_beta);
|
||||
x = causal_conv_transpose1d(x, blk.conv_w, blk.conv_b, stride, blk_name + "_tail");
|
||||
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
||||
x = dac_res_unit(x, blk.res[ir], DAC_DILATIONS[ir], blk_name + "_res" + std::to_string(ir));
|
||||
}
|
||||
cb(x, "wav_dac_block_out", (int) il);
|
||||
}
|
||||
|
||||
x = snake(x, c2w.dac_post_snake_alpha, c2w.dac_post_snake_beta);
|
||||
x = causal_conv1d(x, c2w.dac_post_conv_w, c2w.dac_post_conv_b, 1, "dac_post_conv"); // [n_samples, 1]
|
||||
|
||||
x = ggml_clamp(ctx0, x, -1.0f, 1.0f);
|
||||
x = ggml_reshape_1d(ctx0, x, x->ne[0]);
|
||||
cb(x, "wav_audio_out", -1);
|
||||
return x;
|
||||
}
|
||||
|
||||
// code2wav's persisted state buffers: RoPE position counter, K/V per pre_transformer layer,
|
||||
// left-context/tail per stateful conv. shape lookup only, no graph needed
|
||||
std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model) {
|
||||
const auto & c2w = model.c2w;
|
||||
std::vector<c2w_state_slot> slots;
|
||||
|
||||
if (c2w.pre_conv_w == nullptr) {
|
||||
return slots; // not a code2wav model, it keeps no state between calls
|
||||
}
|
||||
|
||||
slots.push_back({"tfm_pos", 1, 1});
|
||||
|
||||
// prefix is (W-1) frames, the batch itself gives the other N=W frames (see tfm_layer_forward)
|
||||
const int64_t d_head = c2w.tfm_layers[0].q_w->ne[1] / hparams.wav_tfm_n_head;
|
||||
const int64_t kv_ch = d_head * hparams.wav_tfm_n_head_kv;
|
||||
const int64_t prefix = hparams.wav_tfm_swa - 1;
|
||||
for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
|
||||
slots.push_back({"tfm_k_" + std::to_string(il), kv_ch, prefix});
|
||||
slots.push_back({"tfm_v_" + std::to_string(il), kv_ch, prefix});
|
||||
}
|
||||
|
||||
slots.push_back({"pre_conv", c2w.pre_conv_w->ne[0] - 1, c2w.pre_conv_w->ne[1]});
|
||||
|
||||
for (size_t il = 0; il < c2w.upsample.size(); il++) {
|
||||
const auto & up = c2w.upsample[il];
|
||||
slots.push_back({"up" + std::to_string(il) + "_dwconv", up.dwconv_w->ne[0] - 1, up.dwconv_w->ne[2]});
|
||||
}
|
||||
|
||||
slots.push_back({"dac_entry", c2w.dac_entry_w->ne[0] - 1, c2w.dac_entry_w->ne[1]});
|
||||
|
||||
static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 };
|
||||
for (size_t il = 0; il < c2w.dac.size(); il++) {
|
||||
const auto & blk = c2w.dac[il];
|
||||
const int64_t stride = blk.conv_w->ne[0] / 2; // kernel == 2*stride for all 4 blocks
|
||||
const std::string blk_name = "dac" + std::to_string(il);
|
||||
slots.push_back({blk_name + "_tail", stride, blk.conv_w->ne[1]});
|
||||
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
||||
const auto & res = blk.res[ir];
|
||||
slots.push_back({blk_name + "_res" + std::to_string(ir),
|
||||
(res.conv1_w->ne[0] - 1) * DAC_DILATIONS[ir], res.conv1_w->ne[1]});
|
||||
}
|
||||
}
|
||||
|
||||
slots.push_back({"dac_post_conv", c2w.dac_post_conv_w->ne[0] - 1, c2w.dac_post_conv_w->ne[1]});
|
||||
|
||||
return slots;
|
||||
}
|
||||
|
||||
// both sub-graphs are always built, so the topology stays constant
|
||||
// ggml_build_forward_select() then picks the one that actually runs
|
||||
ggml_cgraph * clip_graph_qwen3tts_gen::build() {
|
||||
GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time
|
||||
|
||||
int idx;
|
||||
switch (gen_process) {
|
||||
case CLIP_GEN_PROCESS_GEN_CODE: idx = 0; break;
|
||||
case CLIP_GEN_PROCESS_GEN_WAV: idx = 1; break;
|
||||
default: GGML_ABORT("unknown gen_process");
|
||||
}
|
||||
|
||||
// ---- CLIP_GEN_PROCESS_GEN_CODE: backbone hidden state -> 16 RVQ codes + next-step embd ----
|
||||
// not build_inp_raw(), a GEN_WAV call's `img` has no hidden-state data
|
||||
ggml_tensor * h_state = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
|
||||
ggml_set_name(h_state, "inp_raw"); // must keep this exact name, clip_encode() sets it by name
|
||||
ggml_set_input(h_state);
|
||||
|
||||
ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
|
||||
ggml_set_name(code0, "inp_code0");
|
||||
ggml_set_input(code0);
|
||||
|
||||
ggml_tensor * code0_embd = ggml_get_rows(ctx0, model.gen_code_out_embd_w, code0);
|
||||
code0_embd = ggml_reshape_1d(ctx0, code0_embd, code0_embd->ne[0]);
|
||||
cb(code0_embd, "code0_embd", -1);
|
||||
|
||||
const int64_t n_acoustic = model.gen_code_head_w->ne[2]; // 15
|
||||
const int n_codes = (int) n_acoustic + 1; // 16
|
||||
const int64_t n_kv_pad = n_codes;
|
||||
const int n_layer = (int) model.layers.size();
|
||||
const int n_head = hparams.n_head;
|
||||
const int n_head_kv = hparams.n_head_kv;
|
||||
const int64_t d_head = model.layers[0].q_w->ne[1] / n_head;
|
||||
|
||||
// zero-filled per layer k/v caches, so masked-out rows can't hold garbage
|
||||
std::vector<ggml_tensor *> k_cache(n_layer), v_cache(n_layer);
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
k_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, d_head * n_head_kv, n_kv_pad), 0.0f);
|
||||
v_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, d_head * n_head_kv, n_kv_pad), 0.0f);
|
||||
}
|
||||
|
||||
code_gen cg(*this, top_k, top_p);
|
||||
|
||||
ggml_tensor * out_code_cache = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_codes);
|
||||
out_code_cache = cg.cache_set(out_code_cache, 0, code0);
|
||||
|
||||
ggml_tensor * inp_rand0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
|
||||
ggml_set_name(inp_rand0, "inp_rand_0");
|
||||
ggml_set_input(inp_rand0);
|
||||
|
||||
cg.prefill(k_cache, v_cache, out_code_cache, h_state, code0_embd, inp_rand0);
|
||||
|
||||
for (int g = 1; g < n_acoustic; g++) {
|
||||
ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
|
||||
ggml_set_name(inp_rand, ("inp_rand_" + std::to_string(g)).c_str());
|
||||
ggml_set_input(inp_rand);
|
||||
out_code_cache = cg.step(k_cache, v_cache, out_code_cache, inp_rand, g);
|
||||
}
|
||||
|
||||
// output 1: this frame's 16 sampled codes, for the caller's code2wav window
|
||||
ggml_tensor * out_codes = ggml_cont(ctx0, out_code_cache);
|
||||
ggml_set_name(out_codes, "out_codes");
|
||||
ggml_set_output(out_codes);
|
||||
|
||||
// output 2: sum of all 16 codebook embeddings, fed back to the talker for the next frame
|
||||
ggml_tensor * out_embd = code0_embd;
|
||||
for (int g = 1; g <= n_acoustic; g++) {
|
||||
ggml_tensor * code_g = ggml_view_1d(ctx0, out_code_cache, 1, (size_t) g * out_code_cache->nb[1]);
|
||||
|
||||
ggml_tensor * embd_g = ggml_view_2d(ctx0, model.gen_code_embd_w, model.gen_code_embd_w->ne[0], model.gen_code_embd_w->ne[1],
|
||||
model.gen_code_embd_w->nb[1], (size_t) (g - 1) * model.gen_code_embd_w->nb[2]);
|
||||
ggml_tensor * e = ggml_get_rows(ctx0, embd_g, code_g);
|
||||
e = ggml_reshape_1d(ctx0, e, e->ne[0]);
|
||||
|
||||
out_embd = ggml_add(ctx0, out_embd, e);
|
||||
}
|
||||
out_embd = ggml_reshape_2d(ctx0, out_embd, out_embd->ne[0], 1);
|
||||
cb(out_embd, "gen_audio_out", -1);
|
||||
|
||||
// ---- CLIP_GEN_PROCESS_GEN_WAV: 16 RVQ codes -> raw PCM ----
|
||||
const int n_frames = hparams.wav_tfm_swa; // frames per batch, == the attention window
|
||||
|
||||
ggml_tensor * inp_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_frames, n_codes);
|
||||
ggml_set_name(inp_codes, "inp_codes");
|
||||
ggml_set_input(inp_codes);
|
||||
|
||||
code2wav c2w(*this);
|
||||
for (const auto & slot : list_c2w_state_slots(hparams, model)) {
|
||||
ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
|
||||
ggml_set_name(t, ("state_in_" + slot.name).c_str());
|
||||
ggml_set_input(t);
|
||||
c2w.state_in[slot.name] = t;
|
||||
}
|
||||
|
||||
ggml_tensor * out_audio = c2w.decode(inp_codes);
|
||||
ggml_set_name(out_audio, "out_audio");
|
||||
ggml_set_output(out_audio);
|
||||
|
||||
for (auto & slot : c2w.state_out) {
|
||||
ggml_set_name(slot.second, ("state_out_" + slot.first).c_str());
|
||||
ggml_set_output(slot.second);
|
||||
}
|
||||
|
||||
// out_embd goes last, clip_encode() reads it back via ggml_graph_node(gf, -1)
|
||||
ggml_tensor * outs[2];
|
||||
outs[0] = out_codes; outs[1] = out_audio;
|
||||
ggml_build_forward_select(gf, outs, 2, idx);
|
||||
for (auto & slot : c2w.state_out) {
|
||||
outs[0] = out_codes; outs[1] = slot.second;
|
||||
ggml_build_forward_select(gf, outs, 2, idx);
|
||||
}
|
||||
outs[0] = out_embd; outs[1] = out_audio;
|
||||
ggml_build_forward_select(gf, outs, 2, idx);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
#include "models.h"
|
||||
|
||||
static constexpr int SPK_RES2NET_SCALE = 8; // enc_res2net_scale
|
||||
static constexpr int SPK_DILATIONS[3] = { 2, 3, 4 }; // enc_dilations[1..3]
|
||||
|
||||
// conv1d, kernel K, padding "same" (reflect), dilation d
|
||||
// x: [C, T] (ne[0]=C, ne[1]=T) -> [out_c, T]
|
||||
ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
|
||||
const int K = (int) w->ne[0];
|
||||
const int IC = (int) w->ne[1];
|
||||
const int OC = (int) w->ne[2];
|
||||
const int pad = ((K - 1) * dilation) / 2;
|
||||
|
||||
// ggml_pad_reflect_1d pads ne[0], so bring T onto ne[0] first, same layout as im2col wants
|
||||
ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, IC]
|
||||
if (pad > 0) {
|
||||
x_t = ggml_pad_reflect_1d(ctx0, x_t, pad, pad); // [T + 2*pad, IC]
|
||||
}
|
||||
ggml_tensor * x4d = ggml_reshape_4d(ctx0, x_t, x_t->ne[0], IC, 1, 1);
|
||||
|
||||
// dummy F32 kernel, im2col only reads its shape, so a quantized w does not assert
|
||||
ggml_tensor * dummy = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, K, IC, 1, 1);
|
||||
|
||||
ggml_tensor * col = ggml_im2col(ctx0, dummy, x4d, 1, 1, 0, 0, dilation, 1, false, GGML_TYPE_F32);
|
||||
const int64_t T_out = col->ne[1];
|
||||
col = ggml_reshape_2d(ctx0, col, (int64_t) K * IC, T_out);
|
||||
|
||||
ggml_tensor * w2d = ggml_reshape_2d(ctx0, w, (int64_t) K * IC, OC);
|
||||
ggml_tensor * y = ggml_mul_mat(ctx0, w2d, col); // [OC, T_out]
|
||||
ggml_prec_set_acc(y, GGML_PREC_F32);
|
||||
|
||||
ggml_tensor * b2d = ggml_reshape_2d(ctx0, b, OC, 1);
|
||||
y = ggml_add(ctx0, y, b2d);
|
||||
return y;
|
||||
}
|
||||
|
||||
// Res2Net: split channel axis into `scale` chunks, chain dilated conv1d branches
|
||||
// x: [C, T] -> [C, T]
|
||||
ggml_tensor * clip_graph_qwen3tts_spkenc::res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const {
|
||||
const int64_t C = x->ne[0];
|
||||
const int64_t T = x->ne[1];
|
||||
const int64_t Cs = C / scale;
|
||||
|
||||
std::vector<ggml_tensor *> outs;
|
||||
outs.reserve(scale);
|
||||
|
||||
auto chunk = [&](int i) -> ggml_tensor * {
|
||||
return ggml_view_2d(ctx0, x, Cs, T, x->nb[1], (size_t) i * Cs * x->nb[0]);
|
||||
};
|
||||
|
||||
ggml_tensor * prev = nullptr;
|
||||
for (int i = 0; i < scale; i++) {
|
||||
ggml_tensor * c = ggml_cont(ctx0, chunk(i));
|
||||
if (i == 0) {
|
||||
outs.push_back(c);
|
||||
continue;
|
||||
}
|
||||
ggml_tensor * inp = (i >= 2) ? ggml_add(ctx0, c, prev) : c;
|
||||
ggml_tensor * y = conv1d_same(inp, layer.res2_conv_w[i - 1], layer.res2_conv_b[i - 1], dilation);
|
||||
y = ggml_relu(ctx0, y);
|
||||
outs.push_back(y);
|
||||
prev = y;
|
||||
}
|
||||
|
||||
ggml_tensor * acc = outs[0];
|
||||
for (int i = 1; i < scale; i++) {
|
||||
acc = ggml_concat(ctx0, acc, outs[i], 0);
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
// squeeze-and-excitation gate. x: [C, T] -> [C, T]
|
||||
ggml_tensor * clip_graph_qwen3tts_spkenc::se_block(ggml_tensor * x, const clip_layer & layer) const {
|
||||
// temporal mean, keepdim: transpose so T is on ne[0], reduce, transpose back
|
||||
ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C]
|
||||
ggml_tensor * mean = ggml_mean(ctx0, x_t); // [1, C]
|
||||
mean = ggml_cont(ctx0, ggml_transpose(ctx0, mean)); // [C, 1]
|
||||
|
||||
ggml_tensor * h = conv1d_same(mean, layer.se_conv1_w, layer.se_conv1_b, 1);
|
||||
h = ggml_relu(ctx0, h);
|
||||
h = conv1d_same(h, layer.se_conv2_w, layer.se_conv2_b, 1);
|
||||
h = ggml_sigmoid(ctx0, h); // [C, 1]
|
||||
|
||||
return ggml_mul(ctx0, x, h); // broadcast gate over T
|
||||
}
|
||||
|
||||
// tdnn1 -> res2net -> tdnn2 -> se, plus residual. x: [C, T] -> [C, T]
|
||||
ggml_tensor * clip_graph_qwen3tts_spkenc::se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const {
|
||||
ggml_tensor * residual = x;
|
||||
ggml_tensor * h = conv1d_same(x, layer.conv_pw1_w, layer.conv_pw1_b, 1); // tdnn1
|
||||
h = ggml_relu(ctx0, h);
|
||||
h = res2net(h, layer, dilation, scale);
|
||||
h = conv1d_same(h, layer.conv_pw2_w, layer.conv_pw2_b, 1); // tdnn2
|
||||
h = ggml_relu(ctx0, h);
|
||||
h = se_block(h, layer);
|
||||
return ggml_add(ctx0, h, residual);
|
||||
}
|
||||
|
||||
// attentive statistics pooling. x: [C, T] -> [2*C, 1]
|
||||
ggml_tensor * clip_graph_qwen3tts_spkenc::attentive_stats_pool(ggml_tensor * x) const {
|
||||
const int64_t T = x->ne[1];
|
||||
|
||||
// mean over T: [C, 1]
|
||||
ggml_tensor * x_t = ggml_cont(ctx0, ggml_transpose(ctx0, x));
|
||||
ggml_tensor * mean = ggml_mean(ctx0, x_t);
|
||||
mean = ggml_cont(ctx0, ggml_transpose(ctx0, mean));
|
||||
|
||||
// std over T: sqrt(clamp(mean((x - mean)^2), eps))
|
||||
ggml_tensor * mean_rep = ggml_repeat(ctx0, mean, x);
|
||||
ggml_tensor * centered = ggml_sub(ctx0, x, mean_rep);
|
||||
ggml_tensor * var_t = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_sqr(ctx0, centered)));
|
||||
ggml_tensor * var = ggml_mean(ctx0, var_t);
|
||||
var = ggml_cont(ctx0, ggml_transpose(ctx0, var));
|
||||
var = ggml_scale_bias(ctx0, var, 1.0f, 1e-12f);
|
||||
ggml_tensor * std = ggml_sqrt(ctx0, var);
|
||||
|
||||
// attention input: cat([x, mean, std]) along channel axis -> [3C, T]
|
||||
ggml_tensor * std_rep = ggml_repeat(ctx0, std, x);
|
||||
ggml_tensor * cat = ggml_concat(ctx0, x, mean_rep, 0);
|
||||
cat = ggml_concat(ctx0, cat, std_rep, 0);
|
||||
|
||||
// attention TDNN (3C -> attn_c) + ReLU, tanh, then 1x1 conv (attn_c -> C)
|
||||
ggml_tensor * a = conv1d_same(cat, model.spk_asp_tdnn_w, model.spk_asp_tdnn_b, 1);
|
||||
a = ggml_relu(ctx0, a);
|
||||
a = ggml_tanh(ctx0, a);
|
||||
a = conv1d_same(a, model.spk_asp_attn_w, model.spk_asp_attn_b, 1);
|
||||
|
||||
// softmax over T
|
||||
ggml_tensor * a_t = ggml_cont(ctx0, ggml_transpose(ctx0, a)); // [T, C]
|
||||
ggml_tensor * w_t = ggml_soft_max(ctx0, a_t);
|
||||
ggml_tensor * w = ggml_cont(ctx0, ggml_transpose(ctx0, w_t)); // [C, T]
|
||||
|
||||
// weighted mean: sum(w * x) over T, multiply by T to undo ggml_mean's 1/T scaling
|
||||
ggml_tensor * wx = ggml_mul(ctx0, w, x);
|
||||
ggml_tensor * wx_t = ggml_cont(ctx0, ggml_transpose(ctx0, wx));
|
||||
ggml_tensor * w_mean = ggml_mean(ctx0, wx_t);
|
||||
w_mean = ggml_scale(ctx0, w_mean, (float) T);
|
||||
w_mean = ggml_cont(ctx0, ggml_transpose(ctx0, w_mean)); // [C, 1]
|
||||
|
||||
// weighted std: sum(w * (x - w_mean)^2) over T
|
||||
ggml_tensor * w_mean_rep = ggml_repeat(ctx0, w_mean, x);
|
||||
ggml_tensor * dev = ggml_sub(ctx0, x, w_mean_rep);
|
||||
ggml_tensor * w_var_in = ggml_mul(ctx0, w, ggml_sqr(ctx0, dev));
|
||||
ggml_tensor * w_var_t = ggml_cont(ctx0, ggml_transpose(ctx0, w_var_in));
|
||||
ggml_tensor * w_var = ggml_mean(ctx0, w_var_t);
|
||||
w_var = ggml_scale(ctx0, w_var, (float) T);
|
||||
w_var = ggml_cont(ctx0, ggml_transpose(ctx0, w_var));
|
||||
w_var = ggml_scale_bias(ctx0, w_var, 1.0f, 1e-12f);
|
||||
ggml_tensor * w_std = ggml_sqrt(ctx0, w_var);
|
||||
|
||||
return ggml_concat(ctx0, w_mean, w_std, 0); // [2C, 1]
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_qwen3tts_spkenc::build() {
|
||||
// inp_raw: [T, n_mel, 1, 1], from mtmd_audio_preprocessor_qwen3tts_spk
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
inp = ggml_reshape_2d(ctx0, inp, inp->ne[0], inp->ne[1]);
|
||||
|
||||
// this file's convention is [C, T]; the preprocessor delivers [T, C]
|
||||
ggml_tensor * mel = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [n_mel, T]
|
||||
cb(mel, "mel", -1);
|
||||
|
||||
// frontend conv0 TDNN k=5, dilation=1: 128 -> 512
|
||||
ggml_tensor * cur = conv1d_same(mel, model.conv1d_1_w, model.conv1d_1_b, 1);
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cb(cur, "frontend", -1);
|
||||
|
||||
// 3 SE-Res2Net blocks at dilations 2, 3, 4
|
||||
GGML_ASSERT((int) model.layers.size() == 3);
|
||||
std::vector<ggml_tensor *> blk_out(3);
|
||||
for (int il = 0; il < 3; il++) {
|
||||
cur = se_res2net_block(cur, model.layers[il], SPK_DILATIONS[il], SPK_RES2NET_SCALE);
|
||||
blk_out[il] = cur;
|
||||
cb(cur, "block_out", il);
|
||||
}
|
||||
|
||||
// multi-layer feature aggregation: cat blk[0..2] then TDNN k=1 + ReLU
|
||||
ggml_tensor * cat = ggml_concat(ctx0, blk_out[0], blk_out[1], 0);
|
||||
cat = ggml_concat(ctx0, cat, blk_out[2], 0); // [1536, T]
|
||||
ggml_tensor * mfa = conv1d_same(cat, model.conv_out_w, model.conv_out_b, 1);
|
||||
mfa = ggml_relu(ctx0, mfa);
|
||||
cb(mfa, "mfa", -1);
|
||||
|
||||
// attentive statistics pooling: [1536, T] -> [3072, 1]
|
||||
ggml_tensor * stats = attentive_stats_pool(mfa);
|
||||
cb(stats, "asp", -1);
|
||||
|
||||
// final FC k=1: [3072, 1] -> [enc_dim, 1]
|
||||
ggml_tensor * emb = conv1d_same(stats, model.mm_fc_w, model.mm_fc_b, 1);
|
||||
|
||||
emb = ggml_reshape_1d(ctx0, emb, emb->ne[0]);
|
||||
emb = ggml_cont(ctx0, emb);
|
||||
cb(emb, "spk_embedding", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, emb);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_qwen3vl::build() {
|
||||
GGML_ASSERT(model.patch_bias != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
|
||||
const int batch_size = 1;
|
||||
const int n_pos = n_patches;
|
||||
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
|
||||
|
||||
norm_type norm_t = NORM_TYPE_NORMAL;
|
||||
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
|
||||
ggml_tensor * inp = build_inp_temporal_merge();
|
||||
|
||||
// spatial merge
|
||||
{
|
||||
inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b]
|
||||
inp = ggml_cont_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
inp = ggml_reshape_4d(
|
||||
ctx0, inp,
|
||||
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
|
||||
inp = ggml_cont_3d(
|
||||
ctx0, inp,
|
||||
n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
|
||||
// add patch bias
|
||||
if (model.patch_bias != nullptr) {
|
||||
inp = ggml_add(ctx0, inp, model.patch_bias);
|
||||
cb(inp, "patch_bias", -1);
|
||||
}
|
||||
|
||||
// calculate absolute position embedding and apply
|
||||
ggml_tensor * learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS);
|
||||
learned_pos_embd = ggml_cont_4d(
|
||||
ctx0, learned_pos_embd,
|
||||
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
|
||||
learned_pos_embd = ggml_reshape_4d(
|
||||
ctx0, learned_pos_embd,
|
||||
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
|
||||
learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);
|
||||
learned_pos_embd = ggml_cont_3d(
|
||||
ctx0, learned_pos_embd,
|
||||
n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
inp = ggml_add(ctx0, inp, learned_pos_embd);
|
||||
cb(inp, "inp_pos_emb", -1);
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
// pre-layernorm
|
||||
if (model.pre_ln_w) {
|
||||
inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1);
|
||||
}
|
||||
|
||||
// deepstack features (stack along the feature dimension), [n_embd * len(deepstack_layers), n_patches_x * n_patches_y, batch_size]
|
||||
ggml_tensor * deepstack_features = nullptr;
|
||||
const int merge_factor = hparams.n_merge > 0 ? hparams.n_merge * hparams.n_merge : 4; // default 2x2=4 for qwen3vl
|
||||
|
||||
// loop over layers
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
auto & layer = model.layers[il];
|
||||
|
||||
ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
|
||||
|
||||
// layernorm1
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
|
||||
cb(cur, "ln1", il);
|
||||
|
||||
// self-attention
|
||||
{
|
||||
cur = build_mm(layer.qkv_w, cur);
|
||||
cur = ggml_add(ctx0, cur, layer.qkv_b);
|
||||
|
||||
ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos,
|
||||
/* nb1 */ ggml_row_size(cur->type, d_head),
|
||||
/* nb2 */ cur->nb[1],
|
||||
/* offset */ 0);
|
||||
|
||||
ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos,
|
||||
/* nb1 */ ggml_row_size(cur->type, d_head),
|
||||
/* nb2 */ cur->nb[1],
|
||||
/* offset */ ggml_row_size(cur->type, n_embd));
|
||||
|
||||
ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos,
|
||||
/* nb1 */ ggml_row_size(cur->type, d_head),
|
||||
/* nb2 */ cur->nb[1],
|
||||
/* offset */ ggml_row_size(cur->type, 2 * n_embd));
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
// apply M-RoPE
|
||||
Qcur = ggml_rope_multi(
|
||||
ctx0, Qcur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
|
||||
Kcur = ggml_rope_multi(
|
||||
ctx0, Kcur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
|
||||
|
||||
cb(Qcur, "Qcur_rope", il);
|
||||
cb(Kcur, "Kcur_rope", il);
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b,
|
||||
Qcur, Kcur, Vcur, nullptr, kq_scale, il);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
|
||||
// re-add the layer input, e.g., residual
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
|
||||
inpL = cur; // inpL = residual, cur = hidden_states
|
||||
|
||||
cb(cur, "ffn_inp", il);
|
||||
|
||||
// layernorm2
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
|
||||
cb(cur, "ffn_inp_normed", il);
|
||||
|
||||
// ffn
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
layer.ff_gate_w, layer.ff_gate_b,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
// residual 2
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
cb(cur, "layer_out", il);
|
||||
|
||||
if (layer.has_deepstack()) {
|
||||
ggml_tensor * feat = ggml_reshape_3d(ctx0, cur, n_embd * merge_factor, n_pos / merge_factor, batch_size);
|
||||
feat = build_norm(feat, layer.deepstack_norm_w, layer.deepstack_norm_b, norm_t, eps, il);
|
||||
feat = build_ffn(feat,
|
||||
layer.deepstack_fc1_w, layer.deepstack_fc1_b,
|
||||
nullptr, nullptr,
|
||||
layer.deepstack_fc2_w, layer.deepstack_fc2_b,
|
||||
ffn_op_type::FFN_GELU, il);
|
||||
|
||||
if(!deepstack_features) {
|
||||
deepstack_features = feat;
|
||||
} else {
|
||||
// concat along the feature dimension
|
||||
deepstack_features = ggml_concat(ctx0, deepstack_features, feat, 0);
|
||||
}
|
||||
}
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
// post-layernorm
|
||||
if (model.post_ln_w) {
|
||||
inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, n_layer);
|
||||
}
|
||||
|
||||
// multimodal projection
|
||||
ggml_tensor * embeddings = inpL;
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
|
||||
|
||||
embeddings = build_ffn(embeddings,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
ffn_op_type::FFN_GELU, -1);
|
||||
|
||||
if (deepstack_features) {
|
||||
embeddings = ggml_concat(ctx0, embeddings, deepstack_features, 0);
|
||||
} // concat along the feature dimension
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_sensenova_u1::build() {
|
||||
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
|
||||
GGML_ASSERT(model.patch_bias != nullptr);
|
||||
GGML_ASSERT(model.mm_0_w != nullptr);
|
||||
GGML_ASSERT(model.mm_0_b != nullptr);
|
||||
GGML_ASSERT(hparams.n_merge == 2);
|
||||
|
||||
ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_x, "pos_x");
|
||||
ggml_set_input(pos_x);
|
||||
|
||||
ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_y, "pos_y");
|
||||
ggml_set_input(pos_y);
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
inp = ggml_gelu(ctx0, inp);
|
||||
|
||||
// Apply adjacent-pair 2D RoPE to patch embeddings before downsampling.
|
||||
inp = ggml_reshape_4d(ctx0, inp, n_embd, 1, n_patches, n_batch);
|
||||
inp = build_rope_2d(ctx0, inp, pos_x, pos_y, hparams.rope_theta, false);
|
||||
inp = ggml_reshape_3d(ctx0, inp, n_embd, n_patches, n_batch);
|
||||
cb(inp, "patch_embd_rope", -1);
|
||||
|
||||
inp = ggml_reshape_4d(ctx0, inp, n_embd, n_patches_x, n_patches_y, n_batch);
|
||||
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 2, 0, 1, 3));
|
||||
inp = ggml_conv_2d(ctx0, model.mm_0_w, inp,
|
||||
hparams.n_merge, hparams.n_merge, 0, 0, 1, 1);
|
||||
|
||||
const int n_output_patches = n_patches / (hparams.n_merge * hparams.n_merge);
|
||||
inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 2, 0, 3));
|
||||
inp = ggml_reshape_3d(ctx0, inp, n_mmproj_embd, n_output_patches, n_batch);
|
||||
inp = ggml_add(ctx0, inp, model.mm_0_b);
|
||||
cb(inp, "proj_out", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, inp);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_siglip::build() {
|
||||
ggml_tensor * inp = build_inp();
|
||||
|
||||
ggml_tensor * learned_pos_embd = model.position_embeddings;
|
||||
if (proj_type == PROJECTOR_TYPE_LFM2 || proj_type == PROJECTOR_TYPE_D1OMNI_V || proj_type == PROJECTOR_TYPE_PHI4) {
|
||||
learned_pos_embd = resize_position_embeddings();
|
||||
}
|
||||
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_patches,
|
||||
NORM_TYPE_NORMAL,
|
||||
hparams.ffn_op,
|
||||
learned_pos_embd,
|
||||
nullptr);
|
||||
|
||||
if (proj_type == PROJECTOR_TYPE_GEMMA3) {
|
||||
const int batch_size = 1;
|
||||
GGML_ASSERT(n_patches_x == n_patches_y);
|
||||
const int patches_per_image = n_patches_x;
|
||||
const int kernel_size = hparams.n_merge;
|
||||
|
||||
cur = ggml_transpose(ctx0, cur);
|
||||
cur = ggml_cont_4d(ctx0, cur, patches_per_image, patches_per_image, n_embd, batch_size);
|
||||
|
||||
// doing a pool2d to reduce the number of output tokens
|
||||
cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG, kernel_size, kernel_size, kernel_size, kernel_size, 0, 0);
|
||||
cur = ggml_reshape_3d(ctx0, cur, cur->ne[0] * cur->ne[0], n_embd, batch_size);
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
|
||||
// apply norm before projection
|
||||
cur = ggml_rms_norm(ctx0, cur, eps);
|
||||
cur = ggml_mul(ctx0, cur, model.mm_soft_emb_norm_w);
|
||||
|
||||
// apply projection
|
||||
cur = ggml_mul_mat(ctx0,
|
||||
ggml_cont(ctx0, ggml_transpose(ctx0, model.mm_input_proj_w)),
|
||||
cur);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_IDEFICS3) {
|
||||
// pixel_shuffle
|
||||
// https://github.com/huggingface/transformers/blob/0a950e0bbe1ed58d5401a6b547af19f15f0c195e/src/transformers/models/idefics3/modeling_idefics3.py#L578
|
||||
const int scale_factor = model.hparams.n_merge;
|
||||
cur = build_patch_merge_permute(cur, scale_factor);
|
||||
cur = build_mm(model.mm_fc_w, cur);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_COHERE2V) {
|
||||
// tiles are square, so the pixel shuffle is the same as Idefics3
|
||||
cur = build_patch_merge_permute(cur, model.hparams.n_merge);
|
||||
cur = build_mm(model.mm_1_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_1_b);
|
||||
// linear_1 output is [x, gate], HF computes silu(gate) * x
|
||||
cur = ggml_swiglu_swapped(ctx0, cur);
|
||||
cur = build_mm(model.mm_2_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_2_b);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_LFM2 || proj_type == PROJECTOR_TYPE_D1OMNI_V) {
|
||||
// pixel unshuffle block
|
||||
const int scale_factor = model.hparams.n_merge;
|
||||
cur = build_patch_merge_permute(cur, scale_factor);
|
||||
|
||||
// projection, in LFM2-VL input norm is optional
|
||||
if (model.mm_input_norm_w) {
|
||||
cur = ggml_norm(ctx0, cur, 1e-5); // default nn.LayerNorm
|
||||
cur = ggml_mul(ctx0, cur, model.mm_input_norm_w);
|
||||
}
|
||||
|
||||
if (model.mm_input_norm_b) {
|
||||
cur = ggml_add(ctx0, cur, model.mm_input_norm_b);
|
||||
}
|
||||
|
||||
// d1-omni uses the exact gelu in the projector
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
proj_type == PROJECTOR_TYPE_D1OMNI_V ? FFN_GELU_ERF : FFN_GELU,
|
||||
-1);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_JANUS_PRO) {
|
||||
cur = build_ffn(cur,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
hparams.ffn_op,
|
||||
-1);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_PHI4) {
|
||||
cur = build_ffn(cur,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU,
|
||||
-1);
|
||||
|
||||
} else {
|
||||
GGML_ABORT("SigLIP: Unsupported projector type");
|
||||
}
|
||||
|
||||
// build the graph
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
#include "models.h"
|
||||
#include "../mtmd-audio.h"
|
||||
|
||||
// Soprano Vocos: normalized Qwen3 hidden states -> STFT coefficients.
|
||||
ggml_cgraph * clip_graph_soprano::build() {
|
||||
const auto & v = model.vocos;
|
||||
const int n_inp = hparams.projection_dim;
|
||||
const int n_up = upscale * (n_frames - 1) + 1;
|
||||
ggml_tensor * cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_inp, n_frames);
|
||||
ggml_set_name(cur, "inp_feats");
|
||||
ggml_set_input(cur);
|
||||
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = ggml_interpolate(ctx0, cur, n_up, n_inp, 1, 1, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS);
|
||||
if (ggml_n_dims(v.input_w) == 3) {
|
||||
const int k = v.input_w->ne[0];
|
||||
cur = ggml_conv_1d(ctx0, v.input_w, cur, 1, k / 2, 1);
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
} else {
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = build_mm(v.input_w, cur);
|
||||
}
|
||||
cur = ggml_add(ctx0, cur, v.input_b);
|
||||
cur = build_norm(cur, v.norm_w, v.norm_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
|
||||
for (size_t il = 0; il < v.blocks.size(); ++il) {
|
||||
const auto & b = v.blocks[il];
|
||||
const int k = b.dwconv_w->ne[0];
|
||||
ggml_tensor * residual = cur;
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = ggml_conv_1d_dw(ctx0, b.dwconv_w, cur, 1, k / 2, 1);
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = ggml_add(ctx0, cur, b.dwconv_b);
|
||||
cur = build_norm(cur, b.norm_w, b.norm_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cur = ggml_add(ctx0, build_mm(b.pw1_w, cur), b.pw1_b);
|
||||
cur = ggml_gelu_erf(ctx0, cur);
|
||||
cur = ggml_add(ctx0, build_mm(b.pw2_w, cur), b.pw2_b);
|
||||
cur = ggml_add(ctx0, residual, ggml_mul(ctx0, cur, b.gamma));
|
||||
cb(cur, "vocos_block", il);
|
||||
}
|
||||
cur = build_norm(cur, v.output_norm_w, v.output_norm_b, NORM_TYPE_NORMAL, eps, -1);
|
||||
cur = ggml_add(ctx0, build_mm(v.output_w, cur), v.output_b);
|
||||
cb(cur, "vocos_head", -1);
|
||||
|
||||
// the head holds n_fft/2+1 log-magnitudes followed by n_fft/2+1 phases
|
||||
const int n_bins = v.output_w->ne[1] / 2;
|
||||
ggml_tensor * mag = ggml_view_2d(ctx0, cur, n_bins, n_up, cur->nb[1], 0);
|
||||
ggml_tensor * phase = ggml_view_2d(ctx0, cur, n_bins, n_up, cur->nb[1], n_bins * sizeof(float));
|
||||
mag = ggml_clamp(ctx0, ggml_exp(ctx0, ggml_cont(ctx0, mag)), 0.0f, 100.0f);
|
||||
phase = ggml_cont(ctx0, phase);
|
||||
ggml_tensor * re = ggml_mul(ctx0, mag, ggml_cos(ctx0, phase));
|
||||
ggml_tensor * im = ggml_mul(ctx0, mag, ggml_sin(ctx0, phase));
|
||||
re = ggml_reshape_3d(ctx0, re, 1, n_bins, n_up);
|
||||
im = ggml_reshape_3d(ctx0, im, 1, n_bins, n_up);
|
||||
cur = ggml_concat(ctx0, re, im, 0);
|
||||
ggml_set_name(cur, "out_audio");
|
||||
ggml_set_output(cur);
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
|
||||
void clip_graph_soprano::decode_spectrum(const clip_model & model, const std::vector<float> & spectrum, std::vector<float> & pcm) {
|
||||
const int n_frame_vals = model.vocos.output_w->ne[1];
|
||||
const int n_fft = n_frame_vals - 2;
|
||||
const int hop_length = model.hparams.audio_hop_len;
|
||||
const size_t n_frames = spectrum.size() / n_frame_vals;
|
||||
mtmd_audio_streaming_istft istft(n_fft, hop_length);
|
||||
pcm.clear();
|
||||
for (size_t i = 0; i < n_frames; ++i) {
|
||||
auto frame = istft.process_frame(spectrum.data() + i * n_frame_vals);
|
||||
pcm.insert(pcm.end(), frame.begin(), frame.end());
|
||||
}
|
||||
auto tail = istft.flush();
|
||||
pcm.insert(pcm.end(), tail.begin(), tail.end());
|
||||
// the shared ISTFT removes (n_fft - hop) / 2 samples, torch.istft(center=True) removes n_fft / 2
|
||||
pcm.erase(pcm.begin(), pcm.begin() + hop_length / 2);
|
||||
pcm.resize((n_frames - 1) * hop_length);
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_step3vl::build() {
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
GGML_ASSERT(model.patch_embeddings_0 != nullptr);
|
||||
GGML_ASSERT(model.position_embeddings != nullptr);
|
||||
|
||||
norm_type norm_t = NORM_TYPE_NORMAL;
|
||||
|
||||
ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_h, "pos_h");
|
||||
ggml_set_input(pos_h);
|
||||
|
||||
ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);
|
||||
ggml_set_name(pos_w, "pos_w");
|
||||
ggml_set_input(pos_w);
|
||||
|
||||
ggml_tensor * inp = build_inp();
|
||||
ggml_tensor * learned_pos_embd = resize_position_embeddings();
|
||||
|
||||
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
||||
return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);
|
||||
};
|
||||
|
||||
auto add_spatial_bias = [&](ggml_tensor * cur, ggml_tensor * bias) {
|
||||
if (bias == nullptr) {
|
||||
return cur;
|
||||
}
|
||||
|
||||
const int64_t width = cur->ne[0];
|
||||
const int64_t height = cur->ne[1];
|
||||
const int64_t channels = cur->ne[2];
|
||||
|
||||
cur = ggml_reshape_2d(ctx0, cur, width * height, channels);
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = ggml_add(ctx0, cur, bias);
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = ggml_reshape_3d(ctx0, cur, width, height, channels);
|
||||
|
||||
return cur;
|
||||
};
|
||||
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp,
|
||||
n_patches,
|
||||
norm_t,
|
||||
hparams.ffn_op,
|
||||
learned_pos_embd,
|
||||
add_pos);
|
||||
cb(cur, "vit_out", -1);
|
||||
|
||||
// [n_embd, n_patches] -> [w, h, n_embd] for spatial downsampling convolutions.
|
||||
cur = ggml_permute(ctx0, cur, 1, 0, 2, 3);
|
||||
cur = ggml_cont_3d(ctx0, cur, n_patches_x, n_patches_y, n_embd);
|
||||
|
||||
// First downsampler: Conv2d(1536 -> 3072, k=3, s=2, p=1)
|
||||
cur = ggml_conv_2d(ctx0, model.mm_0_w, cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = add_spatial_bias(cur, model.mm_0_b);
|
||||
cb(cur, "downsample_0", -1);
|
||||
|
||||
// Second downsampler: Conv2d(3072 -> 6144, k=3, s=2, p=1)
|
||||
cur = ggml_conv_2d(ctx0, model.mm_1_w, cur, 2, 2, 1, 1, 1, 1);
|
||||
cur = add_spatial_bias(cur, model.mm_1_b);
|
||||
cb(cur, "downsample_1", -1);
|
||||
|
||||
// [w, h, c] -> [c, w*h]
|
||||
{
|
||||
const int64_t w = cur->ne[0];
|
||||
const int64_t h = cur->ne[1];
|
||||
cur = ggml_reshape_3d(ctx0, cur, w * h, cur->ne[2], cur->ne[3]);
|
||||
cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 0, 2, 3));
|
||||
}
|
||||
cb(cur, "downsample_flatten", -1);
|
||||
|
||||
// Final projector: Linear(6144 -> projection_dim)
|
||||
cur = ggml_mul_mat(ctx0, model.mm_model_proj, cur);
|
||||
cb(cur, "projector_out", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
#include "models.h"
|
||||
|
||||
// the input holds 2 context mel frames followed by the new frames, so the causal conv stem
|
||||
// sees the same left context as on the full spectrogram
|
||||
ggml_cgraph * clip_graph_voxtral_rt::build() {
|
||||
const int n_frames_in = img.nx();
|
||||
const int n_new = (n_frames_in - 2) / 2; // encoder frames, the conv stem has stride 2
|
||||
const int n_window = hparams.attn_window_size;
|
||||
const int stack = hparams.proj_stack_factor;
|
||||
const int n_embd_kv = n_head_kv * d_head;
|
||||
const int n_ctx = state ? state->n_ctx : n_window + n_new;
|
||||
|
||||
GGML_ASSERT(n_window > 0 && stack > 0);
|
||||
GGML_ASSERT(n_new > 0 && n_new % stack == 0);
|
||||
GGML_ASSERT(n_new <= n_ctx - n_window);
|
||||
|
||||
ggml_tensor * inp = build_inp_raw(1); // [n_frames_in, n_mel]
|
||||
|
||||
ggml_tensor * stem_tail = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_embd);
|
||||
ggml_set_name(stem_tail, "inp_stem_tail");
|
||||
ggml_set_input(stem_tail);
|
||||
|
||||
ggml_tensor * pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_new);
|
||||
ggml_set_name(pos, "inp_pos");
|
||||
ggml_set_input(pos);
|
||||
|
||||
ggml_tensor * kv_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_new);
|
||||
ggml_set_name(kv_idx, "inp_kv_idx");
|
||||
ggml_set_input(kv_idx);
|
||||
|
||||
ggml_tensor * kq_mask = build_inp_attn_mask(n_ctx, n_new);
|
||||
|
||||
// causal conv stem
|
||||
ggml_tensor * cur = ggml_conv_1d(ctx0, model.conv1d_1_w, inp, 1, 0, 1);
|
||||
cur = ggml_add(ctx0, cur, model.conv1d_1_b);
|
||||
cur = ggml_gelu_erf(ctx0, cur); // [2*n_new, n_embd]
|
||||
|
||||
// the last frame is the left context of the second conv on the next call
|
||||
ggml_tensor * tail = ggml_view_2d(ctx0, cur, 1, n_embd, cur->nb[1], (2*n_new - 1) * ggml_element_size(cur));
|
||||
tail = ggml_cont(ctx0, tail);
|
||||
ggml_set_name(tail, "out_stem_tail");
|
||||
ggml_set_output(tail);
|
||||
ggml_build_forward_expand(gf, tail);
|
||||
|
||||
cur = ggml_concat(ctx0, stem_tail, cur, 0); // [2*n_new + 1, n_embd]
|
||||
cur = ggml_conv_1d(ctx0, model.conv1d_2_w, cur, 2, 0, 1);
|
||||
cur = ggml_add(ctx0, cur, model.conv1d_2_b);
|
||||
cur = ggml_gelu_erf(ctx0, cur); // [n_new, n_embd]
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [n_embd, n_new]
|
||||
cb(cur, "after_conv1d", -1);
|
||||
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
const auto & layer = model.layers[il];
|
||||
|
||||
ggml_tensor * x = build_norm(cur, layer.ln_1_w, nullptr, NORM_TYPE_RMS, eps, il);
|
||||
|
||||
ggml_tensor * q = ggml_add(ctx0, build_mm(layer.q_w, x), layer.q_b);
|
||||
ggml_tensor * k = build_mm(layer.k_w, x);
|
||||
ggml_tensor * v = ggml_add(ctx0, build_mm(layer.v_w, x), layer.v_b);
|
||||
|
||||
q = ggml_reshape_3d(ctx0, q, d_head, n_head, n_new);
|
||||
k = ggml_reshape_3d(ctx0, k, d_head, n_head_kv, n_new);
|
||||
|
||||
// interleaved rope on absolute frame positions
|
||||
q = ggml_rope_ext(ctx0, q, pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
k = ggml_rope_ext(ctx0, k, pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
cb(q, "q", il);
|
||||
cb(k, "k", il);
|
||||
cb(v, "v", il);
|
||||
|
||||
ggml_tensor * k_cache;
|
||||
ggml_tensor * v_cache;
|
||||
if (state) {
|
||||
k_cache = state->k[il];
|
||||
v_cache = state->v[il];
|
||||
} else {
|
||||
k_cache = ggml_new_tensor_2d(ctx0, GGML_TYPE_F16, n_embd_kv, n_ctx);
|
||||
v_cache = ggml_new_tensor_2d(ctx0, GGML_TYPE_F16, n_embd_kv, n_ctx);
|
||||
ggml_set_input(k_cache);
|
||||
ggml_set_input(v_cache);
|
||||
}
|
||||
|
||||
// write the new frames into the ring, the mask selects the window out of the whole ring
|
||||
ggml_build_forward_expand(gf, ggml_set_rows(ctx0, k_cache, ggml_reshape_2d(ctx0, k, n_embd_kv, n_new), kv_idx));
|
||||
ggml_build_forward_expand(gf, ggml_set_rows(ctx0, v_cache, v, kv_idx));
|
||||
|
||||
ggml_tensor * k_all = ggml_reshape_3d(ctx0, k_cache, d_head, n_head_kv, n_ctx);
|
||||
ggml_tensor * v_all = ggml_reshape_3d(ctx0, v_cache, d_head, n_head_kv, n_ctx);
|
||||
|
||||
x = build_attn(layer.o_w, layer.o_b, q, k_all, v_all, kq_mask, kq_scale, il);
|
||||
cur = ggml_add(ctx0, cur, x);
|
||||
|
||||
x = build_norm(cur, layer.ln_2_w, nullptr, NORM_TYPE_RMS, eps, il);
|
||||
x = build_ffn(x, layer.ff_up_w, nullptr, layer.ff_gate_w, nullptr, layer.ff_down_w, layer.ff_down_b, hparams.ffn_op, il);
|
||||
cur = ggml_add(ctx0, cur, x);
|
||||
cb(cur, "layer_out", il);
|
||||
}
|
||||
|
||||
cur = build_norm(cur, model.post_ln_w, nullptr, NORM_TYPE_RMS, eps, -1);
|
||||
cb(cur, "after_transformer", -1);
|
||||
|
||||
cur = build_stack(cur, stack, n_embd);
|
||||
cur = build_mm(model.mm_1_w, cur);
|
||||
cur = ggml_gelu_erf(ctx0, cur);
|
||||
cur = build_mm(model.mm_2_w, cur);
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_whisper_enc::build() {
|
||||
const int n_frames = img.nx();
|
||||
const int n_pos = n_frames / 2;
|
||||
GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos);
|
||||
|
||||
ggml_tensor * inp = build_inp_raw(1);
|
||||
|
||||
// conv1d block
|
||||
{
|
||||
// convolution + gelu
|
||||
ggml_tensor * cur = ggml_conv_1d_ph(ctx0, model.conv1d_1_w, inp, 1, 1);
|
||||
cur = ggml_add(ctx0, cur, model.conv1d_1_b);
|
||||
|
||||
cur = ggml_gelu_erf(ctx0, cur);
|
||||
|
||||
cur = ggml_conv_1d_ph(ctx0, model.conv1d_2_w, cur, 2, 1);
|
||||
cur = ggml_add(ctx0, cur, model.conv1d_2_b);
|
||||
|
||||
cur = ggml_gelu_erf(ctx0, cur);
|
||||
// transpose
|
||||
inp = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cb(inp, "after_conv1d", -1);
|
||||
}
|
||||
|
||||
// sanity check (only check one layer, but it should be the same for all)
|
||||
GGML_ASSERT(model.layers[0].ln_1_w && model.layers[0].ln_1_b);
|
||||
GGML_ASSERT(model.layers[0].ln_2_w && model.layers[0].ln_2_b);
|
||||
GGML_ASSERT(model.layers[0].q_b);
|
||||
GGML_ASSERT(model.layers[0].v_b);
|
||||
GGML_ASSERT(!model.layers[0].k_b); // no bias for k
|
||||
|
||||
ggml_tensor * pos_embd_selected = ggml_view_2d(
|
||||
ctx0, model.position_embeddings,
|
||||
model.position_embeddings->ne[0], n_pos,
|
||||
model.position_embeddings->nb[1], 0
|
||||
);
|
||||
ggml_tensor * cur = build_vit(
|
||||
inp, n_pos,
|
||||
NORM_TYPE_NORMAL,
|
||||
hparams.ffn_op,
|
||||
pos_embd_selected,
|
||||
nullptr);
|
||||
|
||||
cb(cur, "after_transformer", -1);
|
||||
|
||||
if (model.audio_has_stack_frames()) {
|
||||
// StackAudioFrames
|
||||
// https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_2-1b/blob/main/ultravox_model.py
|
||||
cur = build_stack(cur, hparams.proj_stack_factor, n_embd);
|
||||
cb(cur, "after_stacked", -1);
|
||||
}
|
||||
|
||||
if (proj_type == PROJECTOR_TYPE_ULTRAVOX) {
|
||||
// UltravoxProjector
|
||||
// pre-norm
|
||||
cur = ggml_rms_norm(ctx0, cur, 1e-6);
|
||||
cur = ggml_mul(ctx0, cur, model.mm_norm_pre_w);
|
||||
|
||||
// ffn in
|
||||
cur = build_mm(model.mm_1_w, cur);
|
||||
|
||||
// swiglu
|
||||
// see SwiGLU in ultravox_model.py, the second half passed through is silu, not the first half
|
||||
cur = ggml_swiglu_swapped(ctx0, cur);
|
||||
|
||||
// mid-norm
|
||||
cur = ggml_rms_norm(ctx0, cur, 1e-6);
|
||||
cur = ggml_mul(ctx0, cur, model.mm_norm_mid_w);
|
||||
|
||||
// ffn out
|
||||
cur = build_mm(model.mm_2_w, cur);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_QWEN2A) {
|
||||
// projector
|
||||
cur = build_mm(model.mm_fc_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_fc_b);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_VOXTRAL) {
|
||||
// projector
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU_ERF,
|
||||
-1);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_MUSIC_FLAMINGO) {
|
||||
// projector
|
||||
cur = build_ffn(cur,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU_ERF,
|
||||
-1);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_MERALION) {
|
||||
// stack (above) -> ln -> linear0+silu -> GLU -> out
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_mul(ctx0, cur, model.mm_norm_pre_w);
|
||||
cur = ggml_add(ctx0, cur, model.mm_norm_pre_b);
|
||||
|
||||
cur = ggml_mul_mat(ctx0, model.mm_0_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_0_b);
|
||||
cur = ggml_silu(ctx0, cur);
|
||||
|
||||
ggml_tensor * gate = ggml_mul_mat(ctx0, model.mm_1_w, cur);
|
||||
gate = ggml_add(ctx0, gate, model.mm_1_b);
|
||||
gate = ggml_silu(ctx0, gate);
|
||||
|
||||
ggml_tensor * pool = ggml_mul_mat(ctx0, model.mm_2_w, cur);
|
||||
pool = ggml_add(ctx0, pool, model.mm_2_b);
|
||||
|
||||
cur = ggml_mul(ctx0, gate, pool);
|
||||
|
||||
cur = ggml_mul_mat(ctx0, model.mm_3_w, cur);
|
||||
cur = ggml_add(ctx0, cur, model.mm_3_b);
|
||||
|
||||
} else if (proj_type == PROJECTOR_TYPE_GLMA) {
|
||||
cur = ggml_norm(ctx0, cur, hparams.eps);
|
||||
cur = ggml_mul(ctx0, cur, model.mm_norm_pre_w);
|
||||
cur = ggml_add(ctx0, cur, model.mm_norm_pre_b);
|
||||
cur = build_stack(cur, hparams.proj_stack_factor, n_embd);
|
||||
cur = build_ffn(cur, model.mm_1_w, model.mm_1_b, nullptr, nullptr, model.mm_2_w, model.mm_2_b, hparams.ffn_op, 0);
|
||||
cur = ggml_concat(ctx0, model.mm_boi, cur, 1);
|
||||
cur = ggml_concat(ctx0, cur, model.mm_eoi, 1);
|
||||
} else {
|
||||
GGML_ABORT("%s: unknown projector type", __func__);
|
||||
}
|
||||
|
||||
cb(cur, "projected", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
// ABOUTME: Yasa2 vision encoder graph builder for ConvNeXt-based architecture.
|
||||
// ABOUTME: Implements patch embedding, ConvNeXt stages with GRN, and adaptive pooling.
|
||||
|
||||
#include "models.h"
|
||||
|
||||
static ggml_tensor * add_channel_bias(
|
||||
ggml_context * ctx0,
|
||||
ggml_tensor * x_whcb,
|
||||
ggml_tensor * b_c) {
|
||||
if (!b_c) {
|
||||
return x_whcb;
|
||||
}
|
||||
ggml_tensor * b4 = ggml_reshape_4d(ctx0, b_c, 1, 1, b_c->ne[0], 1);
|
||||
return ggml_add(ctx0, x_whcb, b4);
|
||||
}
|
||||
|
||||
static ggml_tensor * mul_channel_weight(
|
||||
ggml_context * ctx0,
|
||||
ggml_tensor * x_whcb,
|
||||
ggml_tensor * w_c) {
|
||||
if (!w_c) {
|
||||
return x_whcb;
|
||||
}
|
||||
ggml_tensor * w4 = ggml_reshape_4d(ctx0, w_c, 1, 1, w_c->ne[0], 1);
|
||||
return ggml_mul(ctx0, x_whcb, w4);
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_yasa2::layer_norm_channels(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b, float eps) {
|
||||
// Match HF ConvNextLayerNorm(channels_first):
|
||||
// u = mean_c(x), s = mean_c((x-u)^2), x = (x-u)/sqrt(s+eps)
|
||||
// cast back to input dtype before affine.
|
||||
ggml_tensor * cur = ggml_permute(ctx0, inp, 2, 1, 0, 3); // [W,H,C,B] -> [C,H,W,B]
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
|
||||
ggml_tensor * u = ggml_mean(ctx0, cur); // [1,H,W,B]
|
||||
ggml_tensor * xm = ggml_sub(ctx0, cur, u); // [C,H,W,B]
|
||||
|
||||
ggml_tensor * s = ggml_mul(ctx0, xm, xm); // [C,H,W,B]
|
||||
s = ggml_mean(ctx0, s); // [1,H,W,B]
|
||||
s = ggml_clamp(ctx0, s, eps, 1e30f); // avoid div-by-zero in no-alloc warmup
|
||||
s = ggml_sqrt(ctx0, s); // [1,H,W,B]
|
||||
|
||||
ggml_tensor * xhat = ggml_div(ctx0, xm, s); // [C,H,W,B]
|
||||
xhat = ggml_permute(ctx0, xhat, 2, 1, 0, 3); // [W,H,C,B]
|
||||
xhat = ggml_cont(ctx0, xhat);
|
||||
xhat = mul_channel_weight(ctx0, xhat, w);
|
||||
xhat = add_channel_bias(ctx0, xhat, b);
|
||||
return xhat;
|
||||
}
|
||||
|
||||
ggml_tensor * clip_graph_yasa2::convnext_grn(ggml_tensor * inp, ggml_tensor * w, ggml_tensor * b) {
|
||||
// Exact ConvNeXtV2 GRN:
|
||||
// Gx = ||x||_2 over spatial dims (W,H), Nx = Gx / (mean_c(Gx) + eps)
|
||||
// y = w * (x * Nx) + b + x
|
||||
const int64_t wdim = inp->ne[0];
|
||||
const int64_t hdim = inp->ne[1];
|
||||
const int64_t cdim = inp->ne[2];
|
||||
const int64_t bdim = inp->ne[3];
|
||||
|
||||
// Keep GRN math in fp32 for stability; fp16/bf16 accumulation can drift.
|
||||
ggml_tensor * sq = ggml_mul(ctx0, inp, inp);
|
||||
ggml_tensor * sq_flat = ggml_reshape_4d(ctx0, sq, wdim * hdim, cdim, 1, bdim); // [WH,C,1,B]
|
||||
ggml_tensor * gx = ggml_sum_rows(ctx0, sq_flat); // [1,C,1,B]
|
||||
gx = ggml_sqrt(ctx0, gx); // [1,C,1,B]
|
||||
|
||||
ggml_tensor * gx_ch_first = ggml_permute(ctx0, gx, 1, 0, 2, 3); // [C,1,1,B]
|
||||
gx_ch_first = ggml_cont(ctx0, gx_ch_first);
|
||||
ggml_tensor * gx_mean = ggml_mean(ctx0, gx_ch_first); // [1,1,1,B]
|
||||
|
||||
gx_mean = ggml_clamp(ctx0, gx_mean, 1e-6f, 1e30f); // approx +eps, warmup-safe
|
||||
ggml_tensor * nx = ggml_div(ctx0, gx, gx_mean); // [1,C,1,B]
|
||||
nx = ggml_permute(ctx0, nx, 0, 2, 1, 3); // [1,1,C,B]
|
||||
nx = ggml_cont(ctx0, nx);
|
||||
|
||||
ggml_tensor * xnx = ggml_mul(ctx0, inp, nx);
|
||||
xnx = mul_channel_weight(ctx0, xnx, w);
|
||||
xnx = add_channel_bias(ctx0, xnx, b);
|
||||
return ggml_add(ctx0, inp, xnx);
|
||||
}
|
||||
|
||||
ggml_cgraph * clip_graph_yasa2::build() {
|
||||
ggml_tensor * cur = build_inp_raw();
|
||||
|
||||
// Patch embedding Conv2d(kernel=4, stride=4)
|
||||
cur = ggml_conv_2d(ctx0, model.yasa_patch_w, cur, patch_size, patch_size, 0, 0, 1, 1);
|
||||
cur = add_channel_bias(ctx0, cur, model.yasa_patch_b);
|
||||
ggml_set_name(cur, "yasa2_patch_conv_out");
|
||||
cb(cur, "yasa2_patch_conv_out", -1);
|
||||
cur = layer_norm_channels(cur, model.yasa_patch_ln_w, model.yasa_patch_ln_b, eps);
|
||||
ggml_set_name(cur, "yasa2_patch_ln_out");
|
||||
cb(cur, "yasa2_patch_ln_out", -1);
|
||||
|
||||
// ConvNeXt stages
|
||||
for (size_t s = 0; s < model.yasa_stages.size(); ++s) {
|
||||
const auto & stage = model.yasa_stages[s];
|
||||
|
||||
if (stage.down_conv_w) {
|
||||
cur = layer_norm_channels(cur, stage.down_ln_w, stage.down_ln_b, eps);
|
||||
cur = ggml_conv_2d(ctx0, stage.down_conv_w, cur, 2, 2, 0, 0, 1, 1);
|
||||
cur = add_channel_bias(ctx0, cur, stage.down_conv_b);
|
||||
ggml_format_name(cur, "yasa2_stage%zu_down_out", s);
|
||||
}
|
||||
|
||||
for (size_t bi = 0; bi < stage.blocks.size(); ++bi) {
|
||||
const auto & blk = stage.blocks[bi];
|
||||
ggml_tensor * res = cur;
|
||||
|
||||
ggml_tensor * x = ggml_conv_2d_dw(ctx0, blk.dw_w, cur, 1, 1, 3, 3, 1, 1);
|
||||
x = add_channel_bias(ctx0, x, blk.dw_b);
|
||||
x = layer_norm_channels(x, blk.ln_w, blk.ln_b, eps);
|
||||
|
||||
// pwconv1/pwconv2 are HF Linear layers over channels; implement via matmul on tokens.
|
||||
const int64_t w = x->ne[0];
|
||||
const int64_t h = x->ne[1];
|
||||
const int64_t b = x->ne[3];
|
||||
|
||||
ggml_tensor * tok = ggml_reshape_3d(ctx0, x, w * h, x->ne[2], b); // [T,C,B]
|
||||
tok = ggml_permute(ctx0, tok, 1, 0, 2, 3); // [C,T,B]
|
||||
tok = ggml_cont(ctx0, tok);
|
||||
|
||||
tok = ggml_mul_mat(ctx0, blk.pw1_w, tok); // [4C,T,B]
|
||||
if (blk.pw1_b) {
|
||||
ggml_tensor * b1 = ggml_reshape_3d(ctx0, blk.pw1_b, blk.pw1_b->ne[0], 1, 1); // [4C,1,1]
|
||||
tok = ggml_add(ctx0, tok, b1);
|
||||
}
|
||||
x = ggml_permute(ctx0, tok, 1, 0, 2, 3); // [T,4C,B]
|
||||
x = ggml_cont(ctx0, x);
|
||||
x = ggml_reshape_4d(ctx0, x, w, h, tok->ne[0], b); // [W,H,4C,B]
|
||||
x = ggml_gelu_erf(ctx0, x);
|
||||
x = convnext_grn(x, blk.grn_w, blk.grn_b);
|
||||
|
||||
tok = ggml_reshape_3d(ctx0, x, w * h, x->ne[2], b); // [T,4C,B]
|
||||
tok = ggml_permute(ctx0, tok, 1, 0, 2, 3); // [4C,T,B]
|
||||
tok = ggml_cont(ctx0, tok);
|
||||
|
||||
tok = ggml_mul_mat(ctx0, blk.pw2_w, tok); // [C,T,B]
|
||||
if (blk.pw2_b) {
|
||||
ggml_tensor * b2 = ggml_reshape_3d(ctx0, blk.pw2_b, blk.pw2_b->ne[0], 1, 1); // [C,1,1]
|
||||
tok = ggml_add(ctx0, tok, b2);
|
||||
}
|
||||
x = ggml_permute(ctx0, tok, 1, 0, 2, 3); // [T,C,B]
|
||||
x = ggml_cont(ctx0, x);
|
||||
x = ggml_reshape_4d(ctx0, x, w, h, tok->ne[0], b); // [W,H,C,B]
|
||||
|
||||
cur = ggml_add(ctx0, res, x);
|
||||
ggml_format_name(cur, "yasa2_stage%zu_blk%zu_out", s, bi);
|
||||
}
|
||||
}
|
||||
|
||||
// HF path adds vision position embeddings BEFORE adaptive pooling.
|
||||
const int64_t pre_w = cur->ne[0];
|
||||
const int64_t pre_h = cur->ne[1];
|
||||
ggml_tensor * tokens_pre = ggml_reshape_3d(ctx0, cur, pre_w * pre_h, cur->ne[2], cur->ne[3]); // [T,C,B]
|
||||
tokens_pre = ggml_permute(ctx0, tokens_pre, 1, 0, 2, 3); // [C,T,B]
|
||||
tokens_pre = ggml_cont(ctx0, tokens_pre);
|
||||
if (model.yasa_vision_pos_embed && tokens_pre->ne[1] == model.yasa_vision_pos_embed->ne[1]) {
|
||||
const int64_t n_ch = model.yasa_vision_pos_embed->ne[0];
|
||||
const int64_t n_tokens = model.yasa_vision_pos_embed->ne[1];
|
||||
ggml_tensor * pos = ggml_reshape_3d(ctx0, model.yasa_vision_pos_embed, (int) n_ch, (int) n_tokens, 1);
|
||||
tokens_pre = ggml_add(ctx0, tokens_pre, pos);
|
||||
}
|
||||
cur = ggml_permute(ctx0, tokens_pre, 1, 0, 2, 3); // [T,C,B]
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
cur = ggml_reshape_4d(ctx0, cur, pre_w, pre_h, cur->ne[1], cur->ne[2]); // [W,H,C,B]
|
||||
|
||||
// AdaptiveAvgPool2d target is 8x8 for real inputs, but warmup can use tiny images.
|
||||
const int pooled_w = std::min(8, (int) cur->ne[0]);
|
||||
const int pooled_h = std::min(8, (int) cur->ne[1]);
|
||||
const int kw = std::max(1, (int) cur->ne[0] / pooled_w);
|
||||
const int kh = std::max(1, (int) cur->ne[1] / pooled_h);
|
||||
cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG, kw, kh, kw, kh, 0, 0);
|
||||
|
||||
// [W,H,C,B] -> [C,T,B]
|
||||
ggml_tensor * tokens = ggml_reshape_3d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2], cur->ne[3]);
|
||||
tokens = ggml_permute(ctx0, tokens, 1, 0, 2, 3);
|
||||
tokens = ggml_cont(ctx0, tokens);
|
||||
cb(tokens, "yasa2_tokens", -1);
|
||||
|
||||
GGML_ASSERT(model.mm_0_w && model.mm_2_w);
|
||||
ggml_tensor * embeddings = build_ffn(
|
||||
tokens,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_2_w, model.mm_2_b,
|
||||
FFN_GELU_ERF,
|
||||
-1);
|
||||
cb(embeddings, "yasa2_emb", -1);
|
||||
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_cgraph * clip_graph_youtuvl::build() {
|
||||
GGML_ASSERT(model.class_embedding == nullptr);
|
||||
const int batch_size = 1;
|
||||
const bool use_window_attn = !hparams.wa_layer_indexes.empty();
|
||||
const int n_pos = n_patches;
|
||||
const int num_position_ids = n_pos * 4;
|
||||
const int m = 2;
|
||||
const int Wp = n_patches_x;
|
||||
const int Hp = n_patches_y;
|
||||
const int Hm = Hp / m;
|
||||
const int Wm = Wp / m;
|
||||
norm_type norm_t = NORM_TYPE_NORMAL;
|
||||
|
||||
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
||||
|
||||
ggml_tensor * inp = build_inp_raw();
|
||||
|
||||
// change conv3d to linear
|
||||
// reshape and permute to get patches, permute from (patch_size, m, Wm, patch_size, m, Hm, C) to (C, patch_size, patch_size, m, m, Wm, Hm)
|
||||
{
|
||||
inp = ggml_reshape_4d(
|
||||
ctx0, inp,
|
||||
Wm * m * patch_size, m * patch_size, Hm, 3);
|
||||
inp = ggml_permute(ctx0, inp, 1, 2, 3, 0);
|
||||
inp = ggml_cont_4d(
|
||||
ctx0, inp,
|
||||
m * patch_size * 3, Wm, m * patch_size, Hm);
|
||||
|
||||
inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
|
||||
inp = ggml_cont_4d(
|
||||
ctx0, inp,
|
||||
m * patch_size * 3, patch_size, m, Hm * Wm);
|
||||
|
||||
inp = ggml_permute(ctx0, inp, 1, 0, 2, 3);
|
||||
inp = ggml_cont_4d(
|
||||
ctx0, inp,
|
||||
patch_size, 3, patch_size, Hm * Wm * m * m);
|
||||
|
||||
inp = ggml_permute(ctx0, inp, 2, 0, 1, 3);
|
||||
inp = ggml_cont_3d(
|
||||
ctx0, inp,
|
||||
3*patch_size* patch_size, Hm * Wm * m * m, 1);
|
||||
}
|
||||
inp = build_mm(model.patch_embeddings_0, inp);
|
||||
|
||||
if (model.patch_bias) {
|
||||
inp = ggml_add(ctx0, inp, model.patch_bias);
|
||||
}
|
||||
|
||||
inp = ggml_reshape_2d(ctx0, inp, n_embd, n_patches);
|
||||
|
||||
ggml_tensor * inpL = inp;
|
||||
ggml_tensor * window_mask = nullptr;
|
||||
ggml_tensor * window_idx = nullptr;
|
||||
ggml_tensor * inv_window_idx = nullptr;
|
||||
|
||||
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
|
||||
ggml_set_name(positions, "positions");
|
||||
ggml_set_input(positions);
|
||||
|
||||
// pre-layernorm
|
||||
if (model.pre_ln_w) {
|
||||
inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1);
|
||||
}
|
||||
if (use_window_attn) {
|
||||
inv_window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);
|
||||
ggml_set_name(inv_window_idx, "inv_window_idx");
|
||||
ggml_set_input(inv_window_idx);
|
||||
// mask for window attention
|
||||
window_mask = build_inp_attn_mask(n_pos, n_pos);
|
||||
|
||||
// inpL shape: [n_embd, n_patches_x * n_patches_y, batch_size]
|
||||
GGML_ASSERT(batch_size == 1);
|
||||
inpL = ggml_reshape_2d(ctx0, inpL, n_embd * 4, n_patches_x * n_patches_y * batch_size / 4);
|
||||
inpL = ggml_get_rows(ctx0, inpL, inv_window_idx);
|
||||
inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_patches_x * n_patches_y, batch_size);
|
||||
}
|
||||
|
||||
// loop over layers
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
const auto & layer = model.layers[il];
|
||||
const bool full_attn = use_window_attn ? hparams.wa_layer_indexes.count(il) > 0 : true;
|
||||
|
||||
ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
|
||||
|
||||
// layernorm1
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
|
||||
// self-attention
|
||||
{
|
||||
ggml_tensor * Qcur = ggml_add(ctx0,
|
||||
build_mm(layer.q_w, cur), layer.q_b);
|
||||
ggml_tensor * Kcur = ggml_add(ctx0,
|
||||
build_mm(layer.k_w, cur), layer.k_b);
|
||||
ggml_tensor * Vcur = ggml_add(ctx0,
|
||||
build_mm(layer.v_w, cur), layer.v_b);
|
||||
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_patches);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_patches);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_patches);
|
||||
|
||||
Qcur = ggml_rope_multi(
|
||||
ctx0, Qcur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
|
||||
Kcur = ggml_rope_multi(
|
||||
ctx0, Kcur, positions, nullptr,
|
||||
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
|
||||
|
||||
ggml_tensor * attn_mask = full_attn ? nullptr : window_mask;
|
||||
|
||||
cur = build_attn(layer.o_w, layer.o_b,
|
||||
Qcur, Kcur, Vcur, attn_mask, kq_scale, il);
|
||||
}
|
||||
// re-add the layer input, e.g., residual
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
|
||||
inpL = cur; // inpL = residual, cur = hidden_states
|
||||
|
||||
// layernorm2
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
|
||||
|
||||
// ffn
|
||||
cur = build_ffn(cur,
|
||||
layer.ff_up_w, layer.ff_up_b,
|
||||
nullptr, nullptr,
|
||||
layer.ff_down_w, layer.ff_down_b,
|
||||
hparams.ffn_op, il);
|
||||
|
||||
// residual 2
|
||||
cur = ggml_add(ctx0, inpL, cur);
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
ggml_tensor * embeddings = inpL;
|
||||
if (use_window_attn) {
|
||||
const int spatial_merge_unit = 4;
|
||||
window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / spatial_merge_unit);
|
||||
ggml_set_name(window_idx, "window_idx");
|
||||
ggml_set_input(window_idx);
|
||||
GGML_ASSERT(batch_size == 1);
|
||||
embeddings = ggml_reshape_2d(ctx0, embeddings, n_embd * spatial_merge_unit, n_patches / spatial_merge_unit);
|
||||
embeddings = ggml_get_rows(ctx0, embeddings, window_idx);
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd, n_patches, batch_size);
|
||||
cb(embeddings, "window_order_restored", -1);
|
||||
}
|
||||
|
||||
// post-layernorm (part of Siglip2VisionTransformer, applied after encoder)
|
||||
if (model.post_ln_w) {
|
||||
embeddings = build_norm(embeddings, model.post_ln_w, model.post_ln_b, norm_t, eps, n_layer);
|
||||
}
|
||||
|
||||
// Now apply merger (VLPatchMerger):
|
||||
// 1. Apply RMS norm (ln_q in VLPatchMerger)
|
||||
embeddings = build_norm(embeddings, model.mm_input_norm_w, nullptr, NORM_TYPE_RMS, 1e-6, -1);
|
||||
cb(embeddings, "merger_normed", -1);
|
||||
|
||||
// 2. First reshape for spatial merge (merge 2x2 patches)
|
||||
embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
|
||||
cb(embeddings, "merger_reshaped", -1);
|
||||
|
||||
embeddings = build_ffn(embeddings,
|
||||
model.mm_0_w, model.mm_0_b,
|
||||
nullptr, nullptr,
|
||||
model.mm_1_w, model.mm_1_b,
|
||||
FFN_GELU,
|
||||
-1);
|
||||
ggml_build_forward_expand(gf, embeddings);
|
||||
|
||||
return gf;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,213 @@
|
||||
#pragma once
|
||||
|
||||
#include "ggml.h"
|
||||
#include "clip-model.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
#define MTMD_INTERNAL_HEADER
|
||||
|
||||
struct mtmd_audio_mel {
|
||||
int64_t n_len;
|
||||
int64_t n_len_org;
|
||||
int64_t n_mel;
|
||||
|
||||
std::vector<float> data;
|
||||
};
|
||||
|
||||
struct mtmd_audio_mel_filters {
|
||||
int64_t n_mel;
|
||||
int64_t n_fft;
|
||||
|
||||
std::vector<float> data;
|
||||
};
|
||||
|
||||
// cache for audio processing, each processor instance owns its own cache
|
||||
struct mtmd_audio_cache {
|
||||
std::vector<float> sin_vals;
|
||||
std::vector<float> cos_vals;
|
||||
|
||||
std::vector<float> hann_window;
|
||||
|
||||
mtmd_audio_mel_filters filters;
|
||||
|
||||
void fill_sin_cos_table(uint32_t n);
|
||||
|
||||
void fill_hann_window(uint32_t length, bool periodic);
|
||||
|
||||
// Build mel filterbank matrix [n_mel × n_fft_bins] at runtime.
|
||||
// n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257.
|
||||
void fill_mel_filterbank_matrix(int64_t n_mel,
|
||||
int64_t n_fft,
|
||||
int sample_rate, // e.g. 16000
|
||||
float fmin = 0.0f, // e.g. 0.0
|
||||
float fmax = -1.0f, // e.g. sr/2; pass -1 for auto
|
||||
bool slaney_area_norm = true,
|
||||
float scale = 1.0f,
|
||||
bool use_htk = false
|
||||
);
|
||||
};
|
||||
|
||||
// compute one log10-mel frame from n_fft windowed samples, cache must be initialized for n_fft
|
||||
void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samples, int n_fft, int n_mel, float * out);
|
||||
|
||||
struct mtmd_audio_preprocessor {
|
||||
const clip_hparams & hparams;
|
||||
|
||||
mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
|
||||
mtmd_audio_preprocessor(const clip_hparams & hparams): hparams(hparams) {}
|
||||
|
||||
virtual ~mtmd_audio_preprocessor() = default;
|
||||
virtual void initialize() = 0; // NOT thread-safe
|
||||
virtual bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const = 0;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_muse_spark_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override {}
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
mtmd_audio_preprocessor_conformer(const clip_hparams & hparams) : mtmd_audio_preprocessor(hparams) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
// same as conformer, the audio is cut to 30 s and padded to 0.5 s (d1-omni audio.py)
|
||||
struct mtmd_audio_preprocessor_d1omni : mtmd_audio_preprocessor_conformer {
|
||||
using mtmd_audio_preprocessor_conformer::mtmd_audio_preprocessor_conformer;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
};
|
||||
|
||||
// mimi convolves the waveform directly, so this only pads it to a whole number of frames
|
||||
struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||||
void initialize() override {}
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
};
|
||||
|
||||
struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
|
||||
mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { }
|
||||
void initialize() override;
|
||||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||||
|
||||
private:
|
||||
mtmd_audio_cache cache;
|
||||
|
||||
static void worker_thread(int ith, const float * window_func, int window_size,
|
||||
const std::vector<float> & samples, int n_samples,
|
||||
int frame_size, int frame_step, int n_threads,
|
||||
int n_fft_bins,
|
||||
const mtmd_audio_cache & cache, mtmd_audio_mel & mel);
|
||||
};
|
||||
|
||||
//
|
||||
// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
|
||||
//
|
||||
struct mtmd_audio_streaming_istft {
|
||||
mtmd_audio_streaming_istft(int n_fft, int hop_length);
|
||||
|
||||
// reset streaming state
|
||||
void reset();
|
||||
|
||||
// process a single STFT frame (streaming)
|
||||
// frame_spectrum: [n_fft_bins x 2] interleaved real/imag
|
||||
// returns: up to hop_length samples
|
||||
std::vector<float> process_frame(const float * frame_spectrum);
|
||||
|
||||
// flush remaining samples at end of stream
|
||||
std::vector<float> flush();
|
||||
|
||||
private:
|
||||
int n_fft;
|
||||
int hop_length;
|
||||
int n_fft_bins;
|
||||
|
||||
// Own cache for output processing
|
||||
mtmd_audio_cache cache;
|
||||
|
||||
// Streaming state
|
||||
std::vector<float> overlap_buffer;
|
||||
std::vector<float> window_sum_buffer;
|
||||
int padding_to_remove;
|
||||
|
||||
// Working buffers for IFFT
|
||||
std::vector<float> ifft_in;
|
||||
std::vector<float> ifft_out;
|
||||
};
|
||||
@@ -0,0 +1,704 @@
|
||||
#include "arg.h"
|
||||
#include "debug.h"
|
||||
#include "log.h"
|
||||
#include "common.h"
|
||||
#include "sampling.h"
|
||||
#include "llama.h"
|
||||
#include "ggml.h"
|
||||
#include "console.h"
|
||||
#include "chat.h"
|
||||
#include "mtmd.h"
|
||||
#include "mtmd-helper.h"
|
||||
|
||||
#include <vector>
|
||||
#include <limits.h>
|
||||
#include <cinttypes>
|
||||
#include <clocale>
|
||||
|
||||
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
|
||||
#include <signal.h>
|
||||
#include <unistd.h>
|
||||
#elif defined (_WIN32)
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#include <signal.h>
|
||||
#endif
|
||||
|
||||
// volatile, because of signal being an interrupt
|
||||
static volatile bool g_is_generating = false;
|
||||
static volatile bool g_is_interrupted = false;
|
||||
|
||||
/**
|
||||
* Please note that this is NOT a production-ready binary.
|
||||
* It is a playground for trying multimodal support in llama.cpp.
|
||||
* For contributors: please keep this code simple and easy to understand. Do not add unnecessary complexity. The goal is to have a simple CLI for testing multimodal support.
|
||||
*/
|
||||
|
||||
static void show_additional_info(int /*argc*/, char ** argv) {
|
||||
LOG(
|
||||
"Experimental CLI for multimodal\n\n"
|
||||
"Usage: %s [options] -m <model> --mmproj <mmproj> --image <image> --audio <audio> -p <prompt>\n\n"
|
||||
" -m and --mmproj are required\n"
|
||||
" -hf user/repo can replace both -m and --mmproj in most cases\n"
|
||||
" --image, --audio and -p are optional, if NOT provided, the CLI will run in chat mode\n"
|
||||
" to disable using GPU for mmproj model, add --no-mmproj-offload\n",
|
||||
argv[0]
|
||||
);
|
||||
}
|
||||
|
||||
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
|
||||
static void sigint_handler(int signo) {
|
||||
if (signo == SIGINT) {
|
||||
if (g_is_generating) {
|
||||
g_is_generating = false;
|
||||
} else {
|
||||
console::cleanup();
|
||||
if (g_is_interrupted) {
|
||||
_exit(1);
|
||||
}
|
||||
g_is_interrupted = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// this is only used by tests.sh to capture the response ; it's not meant to be used in production
|
||||
static void inject_test_response_marker() {
|
||||
const char * env = std::getenv("MTMD_TEST_RESPONSE_MARKER");
|
||||
if (env) {
|
||||
LOG("%s\n", env);
|
||||
}
|
||||
}
|
||||
|
||||
struct mtmd_cli_context {
|
||||
mtmd::context_ptr ctx_vision;
|
||||
mtmd_helper_cap mtmd_cap = {};
|
||||
common_init_result_ptr llama_init;
|
||||
|
||||
llama_model * model;
|
||||
llama_context * lctx;
|
||||
const llama_vocab * vocab;
|
||||
common_sampler * smpl;
|
||||
common_batch batch;
|
||||
int n_batch;
|
||||
|
||||
mtmd::bitmaps bitmaps;
|
||||
std::vector<mtmd_helper::video_ptr> videos;
|
||||
|
||||
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
|
||||
std::string video_ffmpeg_bin_dir;
|
||||
|
||||
mtmd::batch_ptr mbatch;
|
||||
|
||||
// chat template
|
||||
common_chat_templates_ptr tmpls;
|
||||
std::vector<common_chat_msg> chat_history;
|
||||
bool use_jinja = false;
|
||||
// TODO: support for --system-prompt with /clear command
|
||||
|
||||
// support for legacy templates (models not having EOT token)
|
||||
llama_tokens antiprompt_tokens;
|
||||
|
||||
int n_threads = 1;
|
||||
llama_pos n_past = 0;
|
||||
|
||||
common_debug_cb_user_data cb_data;
|
||||
|
||||
mtmd_cli_context(common_params & params) : llama_init(common_init_from_params(params)) {
|
||||
model = llama_init->model();
|
||||
lctx = llama_init->context();
|
||||
if (!model || !lctx) {
|
||||
exit(1);
|
||||
}
|
||||
vocab = llama_model_get_vocab(model);
|
||||
smpl = common_sampler_init(model, params.sampling);
|
||||
n_threads = params.cpuparams.n_threads;
|
||||
batch = common_batch(lctx); // batch for next token generation
|
||||
n_batch = params.n_batch;
|
||||
|
||||
init_vision_context(params);
|
||||
|
||||
mtmd_cap = mtmd_helper_get_capabilities(model, ctx_vision.get());
|
||||
if (!mtmd_cap.chat) {
|
||||
LOG_ERR("Model does not support chat mode\n");
|
||||
LOG_ERR("Hint: for TTS models, please use llama-tts\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (!llama_model_chat_template(model, nullptr) && params.chat_template.empty()) {
|
||||
LOG_ERR("Model does not have chat template.\n");
|
||||
LOG_ERR(" For old llava models, you may need to use '--chat-template vicuna'\n");
|
||||
LOG_ERR(" For MobileVLM models, use '--chat-template deepseek'\n");
|
||||
LOG_ERR(" For Mistral Small 3.1, use '--chat-template mistral-v7'\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
tmpls = common_chat_templates_init(model, params.chat_template);
|
||||
use_jinja = params.use_jinja;
|
||||
chat_history.clear();
|
||||
LOG_INF("%s: chat template example:\n%s\n", __func__, common_chat_format_example(tmpls.get(), params.use_jinja, params.default_template_kwargs).c_str());
|
||||
|
||||
// load antiprompt tokens for legacy templates
|
||||
if (params.chat_template == "vicuna") {
|
||||
antiprompt_tokens = common_tokenize(lctx, "ASSISTANT:", false, true);
|
||||
} else if (params.chat_template == "deepseek") {
|
||||
antiprompt_tokens = common_tokenize(lctx, "###", false, true);
|
||||
}
|
||||
}
|
||||
|
||||
~mtmd_cli_context() {
|
||||
common_sampler_free(smpl);
|
||||
}
|
||||
|
||||
void init_vision_context(common_params & params) {
|
||||
const char * clip_path = params.mmproj.path.c_str();
|
||||
mtmd_context_params mparams = mtmd_context_params_default();
|
||||
mparams.use_gpu = params.mmproj_use_gpu;
|
||||
mparams.device = params.mmproj_device;
|
||||
mparams.print_timings = true;
|
||||
mparams.n_threads = params.cpuparams.n_threads;
|
||||
mparams.flash_attn_type = params.flash_attn_type;
|
||||
mparams.warmup = params.warmup;
|
||||
mparams.image_min_tokens = params.image_min_tokens;
|
||||
mparams.image_max_tokens = params.image_max_tokens;
|
||||
{
|
||||
// non-causal models need the whole image in one ubatch
|
||||
const int n_ubatch = llama_n_ubatch(lctx);
|
||||
auto mem = mtmd_get_memory_usage(clip_path, mparams);
|
||||
if (mem.use_non_causal && mem.image_max_tokens > n_ubatch) {
|
||||
LOG_WRN("%s: cap image_max_tokens (original=%d) to n_ubatch (%d) because model needs non-causal attention on image\n", __func__, mem.image_max_tokens, n_ubatch);
|
||||
LOG_WRN("%s: increase n_ubatch (-ub) to increase vision token budget\n", __func__);
|
||||
mparams.image_max_tokens = n_ubatch;
|
||||
mparams.image_min_tokens = std::min(mparams.image_min_tokens, n_ubatch);
|
||||
}
|
||||
}
|
||||
if (std::getenv("MTMD_DEBUG_GRAPH") != nullptr) {
|
||||
mparams.cb_eval_user_data = &cb_data;
|
||||
mparams.cb_eval = common_debug_cb_eval;
|
||||
}
|
||||
ctx_vision.reset(mtmd_init_from_file(clip_path, model, mparams));
|
||||
if (!ctx_vision.get()) {
|
||||
LOG_ERR("Failed to load vision model from %s\n", clip_path);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
video_ffmpeg_bin_dir = params.video_ffmpeg_bin_dir;
|
||||
init_opt.video_params.fps_target = params.video_fps;
|
||||
init_opt.video_params.timestamp_interval_ms = params.video_timestamp_interval_ms;
|
||||
init_opt.video_params.ffmpeg_bin_dir = video_ffmpeg_bin_dir.empty()
|
||||
? nullptr : video_ffmpeg_bin_dir.c_str();
|
||||
}
|
||||
|
||||
bool check_antiprompt(const llama_tokens & generated_tokens) {
|
||||
if (antiprompt_tokens.empty() || generated_tokens.size() < antiprompt_tokens.size()) {
|
||||
return false;
|
||||
}
|
||||
return std::equal(
|
||||
generated_tokens.end() - antiprompt_tokens.size(),
|
||||
generated_tokens.end(),
|
||||
antiprompt_tokens.begin()
|
||||
);
|
||||
}
|
||||
|
||||
bool load_media(const std::string & fname) {
|
||||
auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false, init_opt);
|
||||
if (!res.bitmap) {
|
||||
return false;
|
||||
}
|
||||
bitmaps.entries.emplace_back(res.bitmap);
|
||||
if (res.video_ctx) {
|
||||
videos.emplace_back(res.video_ctx);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
static int generate_response(mtmd_cli_context & ctx, int n_predict) {
|
||||
llama_tokens generated_tokens;
|
||||
for (int i = 0; i < n_predict; i++) {
|
||||
if (i > n_predict || !g_is_generating || g_is_interrupted) {
|
||||
LOG("\n");
|
||||
break;
|
||||
}
|
||||
|
||||
llama_token token_id = common_sampler_sample(ctx.smpl, ctx.lctx, -1);
|
||||
generated_tokens.push_back(token_id);
|
||||
common_sampler_accept(ctx.smpl, token_id, true);
|
||||
|
||||
if (llama_vocab_is_eog(ctx.vocab, token_id) || ctx.check_antiprompt(generated_tokens)) {
|
||||
LOG("\n");
|
||||
break; // end of generation
|
||||
}
|
||||
|
||||
LOG("%s", common_token_to_piece(ctx.lctx, token_id).c_str());
|
||||
fflush(stdout);
|
||||
|
||||
if (g_is_interrupted) {
|
||||
LOG("\n");
|
||||
break;
|
||||
}
|
||||
|
||||
// eval the token
|
||||
ctx.batch.clear();
|
||||
ctx.batch.add(token_id, ctx.n_past++, 0, true);
|
||||
if (llama_process(ctx.lctx, LLAMA_PROCESS_TYPE_DECODE, ctx.batch.get())) {
|
||||
LOG_ERR("failed to decode token\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
std::string generated_text = common_detokenize(ctx.lctx, generated_tokens);
|
||||
common_chat_msg msg;
|
||||
msg.role = "assistant";
|
||||
msg.content = generated_text;
|
||||
ctx.chat_history.push_back(std::move(msg));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// feed an audio file to a realtime model chunk by chunk, printing the transcript as it is produced
|
||||
static int run_audio_stream(mtmd_cli_context & ctx, const std::string & fname, int n_predict) {
|
||||
auto res = mtmd_helper_bitmap_init_from_file(ctx.ctx_vision.get(), fname.c_str(), false, ctx.init_opt);
|
||||
if (!res.bitmap) {
|
||||
return 1; // error is already printed by libmtmd
|
||||
}
|
||||
mtmd::bitmap bmp(res.bitmap);
|
||||
if (!mtmd_bitmap_is_audio(bmp.ptr.get())) {
|
||||
LOG_ERR("%s: %s is not an audio file\n", __func__, fname.c_str());
|
||||
return 1;
|
||||
}
|
||||
const float * samples = (const float *) bmp.data();
|
||||
const size_t n_samples = bmp.n_bytes() / sizeof(float);
|
||||
const int rate = mtmd_get_audio_sample_rate(ctx.ctx_vision.get());
|
||||
|
||||
const auto sparams = mtmd_audio_stream_params_default();
|
||||
mtmd::audio_stream_ptr stream(mtmd_audio_stream_init(ctx.ctx_vision.get(), sparams));
|
||||
if (!stream) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const size_t n_chunk = (size_t) rate * sparams.step_ms / 1000;
|
||||
const int n_embd = llama_model_n_embd_inp(ctx.model);
|
||||
|
||||
LOG("Transcribing %.1f s of audio, fed in chunks of %d ms\n\n", (double) n_samples / rate, sparams.step_ms);
|
||||
|
||||
size_t n_fed = 0;
|
||||
int n_gen = 0;
|
||||
llama_token tok = LLAMA_TOKEN_NULL;
|
||||
const int64_t t_start = ggml_time_us();
|
||||
int64_t t_enc = 0; // time spent in the stream (mel + encoder)
|
||||
int64_t t_dec = 0; // time spent in the decoder
|
||||
while (!g_is_interrupted) {
|
||||
const int64_t t0 = ggml_time_us();
|
||||
const int32_t n = mtmd_audio_stream_step(stream.get(), tok);
|
||||
t_enc += ggml_time_us() - t0;
|
||||
if (n < 0) {
|
||||
if (n == -2) {
|
||||
LOG_ERR("%s: stream error\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
break; // end of stream
|
||||
}
|
||||
if (n == 0) {
|
||||
if (n_fed < n_samples) {
|
||||
const size_t cnt = std::min(n_chunk, n_samples - n_fed);
|
||||
mtmd_audio_stream_push(stream.get(), samples + n_fed, cnt);
|
||||
n_fed += cnt;
|
||||
} else if (mtmd_audio_stream_flush(stream.get(), true) != 0) {
|
||||
break; // already ended
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const int64_t t1 = ggml_time_us();
|
||||
const float * rows = mtmd_audio_stream_get_output_embd(stream.get());
|
||||
ctx.batch.clear();
|
||||
for (int32_t i = 0; i < n; i++) {
|
||||
const llama_pos pos[GGML_MROPE_SECTIONS] = { ctx.n_past, ctx.n_past, ctx.n_past, 0 };
|
||||
ctx.n_past++;
|
||||
ctx.batch.add_embd({ rows + (size_t) i * n_embd, 1, (size_t) n_embd }, pos, 0, i == n - 1);
|
||||
}
|
||||
if (llama_process(ctx.lctx, LLAMA_PROCESS_TYPE_DECODE, ctx.batch.get())) {
|
||||
LOG_ERR("%s: failed to decode\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
tok = common_sampler_sample(ctx.smpl, ctx.lctx, -1);
|
||||
common_sampler_accept(ctx.smpl, tok, true);
|
||||
t_dec += ggml_time_us() - t1;
|
||||
if (llama_vocab_is_eog(ctx.vocab, tok)) {
|
||||
break;
|
||||
}
|
||||
// the model emits pad and word-boundary tokens between words
|
||||
if (!llama_vocab_is_control(ctx.vocab, tok)) {
|
||||
LOG("%s", common_token_to_piece(ctx.lctx, tok).c_str());
|
||||
fflush(stdout);
|
||||
}
|
||||
if (++n_gen >= n_predict) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const double t_total = (ggml_time_us() - t_start) / 1e6;
|
||||
LOG("\n\n%s: %d tokens, %.2f s audio in %.2f s (%.2fx realtime)\n", __func__,
|
||||
n_gen, (double) n_samples / rate, t_total, (double) n_samples / rate / t_total);
|
||||
LOG("%s: encoder %.2f s, decoder %.2f s, %.1f ms per 80 ms step\n", __func__,
|
||||
t_enc / 1e6, t_dec / 1e6, (t_enc + t_dec) / 1e3 / std::max(n_gen, 1));
|
||||
llama_perf_context_print(ctx.lctx);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static std::string chat_add_and_format(mtmd_cli_context & ctx, common_chat_msg & new_msg) {
|
||||
LOG_DBG("chat_add_and_format: new_msg.role='%s', new_msg.content='%s'\n",
|
||||
new_msg.role.c_str(), new_msg.content.c_str());
|
||||
auto formatted = common_chat_format_single(ctx.tmpls.get(), ctx.chat_history,
|
||||
new_msg, new_msg.role == "user",
|
||||
ctx.use_jinja);
|
||||
ctx.chat_history.push_back(new_msg);
|
||||
return formatted;
|
||||
}
|
||||
|
||||
static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {
|
||||
inject_test_response_marker();
|
||||
|
||||
bool add_bos = ctx.chat_history.empty();
|
||||
auto formatted_chat = chat_add_and_format(ctx, msg);
|
||||
LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str());
|
||||
|
||||
if (g_is_interrupted) return 0;
|
||||
|
||||
// note: we replace the marker here instead of letting mtmd_tokenize() to do that
|
||||
// because we want to demonstrate how to use mtmd_tokenize_from_parts()
|
||||
|
||||
// split the formatted chat on the media marker to get text segments
|
||||
const std::string marker = mtmd_default_marker();
|
||||
std::vector<std::string> segments;
|
||||
size_t start = 0;
|
||||
size_t pos;
|
||||
while ((pos = formatted_chat.find(marker, start)) != std::string::npos) {
|
||||
segments.push_back(formatted_chat.substr(start, pos - start));
|
||||
start = pos + marker.size();
|
||||
}
|
||||
segments.push_back(formatted_chat.substr(start));
|
||||
|
||||
auto bitmaps_c_ptr = ctx.bitmaps.c_ptr();
|
||||
if (segments.size() - 1 != bitmaps_c_ptr.size()) {
|
||||
LOG_ERR("Number of media markers (%zu) does not match number of loaded media (%zu)\n",
|
||||
segments.size() - 1, bitmaps_c_ptr.size());
|
||||
return 1;
|
||||
}
|
||||
|
||||
// interleave text and media parts
|
||||
std::vector<mtmd_input_text> texts(segments.size());
|
||||
std::vector<mtmd_input_part> parts;
|
||||
for (size_t i = 0; i < segments.size(); i++) {
|
||||
texts[i] = {segments[i].data(), segments[i].size(), /* add_special */ false, /* parse_special */ true};
|
||||
parts.push_back({&texts[i], nullptr});
|
||||
if (i < bitmaps_c_ptr.size()) {
|
||||
parts.push_back({nullptr, bitmaps_c_ptr[i]});
|
||||
}
|
||||
}
|
||||
std::vector<const mtmd_input_part *> parts_ptr;
|
||||
for (const auto & p : parts) {
|
||||
parts_ptr.push_back(&p);
|
||||
}
|
||||
|
||||
mtmd::input_chunks chunks(mtmd_input_chunks_init());
|
||||
int32_t res = mtmd_tokenize_from_parts(ctx.ctx_vision.get(),
|
||||
chunks.ptr.get(), // output
|
||||
parts_ptr.data(),
|
||||
parts_ptr.size(),
|
||||
add_bos);
|
||||
if (res != 0) {
|
||||
LOG_ERR("Unable to tokenize prompt, res = %d\n", res);
|
||||
return 1;
|
||||
}
|
||||
|
||||
ctx.bitmaps.entries.clear();
|
||||
ctx.videos.clear();
|
||||
|
||||
// batch encode all media chunks, then decode each
|
||||
size_t n_chunks = mtmd_input_chunks_size(chunks.ptr.get());
|
||||
for (size_t i = 0; i < n_chunks; i++) {
|
||||
auto chunk = mtmd_input_chunks_get(chunks.ptr.get(), i);
|
||||
auto chunk_type = mtmd_input_chunk_get_type(chunk);
|
||||
|
||||
if (chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
|
||||
// decode text chunk
|
||||
llama_pos new_n_past = ctx.n_past;
|
||||
res = mtmd_helper_eval_chunk_single(ctx.ctx_vision.get(),
|
||||
ctx.lctx,
|
||||
chunk,
|
||||
ctx.n_past,
|
||||
0, // seq_id
|
||||
ctx.n_batch,
|
||||
i == n_chunks - 1, // logits_last
|
||||
&new_n_past);
|
||||
if (res != 0) {
|
||||
LOG_ERR("Unable to eval text chunk %zu\n", i);
|
||||
return 1;
|
||||
}
|
||||
ctx.n_past = new_n_past;
|
||||
} else {
|
||||
// media chunk: try to get embd from existing batch, or create a new batch
|
||||
float * embd = nullptr;
|
||||
if (ctx.mbatch) {
|
||||
embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk);
|
||||
|
||||
if (embd) {
|
||||
LOG_DBG("found embd for media chunk %zu in existing batch\n", i);
|
||||
} else {
|
||||
LOG_DBG("media chunk %zu not found in existing batch, creating new batch\n", i);
|
||||
}
|
||||
}
|
||||
|
||||
if (!embd) {
|
||||
// create and encode a new batch with as many media chunks as possible
|
||||
ctx.mbatch.reset(mtmd_batch_init(ctx.ctx_vision.get()));
|
||||
res = mtmd_batch_add_chunk(ctx.mbatch.get(), chunk);
|
||||
GGML_ASSERT(res == 0); // first chunk must always succeed
|
||||
|
||||
int n_added = 1;
|
||||
// add as many subsequent media chunks as possible
|
||||
for (size_t j = i + 1; j < n_chunks; j++) {
|
||||
auto next_chunk = mtmd_input_chunks_get(chunks.ptr.get(), j);
|
||||
auto next_type = mtmd_input_chunk_get_type(next_chunk);
|
||||
if (next_type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
|
||||
break; // text chunk splits the batch
|
||||
}
|
||||
res = mtmd_batch_add_chunk(ctx.mbatch.get(), next_chunk);
|
||||
if (res != 0) {
|
||||
break; // batch full or incompatible
|
||||
}
|
||||
n_added++;
|
||||
}
|
||||
|
||||
int64_t time_start = ggml_time_ms();
|
||||
LOG_INF("encoding mtmd batch, n_chunks = %d (done = %zu, total = %zu)\n", n_added, i, n_chunks);
|
||||
res = mtmd_batch_encode(ctx.mbatch.get());
|
||||
if (res != 0) {
|
||||
LOG_ERR("Failed to encode mtmd batch, res = %d\n", res);
|
||||
return 1;
|
||||
}
|
||||
LOG_INF("mtmd batch encoding done in %d ms\n", (int)(ggml_time_ms() - time_start));
|
||||
|
||||
embd = mtmd_batch_get_output_embd(ctx.mbatch.get(), chunk);
|
||||
}
|
||||
|
||||
GGML_ASSERT(embd != nullptr);
|
||||
|
||||
llama_pos new_n_past = ctx.n_past;
|
||||
res = mtmd_helper_decode_image_chunk(ctx.ctx_vision.get(),
|
||||
ctx.lctx,
|
||||
chunk,
|
||||
embd,
|
||||
ctx.n_past,
|
||||
0, // seq_id
|
||||
ctx.n_batch,
|
||||
&new_n_past,
|
||||
nullptr, // callback
|
||||
nullptr // user_data
|
||||
);
|
||||
if (res != 0) {
|
||||
LOG_ERR("Unable to decode media chunk %zu\n", i);
|
||||
return 1;
|
||||
}
|
||||
ctx.n_past = new_n_past;
|
||||
}
|
||||
}
|
||||
|
||||
LOG("\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
|
||||
ggml_time_init();
|
||||
|
||||
common_params params;
|
||||
|
||||
common_init();
|
||||
|
||||
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_MTMD, show_additional_info)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
mtmd_helper_log_set(common_log_default_callback, nullptr);
|
||||
|
||||
if (params.mmproj.path.empty()) {
|
||||
show_additional_info(argc, argv);
|
||||
LOG_ERR("ERR: Missing --mmproj argument\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
ggml_backend_load_all();
|
||||
|
||||
mtmd_cli_context ctx(params);
|
||||
LOG_INF("%s: loading model: %s\n", __func__, params.model.path.c_str());
|
||||
|
||||
bool is_single_turn = !params.prompt.empty() && !params.image.empty();
|
||||
|
||||
int n_predict = params.n_predict < 0 ? INT_MAX : params.n_predict;
|
||||
|
||||
console::init(params.simple_io, params.use_color);
|
||||
atexit([]() { console::cleanup(); });
|
||||
|
||||
// Ctrl+C handling
|
||||
{
|
||||
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
|
||||
struct sigaction sigint_action;
|
||||
sigint_action.sa_handler = sigint_handler;
|
||||
sigemptyset (&sigint_action.sa_mask);
|
||||
sigint_action.sa_flags = 0;
|
||||
sigaction(SIGINT, &sigint_action, NULL);
|
||||
#elif defined (_WIN32)
|
||||
auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
|
||||
return (ctrl_type == CTRL_C_EVENT) ? (sigint_handler(SIGINT), true) : false;
|
||||
};
|
||||
SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
|
||||
#endif
|
||||
}
|
||||
|
||||
if (g_is_interrupted) return 130;
|
||||
|
||||
if (ctx.mtmd_cap.audio_stream) {
|
||||
if (params.image.size() != 1) {
|
||||
LOG_ERR("This model transcribes audio in realtime, pass exactly one --audio file\n");
|
||||
return 1;
|
||||
}
|
||||
return run_audio_stream(ctx, params.image[0], n_predict);
|
||||
}
|
||||
|
||||
auto eval_system_prompt_if_present = [&] {
|
||||
if (params.system_prompt.empty()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
common_chat_msg msg;
|
||||
msg.role = "system";
|
||||
msg.content = params.system_prompt;
|
||||
return eval_message(ctx, msg);
|
||||
};
|
||||
|
||||
LOG_WRN("WARN: This is an experimental CLI for testing multimodal capability.\n");
|
||||
LOG_WRN(" For normal use cases, please use the standard llama-cli\n");
|
||||
|
||||
if (eval_system_prompt_if_present()) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (is_single_turn) {
|
||||
g_is_generating = true;
|
||||
if (params.prompt.find(mtmd_default_marker()) == std::string::npos) {
|
||||
for (size_t i = 0; i < params.image.size(); i++) {
|
||||
// most models require the marker before each image
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/17616
|
||||
params.prompt = mtmd_default_marker() + params.prompt;
|
||||
}
|
||||
}
|
||||
|
||||
common_chat_msg msg;
|
||||
msg.role = "user";
|
||||
msg.content = params.prompt;
|
||||
for (const auto & image : params.image) {
|
||||
if (!ctx.load_media(image)) {
|
||||
return 1; // error is already printed by libmtmd
|
||||
}
|
||||
}
|
||||
if (eval_message(ctx, msg)) {
|
||||
return 1;
|
||||
}
|
||||
if (!g_is_interrupted && generate_response(ctx, n_predict)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
} else {
|
||||
LOG("\n Running in chat mode, available commands:");
|
||||
if (mtmd_support_vision(ctx.ctx_vision.get())) {
|
||||
LOG("\n /image <path> load an image");
|
||||
}
|
||||
if (mtmd_support_audio(ctx.ctx_vision.get())) {
|
||||
LOG("\n /audio <path> load an audio");
|
||||
}
|
||||
if (ctx.mtmd_cap.video) {
|
||||
LOG("\n /video <path> load a video");
|
||||
}
|
||||
LOG("\n /clear clear the chat history");
|
||||
LOG("\n /quit or /exit exit the program");
|
||||
LOG("\n");
|
||||
|
||||
std::string content;
|
||||
|
||||
while (!g_is_interrupted) {
|
||||
g_is_generating = false;
|
||||
LOG("\n> ");
|
||||
console::set_display(DISPLAY_TYPE_USER_INPUT);
|
||||
std::string line;
|
||||
console::readline(line, false);
|
||||
if (g_is_interrupted) break;
|
||||
console::set_display(DISPLAY_TYPE_RESET);
|
||||
// a submitted line always ends with a newline, an empty read is EOF
|
||||
if (line.empty()) {
|
||||
break;
|
||||
}
|
||||
line = string_strip(line);
|
||||
if (line.empty()) {
|
||||
continue;
|
||||
}
|
||||
if (line == "/quit" || line == "/exit") {
|
||||
break;
|
||||
}
|
||||
if (line == "/clear") {
|
||||
ctx.n_past = 0;
|
||||
ctx.chat_history.clear();
|
||||
llama_memory_clear(llama_get_memory(ctx.lctx), true);
|
||||
if (eval_system_prompt_if_present()) {
|
||||
return 1;
|
||||
}
|
||||
LOG("Chat history cleared\n\n");
|
||||
continue;
|
||||
}
|
||||
g_is_generating = true;
|
||||
bool is_image = line == "/image" || line.find("/image ") == 0;
|
||||
bool is_audio = line == "/audio" || line.find("/audio ") == 0;
|
||||
bool is_video = line == "/video" || line.find("/video ") == 0;
|
||||
if (is_image || is_audio || is_video) {
|
||||
if (line.size() < 8) {
|
||||
LOG_ERR("ERR: Missing media filename\n");
|
||||
continue;
|
||||
}
|
||||
std::string media_path = line.substr(7);
|
||||
if (ctx.load_media(media_path)) {
|
||||
LOG("%s %s loaded\n", media_path.c_str(), is_image ? "image" : is_audio ? "audio" : "video");
|
||||
content += mtmd_default_marker();
|
||||
}
|
||||
// else, error is already printed by libmtmd
|
||||
continue;
|
||||
} else {
|
||||
content += line;
|
||||
}
|
||||
common_chat_msg msg;
|
||||
msg.role = "user";
|
||||
msg.content = content;
|
||||
int ret = eval_message(ctx, msg);
|
||||
if (ret) {
|
||||
return 1;
|
||||
}
|
||||
if (g_is_interrupted) break;
|
||||
if (generate_response(ctx, n_predict)) {
|
||||
return 1;
|
||||
}
|
||||
content.clear();
|
||||
}
|
||||
}
|
||||
if (g_is_interrupted) LOG("\nInterrupted by user\n");
|
||||
LOG("\n\n");
|
||||
llama_perf_context_print(ctx.lctx);
|
||||
return g_is_interrupted ? 130 : 0;
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
#pragma once
|
||||
|
||||
// shared internal utilities for the mtmd-helper-*.cpp translation units
|
||||
// (mtmd-helper.cpp, mtmd-helper-gen.cpp)
|
||||
// NOT part of the public mtmd-helper.h API
|
||||
|
||||
#include "ggml.h"
|
||||
#include "llama.h"
|
||||
#include "llama-cpp.h"
|
||||
#include "mtmd.h"
|
||||
|
||||
#include <cstdarg>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <vector>
|
||||
|
||||
//
|
||||
// logging
|
||||
//
|
||||
|
||||
struct mtmd_helper_logger {
|
||||
ggml_log_callback default_callback = [](ggml_log_level level, const char * text, void * user_data) {
|
||||
(void) level;
|
||||
(void) user_data;
|
||||
fputs(text, stderr);
|
||||
fflush(stderr);
|
||||
};
|
||||
|
||||
ggml_log_callback log_callback = default_callback;
|
||||
void * log_callback_user_data;
|
||||
|
||||
void log_v(enum ggml_log_level level, const char * format, va_list args) {
|
||||
if (format == NULL) {
|
||||
return;
|
||||
}
|
||||
va_list args_copy;
|
||||
va_copy(args_copy, args);
|
||||
char buffer[128];
|
||||
int len = vsnprintf(buffer, 128, format, args);
|
||||
if (len < 128) {
|
||||
log_callback(level, buffer, log_callback_user_data);
|
||||
} else {
|
||||
char * buffer2 = (char *) calloc(len + 1, sizeof(char));
|
||||
vsnprintf(buffer2, len + 1, format, args_copy);
|
||||
buffer2[len] = 0;
|
||||
log_callback(level, buffer2, log_callback_user_data);
|
||||
free(buffer2);
|
||||
}
|
||||
va_end(args_copy);
|
||||
}
|
||||
|
||||
void log(enum ggml_log_level level, const char * format, ...) {
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
log_v(level, format, args);
|
||||
va_end(args);
|
||||
}
|
||||
};
|
||||
|
||||
// inline, so all TUs including this header share one instance
|
||||
inline mtmd_helper_logger g_logger;
|
||||
|
||||
#define LOG_DBG(...) g_logger.log(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
|
||||
#define LOG_INF(...) g_logger.log(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
|
||||
#define LOG_WRN(...) g_logger.log(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
|
||||
#define LOG_ERR(...) g_logger.log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
|
||||
|
||||
//
|
||||
// embd batch
|
||||
//
|
||||
|
||||
// helper struct to make working with embd batch easier
|
||||
// note: this will be removed after llama_batch_ext refactoring
|
||||
struct decode_embd_batch {
|
||||
int n_pos_per_embd;
|
||||
int n_mmproj_embd;
|
||||
int32_t n_tokens;
|
||||
const float * embd; // [n_tokens, n_mmproj_embd], not owned
|
||||
std::vector<llama_pos> pos; // [n_pos_per_embd, n_tokens], section-major
|
||||
std::vector<llama_pos> pos_view; // sliced positions of the last get_view()
|
||||
std::vector<int8_t> logits;
|
||||
llama_seq_id seq_id = 0;
|
||||
|
||||
llama_batch_ext_ptr batch; // rendered sub-batch, see render()
|
||||
|
||||
decode_embd_batch(const float * embd, int32_t n_tokens, int n_pos_per_embd, int n_mmproj_embd)
|
||||
: n_pos_per_embd(n_pos_per_embd), n_mmproj_embd(n_mmproj_embd), n_tokens(n_tokens), embd(embd) {
|
||||
GGML_ASSERT(n_tokens > 0 && n_pos_per_embd > 0 && n_mmproj_embd > 0);
|
||||
pos .resize((size_t) n_tokens * (size_t) n_pos_per_embd);
|
||||
logits.resize(n_tokens);
|
||||
}
|
||||
|
||||
void set_position_normal(llama_pos pos_0, llama_seq_id seq_id) {
|
||||
this->seq_id = seq_id;
|
||||
for (int i = 0; i < n_tokens; i++) {
|
||||
pos[i] = pos_0 + i;
|
||||
}
|
||||
}
|
||||
|
||||
// M-RoPE for image
|
||||
void set_position_mrope_2d(const std::vector<mtmd_decoder_pos> & rel_pos, llama_seq_id seq_id) {
|
||||
GGML_ASSERT(n_pos_per_embd == 4);
|
||||
GGML_ASSERT(!rel_pos.empty() && (int32_t)rel_pos.size() == n_tokens);
|
||||
this->seq_id = seq_id;
|
||||
for (int32_t i = 0; i < n_tokens; i++) {
|
||||
const size_t idx = (size_t) i;
|
||||
const size_t n = (size_t) n_tokens;
|
||||
pos[idx ] = rel_pos[i].t;
|
||||
pos[idx + n ] = rel_pos[i].y;
|
||||
pos[idx + n * 2] = rel_pos[i].x;
|
||||
pos[idx + n * 3] = rel_pos[i].z;
|
||||
}
|
||||
}
|
||||
|
||||
// M-RoPE for audio
|
||||
void set_position_mrope_1d(llama_pos pos_0, llama_seq_id seq_id) {
|
||||
GGML_ASSERT(n_pos_per_embd == 4);
|
||||
this->seq_id = seq_id;
|
||||
for (int i = 0; i < n_tokens; i++) {
|
||||
const size_t idx = (size_t) i;
|
||||
const size_t n = (size_t) n_tokens;
|
||||
pos[idx ] = pos_0 + i;
|
||||
pos[idx + n ] = pos_0 + i;
|
||||
pos[idx + n * 2] = pos_0 + i;
|
||||
pos[idx + n * 3] = pos_0 + i;
|
||||
}
|
||||
}
|
||||
|
||||
// describe the entries [offset, offset + n) with section-major positions
|
||||
mtmd_helper_embd_batch get_view(int offset, int n) {
|
||||
GGML_ASSERT(offset >= 0 && n > 0 && offset + n <= n_tokens);
|
||||
pos_view.clear();
|
||||
pos_view.reserve((size_t) n * (size_t) n_pos_per_embd);
|
||||
for (int j = 0; j < n_pos_per_embd; j++) {
|
||||
const size_t src = (size_t) j * (size_t) n_tokens + (size_t) offset;
|
||||
pos_view.insert(pos_view.end(), pos.data() + src, pos.data() + src + n);
|
||||
}
|
||||
return {
|
||||
/*n_tokens =*/ n,
|
||||
/*embd =*/ embd + (size_t) offset * n_mmproj_embd,
|
||||
/*n_embd =*/ n_mmproj_embd,
|
||||
/*pos =*/ pos_view.data(),
|
||||
/*n_pos =*/ n_pos_per_embd,
|
||||
/*seq_id =*/ seq_id,
|
||||
};
|
||||
}
|
||||
|
||||
// render the entries [offset, offset + n) into a batch owned by this object, ready for llama_process()
|
||||
llama_batch_ext * render(llama_context * lctx, int offset, int n) {
|
||||
GGML_ASSERT(offset >= 0 && n > 0 && offset + n <= n_tokens);
|
||||
if (!batch) {
|
||||
batch.reset(llama_batch_ext_init(lctx));
|
||||
}
|
||||
llama_batch_ext_clear(batch.get());
|
||||
for (int i = offset; i < offset + n; i++) {
|
||||
const llama_embd e = { embd + (size_t) i * n_mmproj_embd, 1, (size_t) n_mmproj_embd };
|
||||
const int32_t idx = llama_batch_ext_add_embd(batch.get(), seq_id, e);
|
||||
GGML_ASSERT(idx >= 0);
|
||||
|
||||
llama_pos p[GGML_MROPE_SECTIONS] = { 0, 0, 0, 0 };
|
||||
for (int j = 0; j < n_pos_per_embd; j++) {
|
||||
p[j] = pos[(size_t) j * (size_t) n_tokens + (size_t) i];
|
||||
}
|
||||
llama_batch_ext_set_pos(batch.get(), idx, p);
|
||||
|
||||
if (logits[i]) {
|
||||
llama_batch_ext_set_output_logits(batch.get(), idx, true);
|
||||
}
|
||||
}
|
||||
return batch.get();
|
||||
}
|
||||
};
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,373 @@
|
||||
#ifndef MTMD_HELPER_H
|
||||
#define MTMD_HELPER_H
|
||||
|
||||
#include "ggml.h"
|
||||
#include "llama.h"
|
||||
#include "mtmd.h"
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//
|
||||
// libmtmd helper functions
|
||||
//
|
||||
// Please note that these helpers are not guaranteed to be stable.
|
||||
// BREAKING CHANGES are expected.
|
||||
//
|
||||
|
||||
struct mtmd_helper_video;
|
||||
typedef struct mtmd_helper_video mtmd_helper_video;
|
||||
|
||||
struct mtmd_helper_video_init_params {
|
||||
float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f
|
||||
const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH
|
||||
int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms
|
||||
// TODO @ngxson : allow "placeholder" bitmap output for counting tokens
|
||||
};
|
||||
|
||||
MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void);
|
||||
|
||||
// opt for mtmd_helper_bitmap_init_from_*()
|
||||
struct mtmd_helper_init_opt {
|
||||
struct mtmd_helper_video_init_params video_params;
|
||||
};
|
||||
typedef struct mtmd_helper_init_opt mtmd_helper_init_opt;
|
||||
|
||||
MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void);
|
||||
|
||||
// Set callback for all future logging events.
|
||||
// If this is not called, or NULL is supplied, everything is output on stderr.
|
||||
// Note: this also call mtmd_log_set() internally
|
||||
MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data);
|
||||
|
||||
struct mtmd_helper_bitmap_wrapper {
|
||||
mtmd_bitmap * bitmap;
|
||||
mtmd_helper_video * video_ctx;
|
||||
};
|
||||
|
||||
// helper function to construct a mtmd_bitmap from a file
|
||||
// it calls mtmd_helper_bitmap_init_from_buf() internally
|
||||
// returns nullptr on failure
|
||||
// this function is thread-safe
|
||||
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(
|
||||
const mtmd_context * ctx,
|
||||
const char * fname,
|
||||
bool placeholder,
|
||||
struct mtmd_helper_init_opt opt);
|
||||
|
||||
// helper function to construct a mtmd_bitmap from a buffer containing a file
|
||||
// supported formats:
|
||||
// image: formats supported by stb_image: jpg, png, bmp, gif, etc.
|
||||
// webp is decoded via ffmpeg, requires MTMD_VIDEO build with ffmpeg in PATH
|
||||
// audio: formats supported by miniaudio: wav, mp3, flac
|
||||
// note:
|
||||
// - for now, video input is only supported via C++ helper functions
|
||||
// - audio files will be auto-detected based on magic bytes
|
||||
// - output bitmap will have SHA-256 hash (hex string) as the ID
|
||||
// returns nullptr on failure
|
||||
// this function is thread-safe
|
||||
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
|
||||
const mtmd_context * ctx,
|
||||
const unsigned char * buf, size_t len,
|
||||
bool placeholder,
|
||||
struct mtmd_helper_init_opt opt);
|
||||
|
||||
// helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache
|
||||
MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks);
|
||||
|
||||
// helper to count the total position of tokens from a list of chunks, useful to keep track of n_past
|
||||
// normally, n_pos is equal to n_tokens, but for M-RoPE it is different
|
||||
MTMD_API llama_pos mtmd_helper_get_n_pos(const mtmd_input_chunks * chunks);
|
||||
|
||||
// helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE
|
||||
// out_pos must have length == mtmd_helper_get_n_tokens(image)
|
||||
MTMD_API void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * image, llama_pos pos_0, struct mtmd_decoder_pos * out_pos);
|
||||
|
||||
// helper function that automatically:
|
||||
// 1. decode text chunks
|
||||
// 2. run mtmd_encode_chunk() on image chunks, then mtmd_get_output_embd() and then decode the embeddings
|
||||
// if any of the mtmd_encode_chunk() or decode calls return non-zero, stop and forward the error
|
||||
// otherwise, returns 0 on success
|
||||
// this function is NOT thread-safe
|
||||
MTMD_API int32_t mtmd_helper_eval_chunks(mtmd_context * ctx,
|
||||
struct llama_context * lctx,
|
||||
const mtmd_input_chunks * chunks,
|
||||
llama_pos n_past,
|
||||
llama_seq_id seq_id,
|
||||
int32_t n_batch,
|
||||
bool logits_last,
|
||||
llama_pos * new_n_past);
|
||||
|
||||
// works like mtmd_helper_eval_chunks(), but only for a single chunk
|
||||
// this function is NOT thread-safe
|
||||
MTMD_API int32_t mtmd_helper_eval_chunk_single(mtmd_context * ctx,
|
||||
struct llama_context * lctx,
|
||||
const mtmd_input_chunk * chunk,
|
||||
llama_pos n_past,
|
||||
llama_seq_id seq_id,
|
||||
int32_t n_batch,
|
||||
bool logits_last,
|
||||
llama_pos * new_n_past);
|
||||
|
||||
// one decoded sub-batch of embeddings, passed to mtmd_helper_post_decode_callback
|
||||
struct mtmd_helper_embd_batch {
|
||||
int32_t n_tokens;
|
||||
const float * embd; // [n_tokens, n_embd]
|
||||
int32_t n_embd;
|
||||
const llama_pos * pos; // [n_pos, n_tokens], section-major
|
||||
int32_t n_pos; // 4 for M-RoPE models, 1 otherwise
|
||||
llama_seq_id seq_id;
|
||||
};
|
||||
|
||||
typedef int32_t (*mtmd_helper_post_decode_callback)(const struct mtmd_helper_embd_batch * batch, void * user_data);
|
||||
|
||||
// helper function to decode an image whose embeddings have already been calculated
|
||||
// this helper will handle batching and pre/post decoding setup (for ex. gemma 3 requires non-causal attention)
|
||||
// ret 0 on success, -1 on chunk not being a valid image chunk, 1 on decode failure
|
||||
MTMD_API int32_t mtmd_helper_decode_image_chunk(mtmd_context * ctx,
|
||||
struct llama_context * lctx,
|
||||
const mtmd_input_chunk * chunk,
|
||||
float * encoded_embd,
|
||||
llama_pos n_past,
|
||||
llama_seq_id seq_id,
|
||||
int32_t n_batch,
|
||||
llama_pos * new_n_past,
|
||||
mtmd_helper_post_decode_callback callback,
|
||||
void * user_data);
|
||||
|
||||
//
|
||||
// video input helpers (requires ffmpeg/ffprobe installed on the system)
|
||||
// the notion of video only exists at the helper level, it is not visible to the core mtmd library
|
||||
//
|
||||
// NOTE: this implementation is model-agnostic, it can be used with any vision-capable model
|
||||
// however, it may not be accurate for some specific models
|
||||
// (this is expected for now, to keep the implementation simple)
|
||||
//
|
||||
|
||||
struct mtmd_helper_video_info {
|
||||
uint32_t width;
|
||||
uint32_t height;
|
||||
float fps; // effective fps (fps_target if set, else original video fps)
|
||||
int32_t n_frames; // estimated total frames at effective fps (-1 if unknown)
|
||||
};
|
||||
|
||||
// note: mtmd_helper_video_init_params is defined at the top, as it is part of mtmd_helper_init_opt
|
||||
|
||||
// returns NULL on failure (ffprobe not found, file unreadable, etc.)
|
||||
MTMD_API mtmd_helper_video * mtmd_helper_video_init(
|
||||
const struct mtmd_context * mctx,
|
||||
const char * path,
|
||||
struct mtmd_helper_video_init_params params);
|
||||
|
||||
// Same as mtmd_helper_video_init(), but reads from an in-memory buffer.
|
||||
// The buffer is copied internally; the caller does not need to keep it alive.
|
||||
// Note: pipe input is not seekable, so seeking will use output-side seeking
|
||||
// (ffmpeg decodes and discards frames up to the target position).
|
||||
MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf(
|
||||
const struct mtmd_context * mctx,
|
||||
const unsigned char * buf, size_t len,
|
||||
struct mtmd_helper_video_init_params params);
|
||||
MTMD_API void mtmd_helper_video_free(mtmd_helper_video * ctx);
|
||||
MTMD_API struct mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx);
|
||||
|
||||
// Read the next item from the video stream; exactly one of out_bitmap or out_text is set per call.
|
||||
// *out_bitmap - heap-allocated; caller must free with mtmd_bitmap_free()
|
||||
// *out_text - heap-allocated (always via strdup/malloc); caller must free with free()
|
||||
// returns 0 on success, -1 on EOF, -2 on error
|
||||
MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
|
||||
mtmd_bitmap ** out_bitmap,
|
||||
char ** out_text);
|
||||
|
||||
struct mtmd_helper_cap {
|
||||
bool chat; // false for tts-only model (no chat template)
|
||||
|
||||
// input capabilities
|
||||
bool vision;
|
||||
bool audio;
|
||||
bool video; // also depending on MTMD_VIDEO build option
|
||||
|
||||
// realtime capabilities
|
||||
bool audio_stream; // realtime transcription, see mtmd_audio_stream_init()
|
||||
|
||||
// output capabilities
|
||||
bool text_gen; // always true for now
|
||||
bool audio_gen;
|
||||
bool audio_gen_spk_ref; // accepts a speaker reference audio for voice cloning
|
||||
};
|
||||
typedef struct mtmd_helper_cap mtmd_helper_cap;
|
||||
|
||||
// mctx can be NULL (text-only model)
|
||||
MTMD_API struct mtmd_helper_cap mtmd_helper_get_capabilities(const struct llama_model * model, const struct mtmd_context * mctx);
|
||||
|
||||
DEPRECATED(MTMD_API bool mtmd_helper_model_can_chat(const struct llama_context * lctx, const struct mtmd_context * mctx),
|
||||
"use mtmd_helper_get_capabilities() instead");
|
||||
DEPRECATED(MTMD_API bool mtmd_helper_support_video(const struct mtmd_context * ctx),
|
||||
"use mtmd_helper_get_capabilities() instead");
|
||||
|
||||
//
|
||||
// Audio generation helpers
|
||||
// (early-stage experimental, subjected to breaking changes)
|
||||
//
|
||||
|
||||
// audio generation helper context
|
||||
// contains accumulator for generated audio features and PCM audio
|
||||
struct mtmd_helper_gen_audio;
|
||||
typedef struct mtmd_helper_gen_audio mtmd_helper_gen_audio;
|
||||
|
||||
enum mtmd_helper_gen_audio_outtype {
|
||||
MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM, // raw PCM
|
||||
MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono
|
||||
};
|
||||
struct mtmd_helper_gen_audio_inp {
|
||||
bool stream; // if true, output() must be called after each step_gen()
|
||||
llama_seq_id seq_id;
|
||||
|
||||
const char * prompt;
|
||||
size_t prompt_len;
|
||||
|
||||
mtmd_bitmap * speaker_ref; // optional, can be NULL
|
||||
const char * lang; // optional, can be NULL
|
||||
|
||||
int32_t top_k;
|
||||
float top_p;
|
||||
uint32_t seed; // UINT32_MAX for random (default: random)
|
||||
|
||||
enum mtmd_helper_gen_audio_outtype out_type;
|
||||
};
|
||||
|
||||
MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void);
|
||||
|
||||
MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(
|
||||
struct llama_context * lctx,
|
||||
struct mtmd_context * mctx);
|
||||
|
||||
MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx);
|
||||
|
||||
MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx);
|
||||
|
||||
MTMD_API int32_t mtmd_helper_gen_audio_set_input(
|
||||
mtmd_helper_gen_audio * ctx,
|
||||
const struct mtmd_helper_gen_audio_inp * inp);
|
||||
|
||||
// processes at most n_batch prompt tokens per call
|
||||
// returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error
|
||||
MTMD_API int32_t mtmd_helper_gen_audio_step_prompt(
|
||||
mtmd_helper_gen_audio * ctx,
|
||||
int32_t n_batch);
|
||||
|
||||
// generates one frame; must only be called after step_prompt() has returned 0
|
||||
// sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token
|
||||
// out_stop (optional) is set on end-of-speech, the caller must then stop the loop
|
||||
// h_state_out is valid until next step_gen() or reset() call, null if no frame is generated
|
||||
MTMD_API int32_t mtmd_helper_gen_audio_step_gen(
|
||||
mtmd_helper_gen_audio * ctx,
|
||||
llama_token sampled,
|
||||
const float * h_state_in,
|
||||
const float ** h_state_out,
|
||||
bool * out_stop);
|
||||
|
||||
// out_data valid until next get_output() or reset() call
|
||||
// out_n_samples (optional, can be NULL) receives the number of generated PCM samples
|
||||
// if inp->stream is true: returns only audio produced since the previous call, and
|
||||
// *out_data_len == 0 whenever a full window_frames batch hasn't accumulated yet
|
||||
MTMD_API int32_t mtmd_helper_gen_audio_get_output(
|
||||
mtmd_helper_gen_audio * ctx,
|
||||
int32_t * out_sample_rate,
|
||||
const char ** out_data,
|
||||
size_t * out_data_len,
|
||||
int64_t * out_n_samples);
|
||||
|
||||
// forces any buffered codes through code2wav now, regardless of window_frames;
|
||||
// call once when generation has ended, before the last get_output() in stream mode
|
||||
MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
#include <set>
|
||||
#include <memory>
|
||||
|
||||
namespace mtmd_helper {
|
||||
|
||||
//
|
||||
// C++ wrappers
|
||||
//
|
||||
|
||||
// video-related C++ wrappers
|
||||
struct mtmd_helper_video_deleter {
|
||||
void operator()(mtmd_helper_video * val) { mtmd_helper_video_free(val); }
|
||||
};
|
||||
using video_ptr = std::unique_ptr<mtmd_helper_video, mtmd_helper_video_deleter>;
|
||||
|
||||
// audio generation-related C++ wrappers
|
||||
struct mtmd_helper_gen_audio_deleter {
|
||||
void operator()(mtmd_helper_gen_audio * val) { mtmd_helper_gen_audio_free(val); }
|
||||
};
|
||||
using gen_audio_ptr = std::unique_ptr<mtmd_helper_gen_audio, mtmd_helper_gen_audio_deleter>;
|
||||
struct gen_audio {
|
||||
|
||||
// sub-struct, RAII wrapper for mtmd_helper_gen_audio_inp
|
||||
struct inp {
|
||||
mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default();
|
||||
std::string prompt_str;
|
||||
std::string lang_str;
|
||||
mtmd::bitmap_ptr speaker_ref_ptr;
|
||||
|
||||
inp() = default;
|
||||
inp(inp &&) = default;
|
||||
inp & operator=(inp &&) = default;
|
||||
inp(const inp &) = delete;
|
||||
inp & operator=(const inp &) = delete;
|
||||
|
||||
void set_prompt (std::string p) { prompt_str = std::move(p); }
|
||||
void set_lang (std::string l) { lang_str = std::move(l); }
|
||||
void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); }
|
||||
|
||||
// pointers are only valid as long as *this is alive
|
||||
const mtmd_helper_gen_audio_inp * get() {
|
||||
data.prompt = prompt_str.c_str();
|
||||
data.prompt_len = prompt_str.size();
|
||||
data.lang = lang_str.empty() ? nullptr : lang_str.c_str();
|
||||
data.speaker_ref = speaker_ref_ptr.get();
|
||||
return &data;
|
||||
}
|
||||
};
|
||||
|
||||
gen_audio_ptr ctx;
|
||||
void init(struct llama_context * lctx, struct mtmd_context * mctx) {
|
||||
ctx.reset(mtmd_helper_gen_audio_init(lctx, mctx));
|
||||
}
|
||||
bool valid() const {
|
||||
return ctx.get() != nullptr;
|
||||
}
|
||||
void reset() {
|
||||
mtmd_helper_gen_audio_reset(ctx.get());
|
||||
}
|
||||
int32_t set_input(const struct mtmd_helper_gen_audio_inp * inp) {
|
||||
return mtmd_helper_gen_audio_set_input(ctx.get(), inp);
|
||||
}
|
||||
int32_t step_prompt(int32_t n_batch) {
|
||||
return mtmd_helper_gen_audio_step_prompt(ctx.get(), n_batch);
|
||||
}
|
||||
int32_t step_gen(llama_token sampled, const float * h_state, const float ** h_state_out, bool * out_stop = nullptr) {
|
||||
return mtmd_helper_gen_audio_step_gen(ctx.get(), sampled, h_state, h_state_out, out_stop);
|
||||
}
|
||||
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) {
|
||||
return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples);
|
||||
}
|
||||
int32_t flush() {
|
||||
return mtmd_helper_gen_audio_flush(ctx.get());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace mtmd_helper
|
||||
#endif
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,332 @@
|
||||
#pragma once
|
||||
|
||||
#include "ggml.h"
|
||||
#include "clip-model.h"
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
#define MTMD_INTERNAL_HEADER
|
||||
|
||||
struct mtmd_image_preproc_out {
|
||||
std::vector<clip_image_f32> entries;
|
||||
// grid size is required for llava-uhd style models
|
||||
|
||||
clip_image_f32 overview; // overview image (downscaled image)
|
||||
int grid_x = 0;
|
||||
int grid_y = 0;
|
||||
|
||||
void append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true);
|
||||
void append(const clip_hparams & hparams, const std::vector<clip_image_u8> & imgs, bool normalized = true);
|
||||
void append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized = true);
|
||||
|
||||
void append_overview(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true);
|
||||
bool has_overview() const {
|
||||
return overview.nx() > 0 || overview.ny() > 0;
|
||||
}
|
||||
};
|
||||
|
||||
// base class, models must inherit from this class
|
||||
struct mtmd_image_preprocessor {
|
||||
const clip_hparams & hparams;
|
||||
|
||||
mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
|
||||
mtmd_image_preprocessor(const clip_hparams & hparams): hparams(hparams) {}
|
||||
|
||||
virtual ~mtmd_image_preprocessor() = default;
|
||||
virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* implementation of LLaVA-UHD:
|
||||
* - https://arxiv.org/pdf/2403.11703
|
||||
* - https://github.com/thunlp/LLaVA-UHD
|
||||
* - https://github.com/thunlp/LLaVA-UHD/blob/302301bc2175f7e717fb8548516188e89f649753/llava_uhd/train/llava-uhd/slice_logic.py#L118
|
||||
*
|
||||
* overview:
|
||||
* - an image always have a single overview (downscaled image)
|
||||
* - an image can have 0 or multiple slices, depending on the image size
|
||||
* - each slice can then be considered as a separate image
|
||||
*
|
||||
* note: the term "slice" and "tile" are used interchangeably
|
||||
*
|
||||
* for example:
|
||||
*
|
||||
* [overview] --> [slice 1] --> [slice 2]
|
||||
* | |
|
||||
* +--> [slice 3] --> [slice 4]
|
||||
*
|
||||
* NOTE: for the ordering of overview, set "ov_img_first" on the mtmd_context
|
||||
*/
|
||||
struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preprocessor_llava_uhd(const clip_hparams & hparams) : mtmd_image_preprocessor(hparams) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
|
||||
struct slice_coordinates {
|
||||
int x;
|
||||
int y;
|
||||
clip_image_size size;
|
||||
};
|
||||
|
||||
struct slice_instructions {
|
||||
clip_image_size overview_size; // size of downscaled image
|
||||
clip_image_size refined_size; // size of image right before slicing (must be multiple of slice size)
|
||||
clip_image_size grid_size; // grid_size.width * grid_size.height = number of slices
|
||||
std::vector<slice_coordinates> slices;
|
||||
};
|
||||
|
||||
virtual slice_instructions get_slice_instructions(const clip_image_size & original_size) const;
|
||||
|
||||
struct slice_output {
|
||||
clip_image_u8 overview;
|
||||
std::vector<clip_image_u8> slices;
|
||||
};
|
||||
slice_output slice_image(const clip_image_u8 & img, const slice_instructions & inst) const;
|
||||
|
||||
protected:
|
||||
// align slices to a multiple of the merger factor (integer merger tokens per slice)
|
||||
virtual int get_slice_align() const {
|
||||
const int merge = hparams.n_merge > 0 ? hparams.n_merge : 1;
|
||||
return hparams.patch_size * merge;
|
||||
}
|
||||
|
||||
// rounding for snapping a length to a multiple of the align size
|
||||
virtual int align_round(double v) const {
|
||||
return static_cast<int>(std::round(v));
|
||||
}
|
||||
|
||||
clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false) const;
|
||||
|
||||
/**
|
||||
* Selects the best resolution from a list of possible resolutions based on the original size.
|
||||
*
|
||||
* For example, when given a list of resolutions:
|
||||
* - 100x100
|
||||
* - 200x100
|
||||
* - 100x200
|
||||
* - 200x200
|
||||
*
|
||||
* And an input image of size 111x200, then 100x200 is the best fit (least wasted resolution).
|
||||
*
|
||||
* @param original_size The original size of the image
|
||||
* @param possible_resolutions A list of possible resolutions
|
||||
* @return The best fit resolution
|
||||
*/
|
||||
clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions) const;
|
||||
|
||||
private:
|
||||
clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) const;
|
||||
int ensure_divide(int length, int patch_size) const;
|
||||
clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false) const;
|
||||
clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) const;
|
||||
};
|
||||
|
||||
// downscale or upscale the input image to fixed size
|
||||
struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
};
|
||||
|
||||
// resize image to multiple of patch_size*n_merge, while preserving aspect ratio
|
||||
// if image_resize_pad is true, the resized image will be padded, otherwise it will be either stretched or center-cropped depending on image_resize_pad
|
||||
// this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc
|
||||
struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
virtual clip_image_size target_size(const clip_image_size & original_size) const { return get_target_size(hparams, original_size); }
|
||||
};
|
||||
|
||||
// LFM3-VL, same as dyn_size but sized exactly as the transformers Qwen2-VL smart_resize
|
||||
struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size {
|
||||
mtmd_image_preprocessor_lfm3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_dyn_size(ctx) {}
|
||||
clip_image_size target_size(const clip_image_size & original_size) const override;
|
||||
|
||||
static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels);
|
||||
};
|
||||
|
||||
// GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally
|
||||
struct mtmd_image_preprocessor_glm5v : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_glm5v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
};
|
||||
|
||||
// similar to mtmd_image_preprocessor_dyn_size, but cap the longest edge at hparams.image_longest_edge, while preserving aspect ratio
|
||||
struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
};
|
||||
|
||||
// ref: inference/image_processor.py in the HF repo (DeepSeek-V4-Flash-Vision)
|
||||
struct mtmd_image_preprocessor_deepseek4v : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_deepseek4v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
|
||||
private:
|
||||
struct grid_info {
|
||||
int n_llm_h;
|
||||
int n_llm_w;
|
||||
int n_tokens; // token count of the block (incl. newline/pad rows and start/end, excl. lead pads)
|
||||
};
|
||||
static grid_info grid_tokens(int best_height, int best_width, int patch_size, int r);
|
||||
static void solve_resize_ratio(int height, int width, int p, int r, int max_n_token, int & best_height, int & best_width);
|
||||
static void safe_resize(int height, int width, int & best_height, int & best_width, int p, int r, int max_n_token);
|
||||
};
|
||||
|
||||
// ref: inference/image_processor.py in the HF repo (DeepSeek-V4.1-Flash): plan_image_grid / load_image
|
||||
struct mtmd_image_preprocessor_deepseek41v : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_deepseek41v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
// target pixel size (multiples of patch_size) for an image of the given size, exposed for tests
|
||||
static void plan_grid(int width, int height, int p, int r, int max_n_token, int min_pixels,
|
||||
int & best_width, int & best_height);
|
||||
};
|
||||
|
||||
// custom llava-uhd slicing logic for MiniCPM-V
|
||||
struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd {
|
||||
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
|
||||
slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
|
||||
|
||||
protected:
|
||||
// always patch_size * 4, even in 4x mode (the 2x2 vit_merger slot stays)
|
||||
int get_slice_align() const override {
|
||||
return hparams.patch_size * 4;
|
||||
}
|
||||
|
||||
// Python's round() breaks ties to even, unlike std::round
|
||||
int align_round(double v) const override {
|
||||
const double fl = std::floor(v);
|
||||
const double diff = v - fl;
|
||||
if (diff > 0.5) {
|
||||
return static_cast<int>(fl) + 1;
|
||||
}
|
||||
if (diff < 0.5) {
|
||||
return static_cast<int>(fl);
|
||||
}
|
||||
const int lo = static_cast<int>(fl);
|
||||
return (lo % 2 == 0) ? lo : lo + 1;
|
||||
}
|
||||
};
|
||||
|
||||
// MiniCPM-OCR 1.0: find_best_resize aligns to patch_size * n_merge (28 in 4x).
|
||||
struct mtmd_image_preprocessor_minicpm_ocr_1_0 : mtmd_image_preprocessor_minicpmv {
|
||||
using mtmd_image_preprocessor_minicpmv::mtmd_image_preprocessor_minicpmv;
|
||||
|
||||
protected:
|
||||
int get_slice_align() const override {
|
||||
const int merge = hparams.n_merge > 0 ? hparams.n_merge : 2;
|
||||
return hparams.patch_size * merge;
|
||||
}
|
||||
};
|
||||
|
||||
// custom llava-uhd slicing logic for LFM2
|
||||
// tiling params (min/max tiles, tile size, pixels tolerance) come from hparams
|
||||
// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py
|
||||
struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {
|
||||
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
|
||||
|
||||
static bool should_tile(const clip_hparams & hparams, const clip_image_size & original_size);
|
||||
static clip_image_size smart_resize(const clip_hparams & hparams, const clip_image_size & original_size);
|
||||
static clip_image_size get_grid_layout(const clip_hparams & hparams, int height, int width);
|
||||
|
||||
private:
|
||||
static clip_image_size find_closest_aspect_ratio(
|
||||
const clip_hparams & hparams,
|
||||
double aspect_ratio,
|
||||
const std::vector<clip_image_size> & target_ratios,
|
||||
int width, int height);
|
||||
static std::vector<clip_image_size> get_target_ratios(const clip_hparams & hparams);
|
||||
};
|
||||
|
||||
struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd {
|
||||
mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
};
|
||||
|
||||
struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd {
|
||||
mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
};
|
||||
|
||||
// stretch the image to a grid of tiles, add a thumbnail if there is more than 1 tile
|
||||
// ref: https://github.com/huggingface/transformers/blob/main/src/transformers/models/cohere2_vision/image_processing_cohere2_vision.py
|
||||
struct mtmd_image_preprocessor_cohere2v : mtmd_image_preprocessor_llava_uhd {
|
||||
mtmd_image_preprocessor_cohere2v(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;
|
||||
};
|
||||
|
||||
// DeepSeek-OCR (v1/v2) global view + optional local tile grid
|
||||
struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx)
|
||||
: mtmd_image_preprocessor(ctx),
|
||||
fuse_row(clip_get_projector_type(ctx) == PROJECTOR_TYPE_DEEPSEEKOCR),
|
||||
base_size(hparams.image_size),
|
||||
tile_size(hparams.preproc_tile_size),
|
||||
min_tiles(hparams.preproc_min_tiles),
|
||||
max_tiles(hparams.preproc_max_tiles) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
|
||||
private:
|
||||
bool fuse_row; // v1 fuses a tile-row into one image; v2 keeps tiles separate
|
||||
int base_size; // global view
|
||||
int tile_size; // each tile
|
||||
int min_tiles;
|
||||
int max_tiles;
|
||||
|
||||
std::vector<clip_image_size> get_target_ratios() const;
|
||||
clip_image_size find_closest_aspect_ratio(
|
||||
float aspect_ratio,
|
||||
const std::vector<clip_image_size> & target_ratios,
|
||||
int width, int height) const;
|
||||
};
|
||||
|
||||
// custom image preprocessing for Step3VL
|
||||
// ref: https://huggingface.co/stepfun-ai/Step3-VL-10B/blob/main/processing_step3.py
|
||||
struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd {
|
||||
mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size);
|
||||
|
||||
private:
|
||||
static constexpr int default_image_longest_edge = 3024;
|
||||
static constexpr int default_image_crop_size = 504;
|
||||
static constexpr float small_aspect_ratio_limit = 1.5f;
|
||||
static constexpr float wide_aspect_ratio_limit = 4.0f;
|
||||
static constexpr float crop_rounding_threshold = 0.2f;
|
||||
|
||||
void img_u8_resize_bilinear_to_f32(
|
||||
const clip_image_u8 & src,
|
||||
clip_image_f32 & dst,
|
||||
int target_width,
|
||||
int target_height,
|
||||
const float mean[3],
|
||||
const float std[3]) const;
|
||||
static int get_image_longest_edge(const clip_hparams & params);
|
||||
static int determine_window_size(const clip_hparams & params, int longer, int shorter);
|
||||
static int calc_crop_extent(int length, int window_size);
|
||||
static std::vector<int> calc_grid(int length, int window_size);
|
||||
static clip_image_u8 prepare_image(const clip_image_u8 & img, const clip_hparams & params);
|
||||
static clip_image_u8 crop_with_black_padding(const clip_image_u8 & image, int x, int y, int w, int h);
|
||||
};
|
||||
|
||||
struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
};
|
||||
|
||||
// llava-next "anyres": stacks the overview and all tiles into one image, assembled by clip in a single graph
|
||||
struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd {
|
||||
mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
};
|
||||
|
||||
// pick the patch grid closest to the input aspect ratio under the per-image token cap, stretch-resize.
|
||||
struct mtmd_image_preprocessor_muse_glimmer : mtmd_image_preprocessor {
|
||||
mtmd_image_preprocessor_muse_glimmer(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
|
||||
mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
|
||||
};
|
||||
@@ -0,0 +1,21 @@
|
||||
#pragma once
|
||||
|
||||
#include "mtmd.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// !!! Internal header, to be used by mtmd and its unit tests only !!!
|
||||
|
||||
#define MTMD_INTERNAL_HEADER
|
||||
|
||||
// bitmap is null for text parts
|
||||
struct mtmd_internal_part {
|
||||
std::string text;
|
||||
const mtmd_bitmap * bitmap;
|
||||
// only used for text parts
|
||||
bool parse_special = false;
|
||||
};
|
||||
|
||||
// [QWEN_VIDEO] merged parts are erased from `parts`, so one group always maps to one part
|
||||
std::vector<std::vector<const mtmd_bitmap *>> mtmd_group_mergeable_bitmaps(std::vector<mtmd_internal_part> & parts, int n_merge);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,621 @@
|
||||
#ifndef MTMD_H
|
||||
#define MTMD_H
|
||||
|
||||
#include "ggml.h"
|
||||
#include "llama.h"
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <cinttypes>
|
||||
#include <memory>
|
||||
#endif
|
||||
|
||||
/**
|
||||
* libmtmd: A library for multimodal support in llama.cpp.
|
||||
*
|
||||
* WARNING: This API is experimental and subject to many BREAKING CHANGES.
|
||||
* Issues related to API usage may receive lower priority support.
|
||||
*
|
||||
* For the usage, see an example in mtmd-cli.cpp
|
||||
*
|
||||
* For contributors:
|
||||
* - Make sure the C API is aligned with the libllama C API (as in llama.h)
|
||||
* - Do not include model name (e.g., qwen, gemma) in the API, use generic terms instead
|
||||
* - Keep the API minimal, do not expose internal details unless necessary
|
||||
*
|
||||
* IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated.
|
||||
* We encourage human contributors to ensure the quality and reliability of the codebase.
|
||||
*/
|
||||
|
||||
#ifdef LLAMA_SHARED
|
||||
# if defined(_WIN32) && !defined(__MINGW32__)
|
||||
# ifdef LLAMA_BUILD
|
||||
# define MTMD_API __declspec(dllexport)
|
||||
# else
|
||||
# define MTMD_API __declspec(dllimport)
|
||||
# endif
|
||||
# else
|
||||
# define MTMD_API __attribute__ ((visibility ("default")))
|
||||
# endif
|
||||
#else
|
||||
# define MTMD_API
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
enum mtmd_input_chunk_type {
|
||||
MTMD_INPUT_CHUNK_TYPE_TEXT,
|
||||
MTMD_INPUT_CHUNK_TYPE_IMAGE,
|
||||
MTMD_INPUT_CHUNK_TYPE_AUDIO,
|
||||
MTMD_INPUT_CHUNK_TYPE_COUNT, // for validation
|
||||
};
|
||||
|
||||
// opaque types
|
||||
struct mtmd_context;
|
||||
struct mtmd_bitmap;
|
||||
struct mtmd_image_tokens;
|
||||
struct mtmd_input_chunk;
|
||||
struct mtmd_input_chunks;
|
||||
struct mtmd_batch;
|
||||
struct mtmd_gen_state;
|
||||
|
||||
struct mtmd_input_text {
|
||||
const char * text;
|
||||
size_t text_len;
|
||||
bool add_special;
|
||||
bool parse_special;
|
||||
};
|
||||
|
||||
struct mtmd_input_part {
|
||||
// only text or bitmap can be set, not both
|
||||
const struct mtmd_input_text * text;
|
||||
const struct mtmd_bitmap * bitmap;
|
||||
};
|
||||
|
||||
|
||||
//
|
||||
// C API
|
||||
//
|
||||
|
||||
typedef struct mtmd_context mtmd_context;
|
||||
typedef struct mtmd_bitmap mtmd_bitmap;
|
||||
typedef struct mtmd_image_tokens mtmd_image_tokens;
|
||||
typedef struct mtmd_input_chunk mtmd_input_chunk;
|
||||
typedef struct mtmd_input_chunks mtmd_input_chunks;
|
||||
typedef struct mtmd_input_text mtmd_input_text;
|
||||
typedef struct mtmd_input_part mtmd_input_part;
|
||||
typedef struct mtmd_batch mtmd_batch;
|
||||
typedef struct mtmd_gen_state mtmd_gen_state;
|
||||
|
||||
typedef bool (*mtmd_progress_callback)(float progress, void * user_data);
|
||||
|
||||
struct mtmd_context_params {
|
||||
bool use_gpu;
|
||||
ggml_backend_dev_t device;
|
||||
bool print_timings;
|
||||
int n_threads;
|
||||
const char * image_marker; // deprecated, use media_marker instead
|
||||
const char * media_marker;
|
||||
enum llama_flash_attn_type flash_attn_type;
|
||||
bool warmup; // whether to run a warmup encode pass after initialization
|
||||
|
||||
// limit number of image tokens, only for vision models with dynamic resolution
|
||||
int image_min_tokens; // minimum number of tokens for image input (default: read from metadata)
|
||||
int image_max_tokens; // maximum number of tokens for image input (default: read from metadata)
|
||||
|
||||
// callback function passed over to mtmd proper
|
||||
ggml_backend_sched_eval_callback cb_eval;
|
||||
void * cb_eval_user_data;
|
||||
|
||||
// batching params
|
||||
int32_t batch_max_tokens; // maximum number of output tokens in a batch
|
||||
// (note: this is not a hard-limit, the first image will always be added even if it exceeds this limit)
|
||||
// (default: 1024)
|
||||
|
||||
// Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
|
||||
// If the provided progress_callback returns true, model loading continues.
|
||||
// If it returns false, model loading is immediately aborted.
|
||||
mtmd_progress_callback progress_callback;
|
||||
void * progress_callback_user_data;
|
||||
};
|
||||
|
||||
MTMD_API const char * mtmd_default_marker(void);
|
||||
|
||||
MTMD_API struct mtmd_context_params mtmd_context_params_default(void);
|
||||
|
||||
// initialize the mtmd context
|
||||
// return nullptr on failure
|
||||
MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname,
|
||||
const struct llama_model * text_model,
|
||||
const struct mtmd_context_params ctx_params);
|
||||
|
||||
MTMD_API void mtmd_free(mtmd_context * ctx);
|
||||
|
||||
// whether we need to set non-causal mask before llama_decode
|
||||
// if chunk is nullptr, we assume the default case where chunk is an image chunk
|
||||
MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk);
|
||||
|
||||
// whether the current model use M-RoPE for llama_decode
|
||||
MTMD_API bool mtmd_decode_use_mrope(const mtmd_context * ctx);
|
||||
|
||||
// whether the current model supports vision input
|
||||
MTMD_API bool mtmd_support_vision(const mtmd_context * ctx);
|
||||
|
||||
// whether the current model supports audio input
|
||||
MTMD_API bool mtmd_support_audio(const mtmd_context * ctx);
|
||||
|
||||
// get audio sample rate in Hz, for example 16000 for Whisper
|
||||
// return -1 if audio is not supported
|
||||
MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx);
|
||||
|
||||
// get the current marker string
|
||||
MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx);
|
||||
|
||||
// mtmd_bitmap
|
||||
//
|
||||
// if bitmap is image:
|
||||
// length of data must be nx * ny * 3
|
||||
// the data is in RGBRGBRGB... format
|
||||
// note: some video-capable models (i.e. qwen-vl) can merge consecutive bitmaps
|
||||
// into one chunk; mtmd_tokenize() handles this, but remember to set
|
||||
// mtmd_bitmap_set_mergeable(true) for every frame
|
||||
// if bitmap is audio:
|
||||
// length of data must be n_samples * sizeof(float)
|
||||
// the data is in float format (PCM F32)
|
||||
//
|
||||
// if data == nullptr:
|
||||
// the bitmap is considered "empty", and will be treated as a placeholder for counting tokens
|
||||
// you can pass the bitmap via mtmd_tokenize(), then call mtmd_*_get_n_tokens() to count the tokens
|
||||
// note: passing a placeholder bitmap to mtmd_encode() will return an error
|
||||
MTMD_API mtmd_bitmap * mtmd_bitmap_init (uint32_t nx, uint32_t ny, const unsigned char * data);
|
||||
MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data);
|
||||
MTMD_API uint32_t mtmd_bitmap_get_nx (const mtmd_bitmap * bitmap);
|
||||
MTMD_API uint32_t mtmd_bitmap_get_ny (const mtmd_bitmap * bitmap);
|
||||
MTMD_API const unsigned char * mtmd_bitmap_get_data (const mtmd_bitmap * bitmap);
|
||||
MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap);
|
||||
MTMD_API bool mtmd_bitmap_is_audio (const mtmd_bitmap * bitmap);
|
||||
MTMD_API void mtmd_bitmap_free (mtmd_bitmap * bitmap);
|
||||
// bitmap ID is optional, but useful for KV cache tracking
|
||||
// these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data()
|
||||
MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap);
|
||||
MTMD_API void mtmd_bitmap_set_id(mtmd_bitmap * bitmap, const char * id);
|
||||
// if true, this bitmap can be merged (temporal merge) with an adjacent mergeable bitmap by certain video input models
|
||||
MTMD_API void mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool mergeable);
|
||||
|
||||
// mtmd_bitmap lazy
|
||||
//
|
||||
// this is a special bitmap that:
|
||||
// - does not hold the actual data
|
||||
// - can be expanded into one or more chunks (either media to text chunks)
|
||||
// user must provide a callback to fill in the data when mtmd_tokenize() is called
|
||||
// this is useful for large video inputs:
|
||||
// - allow reading video frame by frame, without loading the entire video into memory
|
||||
// - allow tracking the whole video with a single ID (for example, the file hash)
|
||||
|
||||
// set (*out_bitmap) to non-nullptr to emit a bitmap chunk; it will be freed automatically
|
||||
// set (*out_text) to non-nullptr to emit a text chunk; it must be heap-allocated, null-terminated and will be freed automatically
|
||||
// either out_bitmap or out_text can be set, but not both
|
||||
// out_bitmap cannot be another lazy bitmap (no nested lazy allowed)
|
||||
// return value:
|
||||
// 0 on success
|
||||
// -1 on EOF (signal to mtmd_tokenize to move on)
|
||||
// -2 on error (signal to mtmd_tokenize to abort)
|
||||
typedef int(* mtmd_bitmap_lazy_callback)(
|
||||
size_t chunk_idx,
|
||||
void * user_data,
|
||||
mtmd_bitmap ** out_bitmap,
|
||||
char ** out_text);
|
||||
|
||||
MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx,
|
||||
const char * id, // usually set to file hash
|
||||
void * user_data,
|
||||
mtmd_bitmap_lazy_callback callback);
|
||||
|
||||
// mtmd_input_chunks
|
||||
//
|
||||
// this is simply a list of mtmd_input_chunk
|
||||
// the elements can only be populated via mtmd_tokenize()
|
||||
MTMD_API mtmd_input_chunks * mtmd_input_chunks_init(void);
|
||||
MTMD_API size_t mtmd_input_chunks_size(const mtmd_input_chunks * chunks);
|
||||
MTMD_API const mtmd_input_chunk * mtmd_input_chunks_get (const mtmd_input_chunks * chunks, size_t idx);
|
||||
MTMD_API void mtmd_input_chunks_free(mtmd_input_chunks * chunks);
|
||||
|
||||
// mtmd_input_chunk
|
||||
//
|
||||
// the instance will be constructed via mtmd_tokenize()
|
||||
// it will be freed along with mtmd_input_chunks
|
||||
MTMD_API enum mtmd_input_chunk_type mtmd_input_chunk_get_type (const mtmd_input_chunk * chunk);
|
||||
MTMD_API const llama_token * mtmd_input_chunk_get_tokens_text (const mtmd_input_chunk * chunk, size_t * n_tokens_output);
|
||||
MTMD_API const mtmd_image_tokens * mtmd_input_chunk_get_tokens_image(const mtmd_input_chunk * chunk);
|
||||
MTMD_API size_t mtmd_input_chunk_get_n_tokens (const mtmd_input_chunk * chunk);
|
||||
// returns nullptr for ID on text chunk
|
||||
MTMD_API const char * mtmd_input_chunk_get_id (const mtmd_input_chunk * chunk);
|
||||
// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise)
|
||||
MTMD_API llama_pos mtmd_input_chunk_get_n_pos (const mtmd_input_chunk * chunk);
|
||||
|
||||
// in case you want to use custom logic to handle the chunk (i.e. KV cache management)
|
||||
// you can move the chunk ownership to your own code by copying it
|
||||
// remember to free the chunk when you are done with it
|
||||
MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk);
|
||||
MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk);
|
||||
|
||||
// similar to mtmd_input_chunk_copy, but returns a placeholder chunk
|
||||
MTMD_API mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk);
|
||||
|
||||
// save/load an input chunk to/from a buffer (useful for KV save/load)
|
||||
// important: only chunk's metadata will be saved, the actual image/audio data will not be saved
|
||||
// the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode()
|
||||
// out_buf can be nullptr (to query expected_out_len)
|
||||
// returns 0 on success, non-zero on failure
|
||||
MTMD_API int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len);
|
||||
// returns nullptr on failure
|
||||
MTMD_API mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len);
|
||||
|
||||
|
||||
// mtmd_image_tokens
|
||||
//
|
||||
// the instance will be constructed via mtmd_tokenize()
|
||||
// it will be freed along with mtmd_input_chunk
|
||||
MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); // TODO: deprecate
|
||||
MTMD_API const char * mtmd_image_tokens_get_id (const mtmd_image_tokens * image_tokens); // TODO: deprecate
|
||||
// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise)
|
||||
MTMD_API llama_pos mtmd_image_tokens_get_n_pos (const mtmd_image_tokens * image_tokens); // TODO: deprecate
|
||||
|
||||
DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_nx(const mtmd_image_tokens * image_tokens),
|
||||
"use mtmd_image_tokens_get_decoder_pos() instead");
|
||||
DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens),
|
||||
"use mtmd_image_tokens_get_decoder_pos() instead");
|
||||
|
||||
struct mtmd_decoder_pos {
|
||||
uint32_t t;
|
||||
uint32_t x;
|
||||
uint32_t y;
|
||||
uint32_t z; // unused for now, reserved for future use
|
||||
};
|
||||
// get position for decoder attention, to be used by M-RoPE models
|
||||
// i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1
|
||||
// pos_0 is the absolute position of the first token
|
||||
// return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position)
|
||||
MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i);
|
||||
|
||||
// tokenize an input text prompt and a list of bitmaps (image/audio)
|
||||
// the prompt must have the input media marker (default: "<__media__>") in it
|
||||
// the default marker is defined by mtmd_default_marker()
|
||||
// the marker will be replaced with the media chunk
|
||||
// for example:
|
||||
// "here is an image: <__media__>\ndescribe it in detail."
|
||||
// this will gives 3 chunks:
|
||||
// 1. "here is an image: <start_of_image>"
|
||||
// 2. (image/audio tokens)
|
||||
// 3. "<end_of_image>\ndescribe it in detail."
|
||||
// number of bitmaps must be equal to the number of markers in the prompt
|
||||
// this function is thread-safe (shared ctx)
|
||||
// return values:
|
||||
// 0 on success
|
||||
// 1 on number of bitmaps not matching the number of markers
|
||||
// 2 on media preprocessing error
|
||||
MTMD_API int32_t mtmd_tokenize(const mtmd_context * ctx,
|
||||
mtmd_input_chunks * output,
|
||||
const mtmd_input_text * text,
|
||||
const mtmd_bitmap * const * bitmaps,
|
||||
size_t n_bitmaps);
|
||||
|
||||
// same as mtmd_tokenize(), but takes an array of mtmd_input_part
|
||||
// use cases:
|
||||
// - when you don't want to use media markers (they will be tokenized as normal text)
|
||||
// - when you want to control parse_special for each text part
|
||||
// note: per-part add_special will be ignored
|
||||
// return 1 if a part has both text and bitmap set (or neither)
|
||||
MTMD_API int32_t mtmd_tokenize_from_parts(const mtmd_context * ctx,
|
||||
mtmd_input_chunks * output,
|
||||
const mtmd_input_part * const * parts,
|
||||
size_t n_parts,
|
||||
bool add_special);
|
||||
|
||||
DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens),
|
||||
"use mtmd_encode_chunk() instead");
|
||||
|
||||
// text chunk will be ignored silently, only media chunk will be encoded
|
||||
// returns 0 on success
|
||||
// returns 1 on generic error
|
||||
MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx,
|
||||
const mtmd_input_chunk * chunk);
|
||||
|
||||
// get output embeddings from the last encode pass
|
||||
// the reading size (in bytes) is equal to:
|
||||
// llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float)
|
||||
MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx);
|
||||
|
||||
|
||||
// batch encoding API
|
||||
// chunks are not owned by the batch, they will not be freed by mtmd_batch_free()
|
||||
// batch is valid for a given context, cannot be shared across contexts
|
||||
MTMD_API mtmd_batch * mtmd_batch_init(mtmd_context * ctx);
|
||||
MTMD_API void mtmd_batch_free(mtmd_batch * batch);
|
||||
|
||||
// only media chunks are allowed, text chunks will be rejected
|
||||
// returns 0 on success
|
||||
// returns 1 on generic error
|
||||
// returns 2 if the batch is too large (chunk won't be added)
|
||||
// returns 3 if it cannot be batched with the existing chunks in the batch
|
||||
MTMD_API int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk);
|
||||
|
||||
// returns 0 on success
|
||||
// returns 1 on generic error
|
||||
MTMD_API int32_t mtmd_batch_encode(mtmd_batch * batch);
|
||||
MTMD_API float * mtmd_batch_get_output_embd(mtmd_batch * batch, const mtmd_input_chunk * chunk);
|
||||
|
||||
|
||||
// Set callback for all future logging events.
|
||||
// If this is not called, or NULL is supplied, everything is output on stderr.
|
||||
MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);
|
||||
|
||||
// EXPERIMENTAL API to get mmproj's capabilities without initializing the full context
|
||||
// This is only intended to be used by llama-server, breaking changes is expected
|
||||
struct mtmd_caps {
|
||||
bool inp_vision;
|
||||
bool inp_audio;
|
||||
bool gen_audio;
|
||||
};
|
||||
MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname);
|
||||
|
||||
/////////////////////////////////////////
|
||||
// EXPERIMENTAL API for realtime audio input, subjected to breaking changes
|
||||
//
|
||||
// a stream turns incoming PCM into decoder-ready embedding rows, one row per decoder position
|
||||
// it keeps the encoder state between calls, so audio can be fed as it arrives
|
||||
// how text tokens are mixed into the rows is a model detail hidden inside the stream
|
||||
// the text model given to mtmd_init_from_file() is required
|
||||
|
||||
// whether the current model supports mtmd_audio_stream
|
||||
MTMD_API bool mtmd_support_audio_stream(const mtmd_context * ctx);
|
||||
|
||||
struct mtmd_audio_stream;
|
||||
typedef struct mtmd_audio_stream mtmd_audio_stream;
|
||||
|
||||
struct mtmd_audio_stream_params {
|
||||
int32_t step_ms; // audio encoded per encoder call, rounded up to the model step (default: 320)
|
||||
};
|
||||
MTMD_API struct mtmd_audio_stream_params mtmd_audio_stream_params_default(void);
|
||||
|
||||
// returns nullptr on failure
|
||||
MTMD_API mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, struct mtmd_audio_stream_params params);
|
||||
MTMD_API void mtmd_audio_stream_free(mtmd_audio_stream * stream);
|
||||
|
||||
// samples are PCM F32 at mtmd_get_audio_sample_rate()
|
||||
// returns 0 on success, 1 if the stream has ended
|
||||
MTMD_API int32_t mtmd_audio_stream_push(mtmd_audio_stream * stream, const float * samples, size_t n_samples);
|
||||
|
||||
// pad the stream so that all pending rows can be produced
|
||||
// end = false: the stream stays open (use it on a pause, to get a partial transcript)
|
||||
// end = true: no more push() is accepted
|
||||
// returns 0 on success, 1 if the stream has ended
|
||||
MTMD_API int32_t mtmd_audio_stream_flush(mtmd_audio_stream * stream, bool end);
|
||||
|
||||
// produce the next decoder rows
|
||||
// last_token is the token sampled from the previous row, LLAMA_TOKEN_NULL before the first sample
|
||||
// return value:
|
||||
// > 0 number of rows ready, read them with mtmd_audio_stream_get_output_embd()
|
||||
// 0 more audio is needed
|
||||
// -1 end of stream, nothing more will be produced
|
||||
// -2 error
|
||||
MTMD_API int32_t mtmd_audio_stream_step(mtmd_audio_stream * stream, llama_token last_token);
|
||||
|
||||
// rows of the last step, the reading size (in bytes) is n_rows * llama_model_n_embd_inp(model) * sizeof(float)
|
||||
MTMD_API const float * mtmd_audio_stream_get_output_embd(const mtmd_audio_stream * stream);
|
||||
|
||||
// rows produced so far, the prefix rows included
|
||||
MTMD_API int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * stream);
|
||||
|
||||
// rows the stream produces up to the end of the last flush
|
||||
// once get_n_rows() reaches it, every row of the flushed audio has been produced
|
||||
MTMD_API int64_t mtmd_audio_stream_get_n_rows_flushed(const mtmd_audio_stream * stream);
|
||||
|
||||
/////////////////////////////////////////
|
||||
// EXPERIMENTAL API for audio generation, subjected to breaking changes
|
||||
|
||||
// represent the pipeline type
|
||||
enum mtmd_gen_audio_type {
|
||||
MTMD_GEN_AUDIO_TYPE_NONE, // not supported
|
||||
MTMD_GEN_AUDIO_TYPE_QWEN3TTS,
|
||||
MTMD_GEN_AUDIO_TYPE_POCKETTTS,
|
||||
MTMD_GEN_AUDIO_TYPE_SOPRANO,
|
||||
};
|
||||
|
||||
struct mtmd_gen_audio_info {
|
||||
enum mtmd_gen_audio_type type;
|
||||
int32_t sample_rate; // in Hz, for example 24000 for qwen3tts
|
||||
const char * model_variant; // name of the weight variant, can be nullptr if not applicable
|
||||
};
|
||||
|
||||
MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx);
|
||||
|
||||
|
||||
enum mtmd_gen_process_type {
|
||||
MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.)
|
||||
MTMD_GEN_PROCESS_TYPE_GEN_WAV, // convert semantic to PCM audio
|
||||
// for qwen3tts, this is code2wav
|
||||
// for pocket-tts, this is mimi decoder
|
||||
};
|
||||
|
||||
struct mtmd_gen_inp {
|
||||
enum mtmd_gen_process_type type;
|
||||
|
||||
// for MTMD_GEN_PROCESS_TYPE_GEN_CODE
|
||||
int32_t code0; // the sampled codebook 0 entry from backbone
|
||||
float * embd; // the hidden state from backbone, must have n_text_embd elements
|
||||
int32_t top_k;
|
||||
float top_p;
|
||||
float temp; // sampling temperature, or noise scale for flow-matching decoders
|
||||
|
||||
// for MTMD_GEN_PROCESS_TYPE_GEN_WAV
|
||||
// pass either codes (discrete) or feats (continuous), depending on the pipeline
|
||||
int32_t * codes;
|
||||
size_t n_codes;
|
||||
const float * feats;
|
||||
size_t n_feats;
|
||||
};
|
||||
|
||||
struct mtmd_gen_out {
|
||||
// note: output memory is owned by the state, valid until the next process() call with this state
|
||||
|
||||
// for MTMD_GEN_PROCESS_TYPE_GEN_CODE
|
||||
const int32_t * codes;
|
||||
size_t n_codes;
|
||||
const float * feats; // continuous counterpart of codes
|
||||
size_t n_feats;
|
||||
const float * embd; // the generated hidden state, to be fed back to backbone
|
||||
// it must have n_text_embd elements
|
||||
bool is_eos; // only set by pipelines having the EOS head inside mmproj
|
||||
|
||||
// for MTMD_GEN_PROCESS_TYPE_GEN_WAV
|
||||
const float * audio;
|
||||
size_t n_samples;
|
||||
};
|
||||
|
||||
struct mtmd_gen_state_params {
|
||||
uint32_t seed; // UINT32_MAX for random
|
||||
};
|
||||
|
||||
MTMD_API struct mtmd_gen_state_params mtmd_gen_state_params_default(void);
|
||||
|
||||
// returns nullptr if the model does not support audio generation
|
||||
MTMD_API mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, struct mtmd_gen_state_params params);
|
||||
MTMD_API void mtmd_gen_state_free(mtmd_gen_state * state);
|
||||
|
||||
// clear the decoder state (cold start), keep the RNG stream
|
||||
MTMD_API void mtmd_gen_state_reset(mtmd_gen_state * state);
|
||||
|
||||
// defaults tuned for the loaded pipeline, callers override only what they care about
|
||||
MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx);
|
||||
|
||||
// run one step and update the state in place
|
||||
// the caller drives the backbone and collects the audio frames
|
||||
MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx,
|
||||
mtmd_gen_state * state,
|
||||
const struct mtmd_gen_inp * inp,
|
||||
struct mtmd_gen_out * out);
|
||||
|
||||
/////////////////////////////////////////
|
||||
|
||||
// test function, to be used in test-mtmd-c-api.c
|
||||
MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern "C"
|
||||
#endif
|
||||
|
||||
// Get memory usage of the current model in bytes, per backend device
|
||||
// Note: this is an unstable API, used internally by fit_params; it WILL be removed or changed without deprecation
|
||||
#ifdef __cplusplus
|
||||
struct mtmd_memory_usage {
|
||||
std::map<ggml_backend_dev_t, size_t> backend_mem_usage;
|
||||
// for models that use non-causal attention, max_tokens must not exceed n_ubatch of llama_context
|
||||
int image_max_tokens;
|
||||
bool use_non_causal;
|
||||
};
|
||||
MTMD_API struct mtmd_memory_usage mtmd_get_memory_usage(
|
||||
const char * mmproj_fname,
|
||||
struct mtmd_context_params ctx_params);
|
||||
#endif
|
||||
|
||||
//
|
||||
// C++ wrappers
|
||||
//
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
||||
namespace mtmd {
|
||||
|
||||
struct mtmd_context_deleter {
|
||||
void operator()(mtmd_context * val) { mtmd_free(val); }
|
||||
};
|
||||
using context_ptr = std::unique_ptr<mtmd_context, mtmd_context_deleter>;
|
||||
|
||||
struct mtmd_bitmap_deleter {
|
||||
void operator()(mtmd_bitmap * val) { mtmd_bitmap_free(val); }
|
||||
};
|
||||
using bitmap_ptr = std::unique_ptr<mtmd_bitmap, mtmd_bitmap_deleter>;
|
||||
|
||||
struct mtmd_input_chunks_deleter {
|
||||
void operator()(mtmd_input_chunks * val) { mtmd_input_chunks_free(val); }
|
||||
};
|
||||
using input_chunks_ptr = std::unique_ptr<mtmd_input_chunks, mtmd_input_chunks_deleter>;
|
||||
|
||||
struct mtmd_input_chunk_deleter {
|
||||
void operator()(mtmd_input_chunk * val) { mtmd_input_chunk_free(val); }
|
||||
};
|
||||
using input_chunk_ptr = std::unique_ptr<mtmd_input_chunk, mtmd_input_chunk_deleter>;
|
||||
|
||||
struct mtmd_batch_deleter {
|
||||
void operator()(mtmd_batch * val) { mtmd_batch_free(val); }
|
||||
};
|
||||
using batch_ptr = std::unique_ptr<mtmd_batch, mtmd_batch_deleter>;
|
||||
|
||||
struct mtmd_audio_stream_deleter {
|
||||
void operator()(mtmd_audio_stream * val) { mtmd_audio_stream_free(val); }
|
||||
};
|
||||
using audio_stream_ptr = std::unique_ptr<mtmd_audio_stream, mtmd_audio_stream_deleter>;
|
||||
|
||||
struct mtmd_gen_state_deleter {
|
||||
void operator()(mtmd_gen_state * val) { mtmd_gen_state_free(val); }
|
||||
};
|
||||
using gen_state_ptr = std::unique_ptr<mtmd_gen_state, mtmd_gen_state_deleter>;
|
||||
|
||||
struct bitmap {
|
||||
bitmap_ptr ptr;
|
||||
bitmap() : ptr(nullptr) {}
|
||||
bitmap(mtmd_bitmap * bitmap) : ptr(bitmap) {}
|
||||
bitmap(bitmap && other) noexcept : ptr(std::move(other.ptr)) {}
|
||||
bitmap(uint32_t nx, uint32_t ny, const unsigned char * data) {
|
||||
ptr.reset(mtmd_bitmap_init(nx, ny, data));
|
||||
}
|
||||
~bitmap() = default;
|
||||
uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); }
|
||||
uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); }
|
||||
const unsigned char * data() const { return mtmd_bitmap_get_data(ptr.get()); }
|
||||
size_t n_bytes() const { return mtmd_bitmap_get_n_bytes(ptr.get()); }
|
||||
std::string id() const { return mtmd_bitmap_get_id(ptr.get()); }
|
||||
void set_id(const char * id) const { mtmd_bitmap_set_id(ptr.get(), id); }
|
||||
};
|
||||
|
||||
struct bitmaps {
|
||||
std::vector<bitmap> entries;
|
||||
~bitmaps() = default;
|
||||
// return list of pointers to mtmd_bitmap
|
||||
// example:
|
||||
// auto bitmaps_c_ptr = bitmaps.c_ptr();
|
||||
// int32_t res = mtmd_tokenize(... bitmaps_c_ptr.data(), bitmaps_c_ptr.size());
|
||||
std::vector<const mtmd_bitmap *> c_ptr() {
|
||||
std::vector<const mtmd_bitmap *> res(entries.size());
|
||||
for (size_t i = 0; i < entries.size(); i++) {
|
||||
res[i] = entries[i].ptr.get();
|
||||
}
|
||||
return res;
|
||||
}
|
||||
};
|
||||
|
||||
struct input_chunks {
|
||||
input_chunks_ptr ptr;
|
||||
input_chunks() = default;
|
||||
input_chunks(mtmd_input_chunks * chunks) : ptr(chunks) {}
|
||||
~input_chunks() = default;
|
||||
size_t size() const { return mtmd_input_chunks_size(ptr.get()); }
|
||||
const mtmd_input_chunk * operator[](size_t idx) const {
|
||||
return mtmd_input_chunks_get(ptr.get(), idx);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace mtmd
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,6 @@
|
||||
-r ../../requirements/requirements-convert_legacy_llama.txt
|
||||
--extra-index-url https://download.pytorch.org/whl/cpu
|
||||
pillow~=11.3.0
|
||||
|
||||
torch==2.11.0 # check_requirements: ignore "=="
|
||||
torchvision==0.26.0; platform_machine != "s390x" # check_requirements: ignore "=="
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 121 KiB |
Binary file not shown.
Binary file not shown.
Executable
+233
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# make sure we are in the right directory
|
||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
cd $SCRIPT_DIR
|
||||
|
||||
#export LLAMA_CACHE="$SCRIPT_DIR/tmp"
|
||||
|
||||
set -eux
|
||||
|
||||
mkdir -p $SCRIPT_DIR/output
|
||||
|
||||
PROJ_ROOT="$SCRIPT_DIR/../.."
|
||||
cd $PROJ_ROOT
|
||||
|
||||
export MTMD_TEST_RESPONSE_MARKER="<MTMD_TEST_RESPONSE_MARKER>"
|
||||
|
||||
# Check if the first argument is "big", then run test with big models
|
||||
# This is useful if we're running the script on a larger machine, so we can test the big models
|
||||
RUN_BIG_TESTS=false
|
||||
if [ "${1:-}" = "big" ]; then
|
||||
RUN_BIG_TESTS=true
|
||||
echo "Include BIG models..."
|
||||
fi
|
||||
|
||||
RUN_HUGE_TESTS=false
|
||||
if [ "${1:-}" = "huge" ]; then
|
||||
RUN_HUGE_TESTS=true
|
||||
RUN_BIG_TESTS=true
|
||||
echo "Include BIG and HUGE models..."
|
||||
fi
|
||||
|
||||
USE_VIDEO=false
|
||||
if [ "${1:-}" = "video" ]; then
|
||||
USE_VIDEO=true
|
||||
echo "Using video as input..."
|
||||
# behavior of USE_VIDEO:
|
||||
# do NOT check if the output contains "new york", only verify if the exit code is 0
|
||||
# when printing the result, print the OK/FAIL line then print the generated text
|
||||
fi
|
||||
|
||||
# Check if the second argument is "flash", then enable flash attention
|
||||
# This is useful to test if flash attention off works correctly
|
||||
FLASH_ATTN="on"
|
||||
if [ "${2:-}" = "flash_off" ] || [ "${1:-}" = "flash_off" ]; then
|
||||
FLASH_ATTN="off"
|
||||
echo "Flash attention disabled..."
|
||||
fi
|
||||
|
||||
###############
|
||||
|
||||
arr_prefix=()
|
||||
arr_hf=()
|
||||
arr_extra_args=()
|
||||
arr_file=()
|
||||
|
||||
add_test_vision() {
|
||||
local hf=$1
|
||||
shift
|
||||
local extra_args=""
|
||||
if [ $# -gt 0 ]; then
|
||||
extra_args=$(printf " %q" "$@")
|
||||
fi
|
||||
if [ "$USE_VIDEO" = true ]; then
|
||||
arr_file+=("test-3.mp4")
|
||||
else
|
||||
arr_file+=("test-1.jpeg")
|
||||
fi
|
||||
arr_prefix+=("[vision]")
|
||||
arr_hf+=("$hf")
|
||||
arr_extra_args+=("$extra_args")
|
||||
}
|
||||
|
||||
add_test_audio() {
|
||||
if [ "$USE_VIDEO" = true ]; then
|
||||
return 0
|
||||
fi
|
||||
local hf=$1
|
||||
shift
|
||||
local extra_args=""
|
||||
if [ $# -gt 0 ]; then
|
||||
extra_args=$(printf " %q" "$@")
|
||||
fi
|
||||
arr_prefix+=("[audio] ")
|
||||
arr_hf+=("$hf")
|
||||
arr_extra_args+=("$extra_args")
|
||||
arr_file+=("test-2.mp3")
|
||||
}
|
||||
|
||||
add_test_vision "ggml-org/SmolVLM-500M-Instruct-GGUF:Q8_0"
|
||||
add_test_vision "ggml-org/SmolVLM2-2.2B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/SmolVLM2-500M-Video-Instruct-GGUF:Q8_0"
|
||||
add_test_vision "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M"
|
||||
add_test_vision "THUDM/glm-edge-v-5b-gguf:Q4_K_M" -p "name of the newspaper?<__media__>"
|
||||
add_test_vision "second-state/Llava-v1.5-7B-GGUF:Q2_K" --chat-template vicuna
|
||||
add_test_vision "cjpais/llava-1.6-mistral-7b-gguf:Q3_K_M" --chat-template vicuna
|
||||
add_test_vision "ibm-research/granite-vision-3.2-2b-GGUF:Q4_K_M"
|
||||
add_test_vision "second-state/MiniCPM-Llama3-V-2_5-GGUF:Q2_K" # model from openbmb is corrupted
|
||||
add_test_vision "openbmb/MiniCPM-V-2_6-gguf:Q2_K"
|
||||
add_test_vision "openbmb/MiniCPM-o-2_6-gguf:Q4_0"
|
||||
add_test_vision "bartowski/Qwen2-VL-2B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/Qwen2.5-VL-3B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/InternVL2_5-1B-GGUF:Q8_0"
|
||||
add_test_vision "ggml-org/InternVL3-1B-Instruct-GGUF:Q8_0"
|
||||
add_test_vision "ggml-org/Qwen2.5-Omni-3B-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/LFM2-VL-450M-GGUF:Q8_0"
|
||||
add_test_vision "ggml-org/granite-docling-258M-GGUF:Q8_0"
|
||||
add_test_vision "ggml-org/LightOnOCR-1B-1025-GGUF:Q8_0"
|
||||
add_test_vision "ggml-org/DeepSeek-OCR-GGUF:Q8_0" -p "Free OCR." --chat-template deepseek-ocr
|
||||
add_test_vision "ggml-org/dots.ocr-GGUF:Q8_0" -p "OCR"
|
||||
add_test_vision "ggml-org/HunyuanOCR-GGUF:Q8_0" -p "OCR"
|
||||
add_test_vision "ggml-org/gemma-4-E2B-it-GGUF:Q8_0" --jinja
|
||||
|
||||
add_test_audio "ggml-org/ultravox-v0_5-llama-3_2-1b-GGUF:Q8_0"
|
||||
add_test_audio "ggml-org/Qwen2.5-Omni-3B-GGUF:Q4_K_M"
|
||||
add_test_audio "ggml-org/Voxtral-Mini-3B-2507-GGUF:Q4_K_M"
|
||||
add_test_audio "ggml-org/LFM2-Audio-1.5B-GGUF:Q8_0"
|
||||
add_test_audio "ggml-org/gemma-4-E2B-it-GGUF:Q8_0" --jinja
|
||||
add_test_audio "ggml-org/Qwen3-ASR-0.6B-GGUF:Q8_0"
|
||||
|
||||
# to test the big models, run: ./tests.sh big
|
||||
if [ "$RUN_BIG_TESTS" = true ]; then
|
||||
add_test_vision "ggml-org/pixtral-12b-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/Mistral-Small-3.1-24B-Instruct-2503-GGUF" --chat-template mistral-v7
|
||||
add_test_vision "ggml-org/Qwen2-VL-2B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/Qwen2-VL-7B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/Qwen2.5-VL-3B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/Qwen2.5-VL-7B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/Qwen3-VL-2B-Instruct-GGUF:Q8_0"
|
||||
add_test_vision "ggml-org/InternVL3-8B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/InternVL3-14B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/Qwen2.5-Omni-7B-GGUF:Q4_K_M"
|
||||
# add_test_vision "ggml-org/Qwen2.5-VL-32B-Instruct-GGUF:Q4_K_M" # does not work on my mac M3 Ultra
|
||||
# add_test_vision "ggml-org/Kimi-VL-A3B-Thinking-2506-GGUF:Q4_K_M" # not always working
|
||||
add_test_vision "ggml-org/GLM-4.6V-Flash-GGUF:Q4_K_M" -p "extract all texts from this image"
|
||||
|
||||
add_test_audio "ggml-org/ultravox-v0_5-llama-3_1-8b-GGUF:Q4_K_M"
|
||||
add_test_audio "ggml-org/Qwen2.5-Omni-7B-GGUF:Q4_K_M"
|
||||
fi
|
||||
|
||||
# to test the huge models, run: ./tests.sh huge
|
||||
# this will run both the big and huge models
|
||||
# huge models are > 32B parameters
|
||||
if [ "$RUN_HUGE_TESTS" = true ]; then
|
||||
add_test_vision "ggml-org/Qwen2.5-VL-72B-Instruct-GGUF:Q4_K_M"
|
||||
add_test_vision "ggml-org/Llama-4-Scout-17B-16E-Instruct-GGUF:IQ1_S"
|
||||
fi
|
||||
|
||||
# these models always give the wrong answer, not sure why
|
||||
# add_test_vision "ggml-org/SmolVLM-Instruct-GGUF:Q4_K_M"
|
||||
# add_test_vision "ggml-org/SmolVLM-256M-Instruct-GGUF:Q8_0"
|
||||
# add_test_vision "ggml-org/SmolVLM2-256M-Video-Instruct-GGUF:Q8_0"
|
||||
|
||||
# this model has broken chat template, not usable
|
||||
# add_test_vision "cmp-nct/Yi-VL-6B-GGUF:Q5_K"
|
||||
# add_test_vision "guinmoon/MobileVLM-3B-GGUF:Q4_K_M" "deepseek"
|
||||
|
||||
###############
|
||||
|
||||
cmake --build build -j --target llama-mtmd-cli
|
||||
|
||||
arr_res=()
|
||||
|
||||
for i in "${!arr_hf[@]}"; do
|
||||
bin="llama-mtmd-cli"
|
||||
prefix="${arr_prefix[$i]}"
|
||||
hf="${arr_hf[$i]}"
|
||||
extra_args="${arr_extra_args[$i]}"
|
||||
inp_file="${arr_file[$i]}"
|
||||
|
||||
echo "Running test with binary: $bin and HF model: $hf"
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
cmd="$(printf %q "$PROJ_ROOT/build/bin/$bin") \
|
||||
-hf $(printf %q "$hf") \
|
||||
--image $(printf %q "$SCRIPT_DIR/$inp_file") \
|
||||
--temp 0 -n 128 \
|
||||
--flash-attn $(printf %q "$FLASH_ATTN") \
|
||||
${extra_args}"
|
||||
|
||||
# if extra_args does not contain -p, we add a default prompt
|
||||
if ! [[ "$extra_args" =~ "-p" ]]; then
|
||||
cmd+=" -p \"what is the publisher name of the newspaper?\""
|
||||
fi
|
||||
|
||||
exit_code=0
|
||||
output=$(eval "$cmd" 2>&1 | tee /dev/tty) || exit_code=$?
|
||||
|
||||
echo "$output" > $SCRIPT_DIR/output/$bin-$(echo "$hf" | tr '/' '-').log
|
||||
|
||||
if [ "$USE_VIDEO" = true ]; then
|
||||
# for video, only check exit code; do not grep for "new york"
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
result="$prefix \033[32mOK\033[0m: $hf"
|
||||
else
|
||||
result="$prefix \033[31mFAIL\033[0m: $hf"
|
||||
fi
|
||||
# append generated text (after the response marker)
|
||||
generated_text=$(echo "$output" | sed "1,/${MTMD_TEST_RESPONSE_MARKER}/d" | tail -10)
|
||||
if [ -n "$generated_text" ]; then
|
||||
result+="\n$generated_text"
|
||||
fi
|
||||
echo -e "$result"
|
||||
else
|
||||
# either contains "new york" or both "men" and "walk"
|
||||
if echo "$output" | grep -iq "new york" \
|
||||
|| (echo "$output" | grep -iq "men" && echo "$output" | grep -iq "walk")
|
||||
then
|
||||
result="$prefix \033[32mOK\033[0m: $hf"
|
||||
else
|
||||
result="$prefix \033[31mFAIL\033[0m: $hf"
|
||||
fi
|
||||
echo -e "$result"
|
||||
fi
|
||||
arr_res+=("$result")
|
||||
|
||||
echo ""
|
||||
echo ""
|
||||
echo ""
|
||||
echo "#################################################"
|
||||
echo "#################################################"
|
||||
echo ""
|
||||
echo ""
|
||||
done
|
||||
|
||||
set +x
|
||||
|
||||
for i in "${!arr_res[@]}"; do
|
||||
echo -e "${arr_res[$i]}"
|
||||
done
|
||||
echo ""
|
||||
echo "Output logs are saved in $SCRIPT_DIR/output"
|
||||
@@ -0,0 +1,24 @@
|
||||
|
||||
A Powdery Surface
|
||||
Is Closely Explored
|
||||
|
||||
By JOHN NOBLE WILFORD
|
||||
Special to The New York Times
|
||||
|
||||
HOUSTON, Monday, July 21—Men have landed and walked on the moon.
|
||||
|
||||
Two Americans, astronauts of Apollo 11, steered their fragile four-legged lunar module safely and smoothly to the historic landing yesterday at 4:17:40 P.M., Eastern daylight time.
|
||||
|
||||
Neil A. Armstrong, the 38-year-old civilian commander, radioed to earth and the mission control room here:
|
||||
|
||||
"Houston, Tranquility Base here. The Eagle has landed."
|
||||
|
||||
The first men to reach the moon—Mr. Armstrong and his co-pilot, Col. Edwin E. Aldrin Jr. of the Air Force—brought their ship to rest on a level, rock-strewn plain near the southwestern shore of the arid Sea of Tranquility.
|
||||
|
||||
About six and a half hours later, Mr. Armstrong opened the landing craft's hatch, stepped slowly down the ladder and declared as he planted the first human footprint on the lunar crust:
|
||||
|
||||
"That's one small step for man, one giant leap for mankind."
|
||||
|
||||
His first step on the moon came at 10:56:20 P.M., as a television camera outside the craft transmitted his every move to an awed and excited audience of hundreds of millions of people on earth.
|
||||
|
||||
Tentative Steps Test Soil
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 225 KiB |
@@ -0,0 +1,365 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Evaluates llama.cpp's DeepSeek-OCR by comparing its output for a test
|
||||
image to the actual text in part of that image.
|
||||
|
||||
Runs each test image through mtmd-cli, calculates CER and chrF for
|
||||
its output, and holds them against the HF model's scores.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger("deepseek-ocr-test")
|
||||
|
||||
RUN_TIMEOUT = 300
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelSpec:
|
||||
key: str
|
||||
label: str
|
||||
model_arg: str
|
||||
mmproj_arg: str
|
||||
model_default: str
|
||||
mmproj_default: str
|
||||
prompt: str = "Free OCR."
|
||||
n_predict: int = 512
|
||||
n_ctx: int | None = None
|
||||
# Unlimited-OCR's "document parsing" prompt emits <|det|> grounding markup that
|
||||
# the HF reference strips in result.md; drop it before scoring to match.
|
||||
strip_grounding: bool = False
|
||||
# v2/Unlimited loop on hard tiles; DRY caps it the way HF's
|
||||
# no_repeat_ngram_size does. v1 scores fine without it.
|
||||
dry: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestCase:
|
||||
model_key: str
|
||||
label: str
|
||||
image: str
|
||||
ground_truth: str
|
||||
hf_cer: float
|
||||
hf_chrf: float
|
||||
cer_tol: float
|
||||
chrf_tol: float
|
||||
|
||||
@property
|
||||
def cer_max(self) -> float:
|
||||
return self.hf_cer + self.cer_tol
|
||||
|
||||
@property
|
||||
def chrf_min(self) -> float:
|
||||
return self.hf_chrf - self.chrf_tol
|
||||
|
||||
|
||||
MODELS = {
|
||||
"v1": ModelSpec(
|
||||
key="v1", label="DeepSeek-OCR",
|
||||
model_arg="--llama-model", mmproj_arg="--mmproj",
|
||||
model_default="gguf_models/deepseek-ai/deepseek-ocr-bf16.gguf",
|
||||
mmproj_default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-bf16.gguf",
|
||||
),
|
||||
"v2": ModelSpec(
|
||||
key="v2", label="DeepSeek-OCR-2",
|
||||
model_arg="--llama-model-2", mmproj_arg="--mmproj-2",
|
||||
model_default="gguf_models/deepseek-ai/deepseek-ocr-2-bf16.gguf",
|
||||
mmproj_default="gguf_models/deepseek-ai/mmproj-deepseek-ocr-2-bf16.gguf",
|
||||
# v2 keeps generating past 512 on multi-tile; give it room to match the HF ref.
|
||||
n_predict=2048,
|
||||
dry=True,
|
||||
),
|
||||
"unlimited": ModelSpec(
|
||||
key="unlimited", label="Unlimited-OCR",
|
||||
model_arg="--llama-model-unlimited", mmproj_arg="--mmproj-unlimited",
|
||||
model_default="gguf_models/baidu/unlimited-ocr-bf16.gguf",
|
||||
mmproj_default="gguf_models/baidu/mmproj-unlimited-ocr-bf16.gguf",
|
||||
# "Free OCR." immediately emits EOS on this checkpoint; the HF reference
|
||||
# (demo/unlimited_ocr_scores.py) uses "document parsing.", which grounds.
|
||||
prompt="document parsing.",
|
||||
# Grounding emits ~3x the tokens of plain OCR, so it needs a larger budget
|
||||
# and context to reach the article body the ground truth covers.
|
||||
n_predict=4096,
|
||||
n_ctx=16384,
|
||||
strip_grounding=True,
|
||||
dry=True,
|
||||
),
|
||||
}
|
||||
|
||||
CASES = [
|
||||
TestCase(
|
||||
model_key="v1", label="single-view scan",
|
||||
image="tools/mtmd/test-1.jpeg",
|
||||
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
|
||||
# Fragile image: the HF ref itself swings ~0.286-0.314 across precision
|
||||
# configs -- hence the wide tol. llama.cpp bf16 ~0.322/63.8.
|
||||
hf_cer=0.3140, hf_chrf=67.57, cer_tol=0.04, chrf_tol=5.0,
|
||||
),
|
||||
TestCase(
|
||||
model_key="v2", label="single-view scan",
|
||||
image="tools/mtmd/test-1.jpeg",
|
||||
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
|
||||
# 640x488 is below the 768 tiling threshold -- single 1024 global view.
|
||||
# hf_cer/hf_chrf are the deepseek-ai repo's own scores (ImageOps.pad);
|
||||
# the transformers HF processor is *not* the reference -- its pad_to_square
|
||||
# is one pixel off and lands at ~0.69 instead.
|
||||
hf_cer=0.7761, hf_chrf=28.70, cer_tol=0.12, chrf_tol=8.0,
|
||||
),
|
||||
TestCase(
|
||||
model_key="v1", label="multi-tile (dynamic resolution)",
|
||||
image="tools/mtmd/tests/test-1-positive.png",
|
||||
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
|
||||
# 429x806 -- 806 > 640 triggers the v1 "Gundam" path: (1,2) grid ->
|
||||
# 2 local 640 tiles + 1 global 1024 view. Regression guard for the
|
||||
# tiling preprocessor -- a broken tile path craters the score.
|
||||
# hf_cer/hf_chrf are HF v1's measured scores -- it reads this clean crop exactly.
|
||||
hf_cer=0.0000, hf_chrf=100.00, cer_tol=0.03, chrf_tol=3.0,
|
||||
),
|
||||
TestCase(
|
||||
model_key="v2", label="multi-tile (dynamic resolution)",
|
||||
image="tools/mtmd/tests/test-1-positive.png",
|
||||
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
|
||||
# 429x806 -- 806 > 768 triggers the v2 path: (1,2) grid ->
|
||||
# 2 local 768 tiles + 1 global 1024 view = 545 image tokens.
|
||||
hf_cer=0.0236, hf_chrf=97.05, cer_tol=0.03, chrf_tol=3.0,
|
||||
),
|
||||
TestCase(
|
||||
model_key="unlimited", label="single-view scan",
|
||||
image="tools/mtmd/test-1.jpeg",
|
||||
ground_truth="tools/mtmd/tests/test-1-ground-truth.txt",
|
||||
# HF reference: Unlimited-OCR scoring (gundam, bf16) on this image/ground-truth.
|
||||
# Decoder runs full MHA, not R-SWA; the band absorbs that gap + bf16 variance.
|
||||
hf_cer=0.1869, hf_chrf=75.23, cer_tol=0.06, chrf_tol=6.0,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
GROUNDING_TAG_RE = re.compile(r"<\|(ref|det)\|>.*?<\|/\1\|>", re.DOTALL)
|
||||
|
||||
|
||||
def strip_grounding(text: str) -> str:
|
||||
"""Drop <|ref|>..<|/ref|> / <|det|>..<|/det|> grounding markup, matching the
|
||||
cleaned result.md the HF reference scores against."""
|
||||
return GROUNDING_TAG_RE.sub("", text)
|
||||
|
||||
|
||||
def arg_dest(flag: str) -> str:
|
||||
return flag.lstrip("-").replace("-", "_")
|
||||
|
||||
|
||||
def verdict(ok: bool) -> str:
|
||||
return "PASS" if ok else "FAIL"
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""NFC-normalize and collapse whitespace, so line-wrap and spacing
|
||||
don't count as CER errors."""
|
||||
return " ".join(unicodedata.normalize("NFC", text).split())
|
||||
|
||||
|
||||
def locally_align(expected: str, ocr_out: str) -> str:
|
||||
"""Return the span of `ocr_out` that best matches `expected`.
|
||||
|
||||
The ground truth covers part of the article body.
|
||||
But the test image includes half of the newspaper's front page.
|
||||
Fuzzy partial-ratio matching picks out
|
||||
the body so the unrelated text doesn't disturb CER / chrF.
|
||||
"""
|
||||
from rapidfuzz import fuzz
|
||||
alignment = fuzz.partial_ratio_alignment(expected, ocr_out)
|
||||
if alignment is None or alignment.dest_end <= alignment.dest_start:
|
||||
return ocr_out
|
||||
return ocr_out[alignment.dest_start:alignment.dest_end]
|
||||
|
||||
|
||||
def compute_cer(expected: str, ocr_out: str) -> float:
|
||||
"""Character Error Rate. Lower is better.
|
||||
CER: fraction of characters you'd insert/delete/substitute to fix the output; 0 = perfect."""
|
||||
import jiwer
|
||||
return jiwer.cer(expected, ocr_out)
|
||||
|
||||
|
||||
def compute_chrf(expected: str, ocr_out: str) -> float:
|
||||
"""chrF score on 0-100. Higher is better.
|
||||
chrF: F-score over shared character n-grams; more forgiving of small word/spacing drift than CER.
|
||||
"""
|
||||
from sacrebleu.metrics import CHRF
|
||||
return CHRF().sentence_score(ocr_out, [expected]).score
|
||||
|
||||
|
||||
def run_mtmd_cli(spec: "ModelSpec", model_path, mmproj_path, image_path, bin_path) -> str:
|
||||
"""Run mtmd-cli on the image and return its output."""
|
||||
cmd = [
|
||||
str(bin_path),
|
||||
"-m", str(model_path),
|
||||
"--mmproj", str(mmproj_path),
|
||||
"--image", str(image_path),
|
||||
"-p", spec.prompt,
|
||||
"--chat-template", "deepseek-ocr",
|
||||
"--temp", "0",
|
||||
"--flash-attn", "off", # match the HF "eager" attention reference
|
||||
"--no-warmup",
|
||||
"-n", str(spec.n_predict), # cap loops on hard images (KV would otherwise fill)
|
||||
]
|
||||
if spec.dry:
|
||||
# HF decodes with no_repeat_ngram_size; llama.cpp's analog is DRY.
|
||||
# Default DRY breakers include "\n", so they are cleared below.
|
||||
cmd += [
|
||||
"--dry-multiplier", "0.8",
|
||||
"--dry-base", "1.75",
|
||||
"--dry-allowed-length", "2",
|
||||
"--dry-penalty-last-n", "64",
|
||||
"--dry-sequence-breaker", "none",
|
||||
]
|
||||
if spec.n_ctx is not None:
|
||||
cmd += ["-c", str(spec.n_ctx)]
|
||||
logger.debug(f" command: {' '.join(cmd)}")
|
||||
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=False, timeout=RUN_TIMEOUT)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
if e.stderr:
|
||||
logger.error("llama.cpp stderr:\n%s", e.stderr.decode("utf-8", errors="replace"))
|
||||
raise RuntimeError(f"llama-mtmd-cli timed out after {RUN_TIMEOUT}s")
|
||||
|
||||
if result.returncode != 0:
|
||||
logger.error("llama.cpp stderr:\n%s", result.stderr.decode("utf-8", errors="replace"))
|
||||
raise RuntimeError(f"llama-mtmd-cli failed with code {result.returncode}")
|
||||
|
||||
output = result.stdout.decode("utf-8", errors="replace").strip()
|
||||
if spec.strip_grounding:
|
||||
output = strip_grounding(output)
|
||||
if not output:
|
||||
raise RuntimeError("llama-mtmd-cli produced no output on stdout")
|
||||
logger.info(f" output: {len(output)} chars")
|
||||
return output
|
||||
|
||||
|
||||
def read_expected_text(file_path: Path) -> str:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def evaluate(case: "TestCase", expected: str, ocr_out: str) -> bool:
|
||||
expected = normalize_text(expected)
|
||||
ocr_out = normalize_text(ocr_out)
|
||||
aligned = locally_align(expected, ocr_out)
|
||||
|
||||
logger.debug(f"\n--- expected (normalized) ---\n{expected}")
|
||||
logger.debug(f"\n--- OCR output (normalized) ---\n{ocr_out}")
|
||||
logger.debug(f"\n--- aligned span ---\n{aligned}")
|
||||
|
||||
cer = compute_cer(expected, aligned)
|
||||
chrf = compute_chrf(expected, aligned)
|
||||
|
||||
cer_pass = cer <= case.cer_max
|
||||
chrf_pass = chrf >= case.chrf_min
|
||||
passed = cer_pass and chrf_pass
|
||||
|
||||
logger.info("")
|
||||
logger.info("=" * 60)
|
||||
logger.info("OCR evaluation:")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f" CER {cer:>7.4f} (HF {case.hf_cer:.4f}, <= {case.cer_max:>7.4f} -> {verdict(cer_pass)})")
|
||||
logger.info(f" chrF (0-100) {chrf:>7.2f} (HF {case.hf_chrf:.2f}, >= {case.chrf_min:>7.2f} -> {verdict(chrf_pass)})")
|
||||
logger.info(f" Expected chars {len(expected):>7}")
|
||||
logger.info(f" Aligned chars {len(aligned):>7} (of {len(ocr_out)} OCR chars)")
|
||||
logger.info("")
|
||||
logger.info(f" Result: {verdict(passed)}")
|
||||
logger.info("=" * 60)
|
||||
return passed
|
||||
|
||||
|
||||
def argument_parser() -> argparse.ArgumentParser:
|
||||
ap = argparse.ArgumentParser(description="Compare llama.cpp DeepSeek-OCR output with a ground-truth transcript")
|
||||
ap.add_argument("--llama-bin", default="build/bin/llama-mtmd-cli",
|
||||
help="Path to llama-mtmd-cli binary (relative to repo root or absolute)")
|
||||
for spec in MODELS.values():
|
||||
ap.add_argument(spec.model_arg, default=spec.model_default,
|
||||
help=f"Path to the {spec.label} GGUF model (relative to repo root or absolute)")
|
||||
ap.add_argument(spec.mmproj_arg, default=spec.mmproj_default,
|
||||
help=f"Path to the {spec.label} mmproj GGUF file (relative to repo root or absolute)")
|
||||
ap.add_argument("--verbose", action="store_true",
|
||||
help="Also log the expected, OCR, and aligned text")
|
||||
return ap
|
||||
|
||||
|
||||
def configure_logging(verbose: bool) -> None:
|
||||
logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO,
|
||||
format="%(message)s")
|
||||
|
||||
|
||||
def resolve_path(path: str, base: Path) -> Path:
|
||||
p = Path(path)
|
||||
return p if p.is_absolute() else base / p
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = argument_parser().parse_args()
|
||||
configure_logging(args.verbose)
|
||||
|
||||
repo_root = Path(__file__).resolve().parents[3] # tests -> mtmd -> tools -> repo root
|
||||
binary = resolve_path(args.llama_bin, repo_root)
|
||||
|
||||
if not binary.exists():
|
||||
logger.error(f"Error: binary not found: {binary}")
|
||||
return 1
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("DeepSeek-OCR: llama.cpp vs HF parity check")
|
||||
logger.info("=" * 60)
|
||||
|
||||
results = {}
|
||||
for case in CASES:
|
||||
model_spec = MODELS[case.model_key]
|
||||
title = f"{model_spec.label} -- {case.label}"
|
||||
|
||||
logger.info("")
|
||||
logger.info(f"=== {title} ===")
|
||||
|
||||
model = resolve_path(getattr(args, arg_dest(model_spec.model_arg)), repo_root)
|
||||
mmproj = resolve_path(getattr(args, arg_dest(model_spec.mmproj_arg)), repo_root)
|
||||
image = resolve_path(case.image, repo_root)
|
||||
ground_truth = resolve_path(case.ground_truth, repo_root)
|
||||
|
||||
missing = [(lbl, p) for lbl, p in [("model", model), ("mmproj", mmproj),
|
||||
("image", image), ("ground-truth", ground_truth)]
|
||||
if not p.exists()]
|
||||
if missing:
|
||||
for lbl, p in missing:
|
||||
logger.error(f" Error: {lbl} not found: {p}")
|
||||
results[title] = False
|
||||
continue
|
||||
|
||||
expected = read_expected_text(ground_truth)
|
||||
logger.info(f" Image: {case.image}")
|
||||
logger.info(f" Expected text: {len(expected)} chars")
|
||||
logger.info(f" Running llama.cpp prompt {model_spec.prompt!r}")
|
||||
try:
|
||||
ocr_out = run_mtmd_cli(model_spec, model, mmproj, image, binary)
|
||||
except RuntimeError as e:
|
||||
logger.error(f" Error: {e}")
|
||||
results[title] = False
|
||||
continue
|
||||
|
||||
results[title] = evaluate(case, expected, ocr_out)
|
||||
|
||||
logger.info("")
|
||||
logger.info("=== Summary ===")
|
||||
for title, ok in results.items():
|
||||
logger.info(f" {title:<48} {verdict(ok)}")
|
||||
all_passed = all(results.values())
|
||||
logger.info(f"Overall: {verdict(all_passed)}")
|
||||
|
||||
return 0 if all_passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,3 @@
|
||||
jiwer
|
||||
sacrebleu
|
||||
rapidfuzz
|
||||
Reference in new issue
Block a user