zefie/nvfp4_gguf_quant
1
0 Fork 0
Code Actions Activity

include llama.cpp since there were some edits

Browse Source
This commit is contained in:
zefie committed 2026-11-01 16:32:35 -05:00
1 parent db7f28d087
commit 8c66990853
3918 files changed
+1640170 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

llama.cpp/tests/llama-context/cases-kv.cpp
+787
View File
@@ -0,0 +1,787 @@
// KV cache state cases
//
#include "test-case.h"
#include "test-log.h"
#include "test-util.h"
#include "common.h"
#include "../../src/llama-ext.h" // llama_model_set_attn_rot
#include <algorithm>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <functional>
#include <iterator>
#include <string>
#include <utility>
#include <vector>
//
// reference run
//
// - decode all but the last token, saving the state to disk
// - decode the last token
// - generate n_predict tokens
// the logits it produces are the reference of every case with needs_baseline()
struct case_baseline : ctx_test {
const char * name() const override { return "baseline"; }
const char * description() const override {
return "reference generation over the whole prompt, saves the state before the last token";
}
case_result run(model_run & mr) const override {
ctx_opts opts;
opts.n_seq_max = 2;
auto ctx = mr.ctx(opts);
auto smpl = mr.sampler();
if (!ctx) {
return case_result::fail();
}
auto n_past = 0;
if (!common_prompt_batch_decode(ctx.get(), mr.tokens, (int) mr.tokens.size(), n_past, mr.params.n_batch, mr.params.out_file, true)) {
TLOG_ERR("%s: failed to decode the prompt\n", __func__);
return case_result::fail();
}
generation_result result;
if (!generate_tokens(ctx.get(), smpl.get(), n_past, mr.params.n_predict, 0, result)) {
return case_result::fail();
}
if (result.empty()) {
return case_result::fail();
}
mr.baseline = std::move(result);
mr.have_baseline = true;
return case_result::pass();
}
};
//
// sequences
//
// - decode the same prefix into two sequences
// - remove sequence 0
// - verify that sequence 1 remains unchanged
struct case_seq_rm_isolated : ctx_test {
const char * name() const override { return "seq_rm"; }
const char * description() const override {
return "removing one sequence leaves the state of the other sequences untouched";
}
case_result run(model_run & mr) const override {
ctx_opts opts;
opts.n_ctx = 256;
opts.n_seq_max = 2;
opts.kv_unified = 1;
auto ctx = mr.ctx(opts);
if (!ctx) {
return case_result::fail();
}
const size_t n_tokens = std::min<size_t>(mr.tokens.size(), 128);
for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) {
if (!decode_seq(ctx.get(), mr.tokens, 0, (llama_pos) n_tokens, seq_id, true)) {
TLOG_ERR("%s: failed to decode the prompt for sequence %d\n", __func__, seq_id);
return case_result::fail();
}
}
std::vector<uint8_t> before;
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, before)) {
return case_result::fail();
}
if (!seq_rm(ctx.get(), 0)) {
return case_result::fail();
}
std::vector<uint8_t> after;
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, after)) {
return case_result::fail();
}
CTX_REQUIRE(blobs_equal(before, after), "removing seq 0 changed the state of seq 1");
return case_result::pass();
}
};
// - load the state saved by the baseline case
// - replay its last token
// - generate again and compare against the baseline logits
struct case_state_load : ctx_test {
const char * name() const override { return "state_load"; }
const char * description() const override {
return "a saved session replays to the same logits as the run that saved it";
}
bool needs_baseline() const override { return true; }
case_result run(model_run & mr) const override {
ctx_opts opts;
opts.n_seq_max = 2;
auto ctx = mr.ctx(opts);
auto smpl = mr.sampler();
if (!ctx) {
return case_result::fail();
}
int n_past = 0;
if (!load_state_replay_last(ctx.get(), mr.params.out_file, mr.tokens, n_past)) {
return case_result::fail();
}
if (!generate_tokens_compare(ctx.get(), smpl.get(), n_past, mr.params.n_predict, 0, mr.baseline, NMSE_THRESHOLD)) {
return case_result::fail();
}
return case_result::pass();
}
};
// - load the state saved by the baseline case and replay its last token
// - copy sequence 0 to sequence 1 over the given io path, clearing the cache in between
// - continue on sequence 1 and compare against the baseline logits
struct case_seq_cp : ctx_test {
explicit case_seq_cp(bool on_device_) : on_device(on_device_) {}
const char * name() const override { return on_device ? "cp_d" : "cp_h"; }
const char * description() const override {
return on_device ? "copying a sequence over the device io path keeps its state"
: "copying a sequence over the host io path keeps its state";
}
bool needs_baseline() const override { return true; }
case_result run(model_run & mr) const override {
ctx_opts opts;
opts.n_seq_max = 2;
auto ctx = mr.ctx(opts);
auto smpl = mr.sampler();
if (!ctx) {
return case_result::fail();
}
const uint32_t flags = on_device ? LLAMA_STATE_SEQ_FLAGS_ON_DEVICE : LLAMA_STATE_SEQ_FLAGS_NONE;
TLOG_INF("io path: %s\n", on_device ? "device" : "host");
int n_past = 0;
if (!load_state_replay_last(ctx.get(), mr.params.out_file, mr.tokens, n_past)) {
return case_result::fail();
}
{
std::vector<uint8_t> seq_store;
if (!get_seq_state(ctx.get(), 0, flags, seq_store)) {
return case_result::fail();
}
TLOG_TRC("%s: seq 0 copied, %zu bytes\n", __func__, seq_store.size());
llama_memory_clear(llama_get_memory(ctx.get()), true);
TLOG_TRC("%s: kv cache cleared\n", __func__);
if (!set_seq_state(ctx.get(), 1, flags, seq_store)) {
return case_result::fail();
}
TLOG_TRC("%s: seq 1 restored, %zu bytes\n", __func__, seq_store.size());
}
if (!generate_tokens_compare(ctx.get(), smpl.get(), n_past, mr.params.n_predict, 1, mr.baseline, NMSE_THRESHOLD)) {
return case_result::fail();
}
return case_result::pass();
}
private:
bool on_device;
};
// - decode the same prefix on two sequences, interleaving seq 0 cells between the seq 1 cells
// - save the seq 1 state, free the interleaved seq 0 cells, and restore via the given io path
// - the restore destination is non-contiguous: scatter reads are batched per contiguous run
// - save again on the host and compare the two blobs byte for byte
struct case_seq_cp_scatter : ctx_test {
explicit case_seq_cp_scatter(bool on_device_) : on_device(on_device_) {}
const char * name() const override { return on_device ? "cp_d_s" : "cp_h_s"; }
const char * description() const override {
return "saving and restoring a sequence with a fragmented (scattered) cell range";
}
case_result run(model_run & mr) const override {
ctx_opts opts;
opts.n_ctx = 256;
opts.n_seq_max = 2;
opts.kv_unified = 1;
auto ctx = mr.ctx(opts);
if (!ctx) {
return case_result::fail();
}
const uint32_t flags = on_device ? LLAMA_STATE_SEQ_FLAGS_ON_DEVICE : LLAMA_STATE_SEQ_FLAGS_NONE;
TLOG_INF("io path: %s\n", on_device ? "device" : "host");
// seq 0 cells 0,1,4 interleave the seq 1 cells 2,3,5
if (!decode_one(ctx.get(), mr.tokens[0], 0, 0) ||
!decode_one(ctx.get(), mr.tokens[1], 1, 0) ||
!decode_one(ctx.get(), mr.tokens[0], 0, 1) ||
!decode_one(ctx.get(), mr.tokens[1], 1, 1) ||
!decode_one(ctx.get(), mr.tokens[2], 2, 0) ||
!decode_one(ctx.get(), mr.tokens[2], 2, 1)) {
TLOG_ERR("%s: failed to build the interleaved state\n", __func__);
return case_result::fail();
}
// host blob: contains the KV data, used for the byte-for-byte comparison
std::vector<uint8_t> state_before;
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_before)) {
return case_result::fail();
}
// save via the io path under test
std::vector<uint8_t> state_save;
if (!get_seq_state(ctx.get(), 1, flags, state_save)) {
return case_result::fail();
}
TLOG_TRC("%s: seq 1 saved via %s, %zu bytes\n", __func__, on_device ? "device" : "host", state_save.size());
// free seq 0's cells so the ring is fragmented: the restore destination (seq 1's interleaved cells) stays non-contiguous
if (!seq_rm(ctx.get(), 0)) {
return case_result::fail();
}
// restore via the io path under test
if (!set_seq_state(ctx.get(), 1, flags, state_save)) {
return case_result::fail();
}
TLOG_TRC("%s: seq 1 restored via %s, %zu bytes\n", __func__, on_device ? "device" : "host", state_save.size());
std::vector<uint8_t> state_after;
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_after)) {
return case_result::fail();
}
// the blob is serialized in sequence cell order, so identical bytes iff the restore wrote the same KV
CTX_REQUIRE(blobs_equal(state_before, state_after), "the restored KV state is not byte-identical to the saved state");
return case_result::pass();
}
private:
bool on_device;
};
//
// sessions
//
// - save a sequence, erase it, restore it, save it again
// - the two blobs must be identical
// compares blobs rather than generated text: a partially restored cell still decodes to plausible tokens
struct case_state_roundtrip : ctx_test {
const char * name() const override { return "rt"; }
const char * description() const override {
return "save, erase, restore, save again - the two blobs must be identical";
}
case_result run(model_run & mr) const override {
auto ctx = mr.ctx();
if (!ctx) {
return case_result::fail();
}
common_batch batch = common_batch_get_one(ctx.get(), mr.tokens);
if (llama_process(ctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
TLOG_ERR("%s: failed to decode the prompt\n", __func__);
return case_result::fail();
}
std::vector<uint8_t> blob_a;
if (!get_seq_state(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, blob_a)) {
return case_result::fail();
}
if (!seq_rm(ctx.get(), 0)) {
return case_result::fail();
}
if (!set_seq_state(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, blob_a)) {
TLOG_ERR("%s: failed to restore seq 0\n", __func__);
return case_result::fail();
}
std::vector<uint8_t> blob_b;
if (!get_seq_state(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, blob_b)) {
return case_result::fail();
}
CTX_REQUIRE(blobs_equal(blob_a, blob_b), "the state changed across a save / erase / restore cycle");
return case_result::pass();
}
};
// the partial state holds what seq_rm cannot roll back: recurrent states and SWA windows
struct case_state_partial_size : ctx_test {
const char * name() const override { return "ps"; }
const char * description() const override {
return "the partial state is not larger than the full state and, without SWA, does not grow with the tokens";
}
case_result run(model_run & mr) const override {
auto ctx = mr.ctx();
if (!ctx) {
return case_result::fail();
}
const llama_pos n_tokens = mr.tokens.size();
const llama_pos n_half = n_tokens / 2;
if (!decode_seq(ctx.get(), mr.tokens, 0, n_half, 0, true)) {
return case_result::fail();
}
const size_t partial_a = llama_state_seq_get_size_ext(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
const size_t full_a = llama_state_seq_get_size_ext(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE);
if (!decode_seq(ctx.get(), mr.tokens, n_half, n_tokens, 0, true)) {
return case_result::fail();
}
const size_t partial_b = llama_state_seq_get_size_ext(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
const size_t full_b = llama_state_seq_get_size_ext(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE);
TLOG_INF("%s: partial %zu -> %zu, full %zu -> %zu\n", __func__, partial_a, partial_b, full_a, full_b);
CTX_REQUIRE(partial_a <= full_a && partial_b <= full_b, "the partial state is larger than the full state");
if (llama_model_n_swa(mr.model) == 0) {
CTX_REQUIRE(partial_a == partial_b, "the partial state grows with the tokens");
}
return case_result::pass();
}
};
// a failed restore must leave the sequence empty and must not change the logits of other sequences
struct case_state_restore_failure : ctx_test {
const char * name() const override { return "rf"; }
const char * description() const override {
return "restoring a corrupted state fails and leaves the other sequences untouched";
}
case_result run(model_run & mr) const override {
ctx_opts opts;
opts.n_ctx = 256;
opts.n_seq_max = 4;
opts.kv_unified = 1;
// without flash attention, corrupted data left behind by the restore shows up as NaN logits on the other sequences
opts.fa = LLAMA_FLASH_ATTN_TYPE_DISABLED;
auto ctx = mr.ctx(opts);
if (!ctx) {
return case_result::fail();
}
llama_memory_t mem = llama_get_memory(ctx.get());
if (mem == nullptr) {
TLOG_INF("no memory to test\n");
return case_result::pass();
}
const auto decode = [&](const llama_tokens & inp, llama_seq_id seq_id, std::vector<float> * logits_out) {
if (!decode_tokens(ctx.get(), inp, seq_id)) {
TLOG_ERR("%s: failed to decode on sequence %d\n", __func__, seq_id);
return false;
}
if (logits_out && !get_current_logits(ctx.get(), *logits_out)) {
TLOG_ERR("%s: failed to get logits\n", __func__);
return false;
}
return true;
};
const llama_tokens tokens_save (mr.tokens.begin(), mr.tokens.begin() + std::min<size_t>(24, mr.tokens.size()));
const llama_tokens tokens_verify(mr.tokens.end() - std::min<size_t>(8, mr.tokens.size()), mr.tokens.end());
// the cases share a working directory, so the state file is named after the model
const std::string path = "state-restore-failure." + mr.name + ".tmp.bin";
llama_memory_clear(mem, true);
std::vector<float> baseline;
if (!decode(tokens_verify, 1, &baseline)) {
return case_result::fail();
}
const std::vector<std::pair<const char *, std::function<bool()>>> sub_cases = {
{ "buffer", [&]() {
std::vector<uint8_t> state(llama_state_seq_get_size(ctx.get(), 0));
GGML_ASSERT(llama_state_seq_get_data(ctx.get(), state.data(), state.size(), 0) == state.size());
llama_memory_seq_rm(mem, 0, -1, -1);
if (!corrupt_state(state)) {
return false;
}
return llama_state_seq_set_data(ctx.get(), state.data(), state.size(), 0) == 0;
}},
{ "file", [&]() {
GGML_ASSERT(llama_state_seq_save_file(ctx.get(), path.c_str(), 0, tokens_save.data(), tokens_save.size()) > 0);
llama_memory_seq_rm(mem, 0, -1, -1);
std::vector<uint8_t> data;
{
std::ifstream f(path, std::ios::binary);
data.assign(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>());
}
if (!corrupt_state(data)) {
std::remove(path.c_str());
return false;
}
{
std::ofstream f(path, std::ios::binary);
f.write((const char *) data.data(), data.size());
}
llama_tokens tokens_out(tokens_save.size());
size_t n_token_count = 0;
const size_t nread = llama_state_seq_load_file(ctx.get(), path.c_str(), 0, tokens_out.data(), tokens_out.size(), &n_token_count);
std::remove(path.c_str());
return nread == 0;
}},
};
for (const auto & [name, restore_failed] : sub_cases) {
llama_memory_clear(mem, true);
if (!decode(tokens_save, 0, nullptr)) {
return case_result::fail();
}
CTX_REQUIRE(restore_failed(), std::string(name) + ": restoring a corrupted state did not fail");
CTX_REQUIRE(llama_memory_seq_pos_max(mem, 0) == -1,
std::string(name) + ": the sequence is not empty after a failed restore");
std::vector<float> logits;
if (!decode(tokens_verify, 1, &logits)) {
return case_result::fail();
}
const logits_diff diff = compare_logits(baseline.data(), logits.data(), logits.size());
CTX_REQUIRE(diff.n_nan == 0 && diff.diff_max <= 1e-6f,
std::string(name) + ": the logits changed after a failed restore (max diff = " +
std::to_string(diff.diff_max) + ", nan = " + std::to_string(diff.n_nan) + ")");
TLOG_TRC("%s: %s: logits match (max diff = %g)\n", __func__, name, (double) diff.diff_max);
}
return case_result::pass();
}
};
// a KV state saved with attention rotation enabled must restore only into a context with the
// same setting
// note: rotation is only active for quantized KV caches with a head size that is a multiple of
// 64, for other models the restore into the rotation-disabled context is valid and the
// test passes vacuously
struct case_state_rotation : ctx_test {
const char * name() const override { return "rot"; }
const char * description() const override {
return "a state saved with attention rotation restores only into a matching context";
}
case_result run(model_run & mr) const override {
const auto make_context = [&](ggml_type type_k, ggml_type type_v, bool disable_rotation) {
// a setting of the model, not of the process - a sweep that runs several models at a
// time must not let this leak into the contexts of another model
llama_model_set_attn_rot(mr.model, !disable_rotation);
ctx_opts opts;
opts.n_ctx = 32;
opts.n_batch = 1;
opts.n_ubatch = 1;
opts.type_k = type_k;
opts.type_v = type_v;
opts.fa = LLAMA_FLASH_ATTN_TYPE_ENABLED;
// probing whether this KV type is supported is expected to fail for some models
opts.quiet = true;
return mr.ctx(opts);
};
std::vector<std::pair<ggml_type, ggml_type>> type_pairs;
for (const auto & types : { std::pair{GGML_TYPE_Q8_0, GGML_TYPE_Q8_0} }) {
if (make_context(types.first, types.second, false)) {
type_pairs.push_back(types);
}
}
if (type_pairs.empty()) {
return case_result::skip("no supported quantized KV cache type combination");
}
bool success = true;
for (const auto & types : type_pairs) {
auto src = make_context(types.first, types.second, false);
if (!src) {
TLOG_ERR("%s: failed to create the source context\n", __func__);
success = false;
break;
}
llama_token token = 0;
if (llama_decode(src.get(), llama_batch_get_one(&token, 1))) {
TLOG_ERR("%s: failed to decode a token\n", __func__);
success = false;
break;
}
std::vector<uint8_t> state;
if (!get_seq_state(src.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, state)) {
if (state.empty()) {
continue; // no KV state to test
}
success = false;
break;
}
auto matching = make_context(types.first, types.second, false);
if (!matching || !set_seq_state(matching.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, state)) {
TLOG_ERR("%s: failed to restore a matching rotation\n", __func__);
success = false;
break;
}
auto mismatched = make_context(types.first, types.second, true);
if (!mismatched) {
TLOG_ERR("%s: failed to create a mismatched rotation context\n", __func__);
success = false;
break;
}
// a model that does not use attention rotation accepts the mismatched restore
if (set_seq_state(mismatched.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, state)) {
TLOG_TRC("%s: state restored into rotation-disabled context, model does not use attention rotation\n", __func__);
}
}
llama_model_set_attn_rot(mr.model, true); // the default, the cases that follow rely on it
if (!success) {
return case_result::fail();
}
return case_result::pass();
}
};
// restore a sequence into a fragmented cache: the slots freed by the removed sequence are
// scattered between live cells of the other sequences, so the restore cannot claim one
// contiguous block
// the restored state must come back byte for byte, and the sequence must continue to behave
// exactly as the same sequence in a context where it was never removed
// ref: https://github.com/ggml-org/llama.cpp/issues/17527
struct case_state_restore_fragmented : ctx_test {
const char * name() const override { return "frag"; }
const char * description() const override {
return "restore a sequence into a cache fragmented by the removal of another sequence";
}
std::string skip_reason(model_run & mr) const override {
if (arch_reserves_single_seq(mr.model)) {
return "the interleaved batch is a multi-seq graph";
}
return ctx_test::skip_reason(mr);
}
case_result run(model_run & mr) const override {
const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(mr.model));
ctx_opts opts;
opts.n_ctx = 256;
opts.n_seq_max = 3;
opts.kv_unified = 1;
opts.n_batch = 3*70;
opts.n_ubatch = 3*70; // the interleaved prompt is a single multi-seq batch
// ctx removes and restores seq 1, ctx_keep keeps it untouched as the reference
llama_context_ptr ctx = mr.ctx(opts);
llama_context_ptr ctx_keep = mr.ctx(opts);
if (!ctx || !ctx_keep) {
return case_result::fail();
}
const llama_pos n_tokens = (llama_pos) std::min<size_t>(70, mr.tokens.size());
// interleave the 3 sequences: 01201230123... - seq 1 owns every third cell
const auto decode_interleaved = [&](llama_context * cur) {
common_batch batch(cur);
for (llama_pos i = 0; i < n_tokens; i++) {
for (llama_seq_id s = 0; s < 3; ++s) {
batch.add(mr.tokens[i], i, s, false);
}
}
batch.set_output(batch.size() - 1, true);
return llama_process(cur, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
};
if (!decode_interleaved(ctx.get()) || !decode_interleaved(ctx_keep.get())) {
TLOG_ERR("%s: failed to decode the interleaved prompt\n", __func__);
return case_result::fail();
}
TLOG_INF("%s: processed the prompt on seq 0, 1, 2 (%d tokens each)\n", __func__, n_tokens);
std::vector<uint8_t> state_before;
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_before)) {
return case_result::fail();
}
// clearing seq 1 leaves holes where its cells were - no contiguous block large enough
// for the seq 1 state is left in the cache
llama_memory_seq_rm(llama_get_memory(ctx.get()), 1, -1, -1);
if (!set_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_before)) {
TLOG_ERR("%s: failed to restore the seq state into a fragmented cache\n", __func__);
return case_result::fail();
}
TLOG_INF("%s: restored seq 1 into a fragmented cache, %zu bytes\n", __func__, state_before.size());
// the restore has to be lossless - the cells the sequence ends up in are irrelevant, its
// state has to come back exactly as it was saved
std::vector<uint8_t> state_after;
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_after)) {
return case_result::fail();
}
CTX_REQUIRE(blobs_equal(state_before, state_after), "restoring into a fragmented cache changed the state of seq 1");
// the restored sequence must continue exactly as the untouched one - decode the same token
// on seq 1 in both contexts and compare the logits they produce
const llama_token token = gen_token(n_vocab, 1, n_tokens);
if (!decode_one(ctx.get(), token, n_tokens, 1) || !decode_one(ctx_keep.get(), token, n_tokens, 1)) {
TLOG_ERR("%s: failed to decode with the restored state\n", __func__);
return case_result::fail();
}
const float * logits = llama_get_logits(ctx.get());
const float * logits_keep = llama_get_logits(ctx_keep.get());
if (!logits || !logits_keep) {
TLOG_ERR("%s: missing logits after the restore\n", __func__);
return case_result::fail();
}
const logits_diff diff = compare_logits(logits, logits_keep, n_vocab);
CTX_REQUIRE(diff.nmse() <= NMSE_THRESHOLD,
"the logits of the restored seq 1 differ from the reference: nmse " + std::to_string(diff.nmse()));
TLOG_INF("%s: restored seq 1 continued identically to the reference, nmse %g\n", __func__, diff.nmse());
return case_result::pass();
}
};
//
// no_alloc
//
// - load a second instance of the model with no_alloc, so the tensor data stays on disk
// - create a context from it, which reserves the compute graphs over the dummy buffers
// - the memory use must still be reported, and nothing real may be allocated for the memory
struct case_noalloc : ctx_test {
const char * name() const override { return "noalloc"; }
const char * description() const override {
return "with no_alloc the memory use is simulated: non-zero breakdown, null buffer bases";
}
case_result run(model_run & mr) const override {
const std::string & path = mr.params.model.path;
CTX_REQUIRE(!path.empty(), "the model path is not known");
llama_model_params mparams = llama_model_default_params();
mparams.no_alloc = true;
mparams.load_mode = LLAMA_LOAD_MODE_NONE;
llama_model_ptr model(llama_model_load_from_file(path.c_str(), mparams));
CTX_REQUIRE(model, "failed to load the model with no_alloc");
auto ctx = init_ctx(model.get(), common_context_params_to_llama(mr.params), 0);
CTX_REQUIRE(ctx, "failed to create the context with no_alloc");
size_t n_ctx_mem = 0;
for (const auto & [_, data] : llama_get_memory_breakdown(ctx.get())) {
n_ctx_mem += data.context;
}
// models without a memory module (BERT, diffusion) have a null memory
if (llama_get_memory(ctx.get()) != nullptr) {
CTX_REQUIRE(n_ctx_mem > 0, "the simulated memory breakdown is zero");
} else {
CTX_REQUIRE(n_ctx_mem == 0, "a model without memory reported context memory");
}
for (ggml_backend_buffer_t buf : llama_memory_get_buffers(llama_get_memory(ctx.get()))) {
CTX_REQUIRE(ggml_backend_buffer_get_base(buf) == nullptr, "a memory buffer was really allocated");
}
return case_result::pass();
}
};
std::vector<ctx_test_ptr> make_kv_cases() {
std::vector<ctx_test_ptr> cases;
cases.push_back(std::make_unique<case_baseline>());
cases.push_back(std::make_unique<case_seq_rm_isolated>());
cases.push_back(std::make_unique<case_state_load>());
cases.push_back(std::make_unique<case_seq_cp>(false));
cases.push_back(std::make_unique<case_seq_cp>(true));
cases.push_back(std::make_unique<case_seq_cp_scatter>(false));
cases.push_back(std::make_unique<case_seq_cp_scatter>(true));
cases.push_back(std::make_unique<case_state_roundtrip>());
cases.push_back(std::make_unique<case_state_partial_size>());
cases.push_back(std::make_unique<case_state_restore_failure>());
cases.push_back(std::make_unique<case_state_rotation>());
cases.push_back(std::make_unique<case_state_restore_fragmented>());
cases.push_back(std::make_unique<case_noalloc>());
return cases;
}
llama.cpp/tests/llama-context/cases-rs.cpp
+404
View File
@@ -0,0 +1,404 @@
// recurrent state cases
#include "test-case.h"
#include "test-log.h"
#include "test-util.h"
#include "common.h"
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <string>
#include <vector>
// roll back multiple sequences, then replay them in a single batch whose per-seq token count
// exceeds n_ubatch: each seq's replay spans several ubatches while its rollback restore is
// still pending. Compared against a reference context that never advanced past the rollback
// point and decodes the identical replay batch.
struct case_multi_seq_split_replay : ctx_test {
test_group group() const override { return TEST_GROUP_RS; }
const char * name() const override { return "replay"; }
const char * description() const override {
return "replay rolled-back sequences in a batch wider than n_ubatch, against a reference context";
}
case_result run(model_run & mr) const override {
const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(mr.model));
constexpr uint32_t n_seqs = 2;
constexpr uint32_t n_ubatch = 16;
constexpr uint32_t n_prompt = 20;
constexpr uint32_t n_rollback = 3;
constexpr uint32_t n_replay = 40; // > n_ubatch so each seq spans multiple ubatches
constexpr llama_pos p0 = n_prompt - n_rollback;
// both contexts have the same layout, only what they decode differs
const auto make_ctx_multi = [&]() {
ctx_opts opts;
opts.n_ctx = 256;
opts.n_batch = 256;
opts.n_ubatch = n_ubatch;
opts.n_seq_max = n_seqs;
opts.n_rs_seq = 8;
opts.kv_unified = 0;
return mr.ctx(opts);
};
llama_context_ptr ctx_roll = make_ctx_multi();
llama_context_ptr ctx_ref = make_ctx_multi();
if (!ctx_roll || !ctx_ref) {
return case_result::fail();
}
if (llama_n_rs_seq(ctx_roll.get()) < n_rollback) {
return case_result::skip("n_rs_seq is too small");
}
const auto tok = [&](uint32_t seq, llama_pos pos) {
return gen_token(n_vocab, (llama_seq_id) seq, pos);
};
bool ok = true;
// keep the anchor at p0 - 1 when rolling back the tail
for (uint32_t s = 0; s < n_seqs && ok; ++s) {
ok = ok && decode_gen(ctx_roll.get(), n_vocab, (llama_seq_id) s, 0, (llama_pos) p0 - 1);
ok = ok && decode_gen(ctx_ref.get(), n_vocab, (llama_seq_id) s, 0, (llama_pos) p0 - 1);
ok = ok && decode_gen(ctx_roll.get(), n_vocab, (llama_seq_id) s, (llama_pos) p0 - 1, (llama_pos) n_prompt);
ok = ok && decode_gen(ctx_ref.get(), n_vocab, (llama_seq_id) s, (llama_pos) p0 - 1, (llama_pos) p0);
ok = ok && llama_memory_seq_rm(llama_get_memory(ctx_roll.get()), (llama_seq_id) s, p0, -1);
// a second partial removal while one is pending must be refused
ok = ok && !llama_memory_seq_rm(llama_get_memory(ctx_roll.get()), (llama_seq_id) s, p0 - 1, -1);
}
CTX_REQUIRE(ok, "the multi-seq prefill / rollback failed");
// all seqs replay in a single batch
const auto decode_replay = [&](llama_context * ctx) {
common_batch batch(ctx);
for (uint32_t s = 0; s < n_seqs; ++s) {
for (uint32_t i = 0; i < n_replay; ++i) {
const llama_pos pos = p0 + (llama_pos) i;
batch.add(tok(s, pos), pos, (llama_seq_id) s, true);
}
}
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
};
ok = decode_replay(ctx_roll.get()) && decode_replay(ctx_ref.get());
CTX_REQUIRE(ok, "the multi-seq replay decode failed");
// both contexts decode identical batches, so the logits should match;
// random dummy models can still drift up to ~1.7e-5, so the bound is 1e-4
constexpr float nmse_eps = 1e-4f;
logits_diff diff;
for (uint32_t i = 0; i < n_seqs*n_replay; ++i) {
const float * l_roll = llama_get_logits_ith(ctx_roll.get(), i);
const float * l_ref = llama_get_logits_ith(ctx_ref.get(), i);
if (l_roll == nullptr || l_ref == nullptr) {
TLOG_ERR("%s: missing multi-seq logits at index %u\n", __func__, i);
return case_result::fail();
}
diff.add(l_roll, l_ref, n_vocab);
}
CTX_REQUIRE(diff.nmse() <= nmse_eps,
"multi-seq split replay logits mismatch (max diff " + std::to_string(diff.diff_max) +
", nmse " + std::to_string(diff.nmse()) + ", first difference at index " + std::to_string(diff.i_first) + ")");
TLOG_INF("%s: multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff.diff_max, diff.nmse());
// seq-1-only decodes must be independent of seq 0's content: diverge seq 0
// in ctx_ref only, then compare identical seq-1-only continuations
constexpr uint32_t n_tail = 4;
{
common_batch batch_tail(ctx_ref.get());
for (uint32_t i = 0; i < n_tail; ++i) {
const llama_pos pos = p0 + (llama_pos) (n_replay + i);
batch_tail.add(tok(0, pos + 7), pos, 0, false);
}
ok = llama_process(ctx_ref.get(), LLAMA_PROCESS_TYPE_DECODE, batch_tail.get()) == 0;
}
logits_diff diff_tail;
for (uint32_t i = 0; i < n_tail && ok; ++i) {
const llama_pos pos = p0 + (llama_pos) (n_replay + i);
ok = decode_one(ctx_roll.get(), tok(1, pos), pos, 1) &&
decode_one(ctx_ref.get(), tok(1, pos), pos, 1);
if (!ok) {
break;
}
const float * l_roll = llama_get_logits_ith(ctx_roll.get(), 0);
const float * l_ref = llama_get_logits_ith(ctx_ref.get(), 0);
if (l_roll == nullptr || l_ref == nullptr) {
TLOG_ERR("%s: missing tail logits at step %u\n", __func__, i);
return case_result::fail();
}
diff_tail.add(l_roll, l_ref, n_vocab);
}
CTX_REQUIRE(ok && diff_tail.nmse() <= nmse_eps,
"a seq-1-only decode leaked the state of seq 0 (nmse " + std::to_string(diff_tail.nmse()) + ")");
TLOG_INF("%s: seq-1-only decode is independent of seq 0 (max diff %g, nmse %g)\n",
__func__, (double) diff_tail.diff_max, diff_tail.nmse());
return case_result::pass();
}
};
// save a rolled-back single-seq state, restore it into a fresh and into a dirty context, and
// verify exact logit matches on replay
struct case_rollback : ctx_test {
test_group group() const override { return TEST_GROUP_RS; }
const char * name() const override { return "rollback"; }
const char * description() const override {
return "a rolled-back recurrent state restores into fresh and dirty contexts with identical replay";
}
case_result run(model_run & mr) const override {
const llama_vocab * vocab = llama_model_get_vocab(mr.model);
const int n_vocab = llama_vocab_n_tokens(vocab);
const auto make_rs_ctx = [&]() {
ctx_opts opts;
opts.n_seq_max = 1;
opts.n_rs_seq = 8;
return mr.ctx(opts);
};
llama_context_ptr ctx_src = make_rs_ctx();
llama_context_ptr ctx_dst = make_rs_ctx();
if (!ctx_src || !ctx_dst) {
return case_result::fail();
}
const uint32_t n_rs_seq = llama_n_rs_seq(ctx_src.get());
if (n_rs_seq == 0) {
return case_result::skip("n_rs_seq is disabled");
}
constexpr uint32_t n_rollback = 3;
if (n_rs_seq < n_rollback) {
return case_result::skip("n_rs_seq is too small");
}
llama_tokens tokens;
if (!mr.has_vocab()) {
tokens = { 1, 2, 3, 4, 5, 6, 7, 8, 9 };
} else {
tokens = common_tokenize(ctx_src.get(), "The quick brown fox jumps over the lazy dog", true);
}
if (tokens.empty()) {
TLOG_ERR("%s: not enough prompt tokens\n", __func__);
return case_result::fail();
}
tokens.resize(n_rs_seq + 1, tokens.back());
const llama_pos rollback_pos = (llama_pos) tokens.size() - n_rollback;
// Decode the full prompt on the source, then roll back three positions.
// Replaying them crosses DSV4's ratio-4 compressor boundary.
// Rollback leaves the recurrent memory in a snapshot state (rs_idx != 0).
if (!decode_tokens(ctx_src.get(), tokens)) {
TLOG_ERR("%s: failed to decode the prompt\n", __func__);
return case_result::fail();
}
if (!llama_memory_seq_rm(llama_get_memory(ctx_src.get()), 0, rollback_pos, -1)) {
TLOG_ERR("%s: rollback failed\n", __func__);
return case_result::fail();
}
// Save the rolled-back state and restore it into a fresh context.
common_prompt_checkpoint ckpt;
ckpt.update_tgt(ctx_src.get(), 0, 0);
ckpt.load_tgt(ctx_dst.get(), 0, 0);
// exact match - the same context configuration replaying the same tokens
constexpr float nmse_eps = 0.0;
std::vector<std::vector<float>> logits_src_replay(n_rollback);
for (uint32_t i = 0; i < n_rollback; ++i) {
const llama_pos pos = rollback_pos + i;
if (!decode_one(ctx_src.get(), tokens[pos], pos) ||
!decode_one(ctx_dst.get(), tokens[pos], pos)) {
TLOG_ERR("%s: replay failed at position %d\n", __func__, pos);
return case_result::fail();
}
const float * logits_src = llama_get_logits_ith(ctx_src.get(), 0);
const float * logits_dst = llama_get_logits_ith(ctx_dst.get(), 0);
if (logits_src == nullptr || logits_dst == nullptr) {
TLOG_ERR("%s: missing replay logits at position %d\n", __func__, pos);
return case_result::fail();
}
logits_src_replay[i].assign(logits_src, logits_src + n_vocab);
const logits_diff diff = compare_logits(logits_src, logits_dst, n_vocab);
CTX_REQUIRE(diff.nmse() <= nmse_eps,
"replay logits mismatch at position " + std::to_string(pos) + ", nmse " + std::to_string(diff.nmse()));
}
// TODO: this test is invalid because RS rollback is only correct once after a ubatch with more than n_rs_seq tokens
// this is not the case here. add asserts and guardrails to prevent such attempts
//if (!llama_memory_seq_rm(llama_get_memory(ctx_src.get()), 0, rollback_pos, -1) ||
// !llama_memory_seq_rm(llama_get_memory(ctx_dst.get()), 0, rollback_pos, -1)) {
// TLOG_ERR("%s: partial rollback failed\n", __func__);
// return case_result::fail();
//}
//constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY;
//common_prompt_checkpoint ckpt_partial;
//ckpt_partial.update_tgt(ctx_src.get(), 0, partial_flags);
//ckpt_partial.load_tgt(ctx_dst.get(), 0, partial_flags);
// Repeat the load into a context that already has its own rollback state:
// groups 1..n_rs_seq hold a different prompt's history, and rs_idx[0] is
// non-zero at load time. The restore must wipe that state and still match.
llama_context_ptr ctx_dirty = make_rs_ctx();
if (!ctx_dirty) {
return case_result::fail();
}
llama_tokens noise = tokens;
for (auto & tok : noise) {
tok = (tok + 1) % n_vocab;
if (tok < 0) {
tok = 0;
}
}
if (!decode_tokens(ctx_dirty.get(), noise)) {
TLOG_ERR("%s: failed to decode the dirty prompt\n", __func__);
return case_result::fail();
}
if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty.get()), 0, rollback_pos, -1)) {
TLOG_ERR("%s: dirty rollback failed\n", __func__);
return case_result::fail();
}
ckpt.load_tgt(ctx_dirty.get(), 0, 0);
for (uint32_t i = 0; i < n_rollback; ++i) {
const llama_pos pos = rollback_pos + i;
if (!decode_one(ctx_dirty.get(), tokens[pos], pos)) {
TLOG_ERR("%s: dirty replay failed at position %d\n", __func__, pos);
return case_result::fail();
}
const float * logits_dirty = llama_get_logits_ith(ctx_dirty.get(), 0);
if (logits_dirty == nullptr) {
TLOG_ERR("%s: missing dirty logits at position %d\n", __func__, pos);
return case_result::fail();
}
const logits_diff diff = compare_logits(logits_src_replay[i].data(), logits_dirty, n_vocab);
CTX_REQUIRE(diff.nmse() <= nmse_eps,
"dirty-ctx logits mismatch at position " + std::to_string(pos) + ", nmse " + std::to_string(diff.nmse()));
}
TLOG_INF("%s: recurrent rollback checkpoint restored successfully\n", __func__);
return case_result::pass();
}
};
// decode a prompt into seq 0, share its cells with a second sequence, then keep decoding both
struct case_shared_seq_reserve : ctx_test {
test_group group() const override { return TEST_GROUP_RS; }
const char * name() const override { return "shared"; }
const char * description() const override {
return "sequences sharing cells keep decoding correctly after a seq_cp";
}
std::string skip_reason(model_run & mr) const override {
if (arch_reserves_single_seq(mr.model)) {
return "the shared-seq batch is a multi-seq graph";
}
return ctx_test::skip_reason(mr);
}
case_result run(model_run & mr) const override {
const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(mr.model));
constexpr uint32_t n_seqs = 2;
constexpr uint32_t n_prompt = 128;
constexpr uint32_t n_continue = 32;
ctx_opts opts;
opts.n_ctx = 512;
opts.n_batch = 256;
opts.n_ubatch = 64;
opts.n_seq_max = n_seqs;
opts.kv_unified = 1; // only a unified cache shares cells on seq_cp
auto ctx = mr.ctx(opts);
if (!ctx) {
return case_result::fail();
}
if (!decode_gen(ctx.get(), n_vocab, 0, 0, (llama_pos) n_prompt)) {
TLOG_ERR("%s: prompt decode failed\n", __func__);
return case_result::fail();
}
// this is what llama-batched-bench does for -pps
llama_memory_seq_cp(llama_get_memory(ctx.get()), 0, 1, -1, -1);
for (uint32_t i = 0; i < n_continue; ++i) {
const llama_pos pos = (llama_pos) (n_prompt + i);
common_batch batch(ctx.get());
for (uint32_t s = 0; s < n_seqs; ++s) {
batch.add(gen_token(n_vocab, (llama_seq_id) s, pos), pos, (llama_seq_id) s, true);
}
CTX_REQUIRE(llama_process(ctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0,
"shared-seq decode failed at step " + std::to_string(i));
for (uint32_t s = 0; s < n_seqs; ++s) {
const float * logits = llama_get_logits_ith(ctx.get(), (int) s);
if (logits == nullptr) {
TLOG_ERR("%s: missing shared-seq logits at index %u\n", __func__, s);
return case_result::fail();
}
bool finite = true;
for (int t_ = 0; t_ < n_vocab; ++t_) {
finite = finite && std::isfinite(logits[t_]);
}
CTX_REQUIRE(finite, "non-finite shared-seq logit at step " + std::to_string(i) + ", seq " + std::to_string(s));
}
}
TLOG_INF("%s: shared-seq decode succeeded (%u tokens after seq_cp)\n", __func__, n_continue*n_seqs);
return case_result::pass();
}
};
std::vector<ctx_test_ptr> make_rs_cases() {
std::vector<ctx_test_ptr> cases;
cases.push_back(std::make_unique<case_rollback>());
cases.push_back(std::make_unique<case_multi_seq_split_replay>());
cases.push_back(std::make_unique<case_shared_seq_reserve>());
return cases;
}
llama.cpp/tests/llama-context/test-case.cpp
+69
View File
@@ -0,0 +1,69 @@
#include "test-case.h"
#include <iterator>
const char * test_status_str(test_status status) {
switch (status) {
case test_status::PASS: return "\033[1;32mOK\033[0m";
case test_status::FAIL: return "\033[1;31mFAIL\033[0m";
case test_status::SKIP: return "\033[1;33mSKIP\033[0m";
}
return "";
}
// the printable width of a verdict, the color escapes of test_status_str are not part of it
size_t test_status_width(test_status status) {
switch (status) {
case test_status::PASS: return 2; // "OK"
case test_status::FAIL: return 4; // "FAIL"
case test_status::SKIP: return 4; // "SKIP"
}
return 0;
}
test_status merge_status(test_status a, test_status b) {
if (a == test_status::FAIL || b == test_status::FAIL) {
return test_status::FAIL;
}
if (a == test_status::PASS || b == test_status::PASS) {
return test_status::PASS;
}
return test_status::SKIP;
}
case_result case_result::pass() {
return { test_status::PASS, {} };
}
case_result case_result::fail(const std::string & why) {
return { test_status::FAIL, why };
}
case_result case_result::skip(const std::string & why) {
return { test_status::SKIP, why };
}
std::string ctx_test::skip_reason(model_run & mr) const {
// the recurrent cases need a recurrent cache, the KV cases work with any model
if (group() == TEST_GROUP_RS && !mr.is_recurrent()) {
return "not a recurrent model";
}
return {};
}
const std::vector<ctx_test_ptr> & test_cases() {
static const std::vector<ctx_test_ptr> cases = [] {
std::vector<ctx_test_ptr> res;
for (const auto & maker : { make_kv_cases, make_rs_cases }) {
auto part = maker();
res.insert(res.end(), std::make_move_iterator(part.begin()), std::make_move_iterator(part.end()));
}
return res;
}();
return cases;
}
llama.cpp/tests/llama-context/test-case.h
+83
View File
@@ -0,0 +1,83 @@
#pragma once
#include "test-util.h"
#include <memory>
#include <string>
#include <vector>
//
// the interface of a single context state case
//
enum class test_status {
PASS,
FAIL,
SKIP,
};
// the verdict as printed in the result table, colored
const char * test_status_str(test_status status);
// how many columns the verdict takes in the table - the escapes of the color are not printed
size_t test_status_width(test_status status);
// a skipped case does not fail the run, a failed case always does
test_status merge_status(test_status a, test_status b);
enum test_group {
TEST_GROUP_STATE, // KV state save / load / copy
TEST_GROUP_RS, // recurrent state, also run against a dirty cache
};
struct case_result {
test_status status = test_status::PASS;
std::string reason; // why the case was skipped or failed, empty when it passed
static case_result pass();
static case_result fail(const std::string & why = {});
static case_result skip(const std::string & why);
};
struct ctx_test {
virtual ~ctx_test() = default;
// the column header of the --models table, keep it short
virtual const char * name() const = 0;
// one line of what the case verifies, printed by --list
virtual const char * description() const { return ""; }
virtual test_group group() const { return TEST_GROUP_STATE; }
// the case compares against the logits produced by the baseline case
virtual bool needs_baseline() const { return false; }
// the case is also run against a cache pre-filled with a known pattern, so that a restore
// reading cells it never wrote cannot pass
virtual bool uses_fill() const { return group() == TEST_GROUP_RS; }
// non-empty means the case does not apply to this model, the text is the reported reason
virtual std::string skip_reason(model_run & mr) const;
// the scenario, the checks are written with CTX_REQUIRE
virtual case_result run(model_run & mr) const = 0;
};
// record an expectation that did not hold and give up on the case - the runner reports the
// reason, the body does not print anything
#define CTX_REQUIRE(cond, msg) \
do { \
if (!(cond)) { \
return case_result::fail(msg); \
} \
} while (0)
using ctx_test_ptr = std::unique_ptr<const ctx_test>;
// the cases of one source file, in table order
std::vector<ctx_test_ptr> make_kv_cases(); // cases-kv.cpp
std::vector<ctx_test_ptr> make_rs_cases(); // cases-rs.cpp
// all cases, in table order
const std::vector<ctx_test_ptr> & test_cases();
llama.cpp/tests/llama-context/test-log.cpp
+119
View File
@@ -0,0 +1,119 @@
#include "test-log.h"
#include <algorithm>
#include <cstdarg>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <iterator>
namespace {
// a log is printed after the row of its model, what does not fit stays in the file
const size_t LOG_DUMP_MAX_BYTES = 64 * 1024;
// one worker thread runs one model and the log callback fires on the thread that logged the
// message, so the file of a model is held by the thread that runs it
thread_local FILE * t_file;
char level_char(enum ggml_log_level level) {
switch (level) {
case GGML_LOG_LEVEL_DEBUG: return 'D';
case GGML_LOG_LEVEL_INFO: return 'I';
case GGML_LOG_LEVEL_WARN: return 'W';
case GGML_LOG_LEVEL_ERROR: return 'E';
case GGML_LOG_LEVEL_CONT: return 'C';
default: return '?';
}
}
// every line of a model log carries its level, so that a failure can be looked up with a grep
void test_log_level(enum ggml_log_level level) {
if (level != GGML_LOG_LEVEL_NONE) {
fprintf(t_file, "%c ", level_char(level));
}
}
} // namespace
void test_log_capture(const std::string & path) {
if (t_file) {
fclose(t_file);
t_file = nullptr;
}
if (path.empty()) {
return;
}
t_file = fopen(path.c_str(), "w");
if (!t_file) {
fprintf(stderr, "%s: failed to open '%s'\n", __func__, path.c_str());
}
}
void test_log_callback(enum ggml_log_level level, const char * text, void * user_data) {
if (!t_file) {
common_log_default_callback(level, text, user_data);
return;
}
// gate the capture the way the console would be - the debug chatter of the graph builder is
// orders of magnitude more than everything else, a capture never holds less than info
if (common_log_get_verbosity(level) <= std::max(common_log_get_verbosity_thold(), LOG_LEVEL_INFO)) {
test_log_level(level);
fputs(text, t_file);
}
}
void test_log_add(enum ggml_log_level level, const char * fmt, ...) {
va_list args;
va_start(args, fmt);
if (t_file) {
test_log_level(level);
vfprintf(t_file, fmt, args);
} else if (level == GGML_LOG_LEVEL_ERROR) {
// without a capture the run is followed on the console, a detail line has no place there
vfprintf(stderr, fmt, args);
fflush(stderr);
}
va_end(args);
}
bool test_log_dump(const std::string & path) {
std::ifstream f(path, std::ios::binary);
if (!f) {
fprintf(stderr, "failed to read the captured log '%s'\n", path.c_str());
return false;
}
const std::string text((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
f.close();
size_t skip = 0;
if (text.size() > LOG_DUMP_MAX_BYTES) {
skip = text.find('\n', text.size() - LOG_DUMP_MAX_BYTES);
skip = skip == std::string::npos ? 0 : skip + 1;
}
if (skip > 0) {
const size_t n_all = std::count(text.begin(), text.end(), '\n');
const size_t n_shown = std::count(text.begin() + skip, text.end(), '\n');
fprintf(stdout, "... %zu of %zu lines omitted, full log: %s\n", n_all - n_shown, n_all, path.c_str());
}
fwrite(text.data() + skip, 1, text.size() - skip, stdout);
fflush(stdout);
return skip > 0;
}
std::string test_log_path(const std::string & logs_dir, const std::string & model_name) {
const std::string file = "test-llama-context." + model_name + ".log";
return logs_dir.empty() ? file : (std::filesystem::path(logs_dir) / file).string();
}
llama.cpp/tests/llama-context/test-log.h
+33
View File
@@ -0,0 +1,33 @@
#pragma once
#include "log.h"
#include <string>
//
// the log of a model goes to a file of its own, so that a model which failed can be read on its
// own. a worker thread runs one model and the log callback fires on the thread that produced the
// message, so a capture belongs to the thread that opened it
//
// open the log of this thread, an empty path closes it - a capture holds what the verbosity of
// the process allows, never less than info, the details of a failure are looked for in the file
void test_log_capture(const std::string & path);
// llama, ggml and the backends log through this: into the file of this thread when it captures,
// into the default callback otherwise
void test_log_callback(enum ggml_log_level level, const char * text, void * user_data);
// the lines of the test itself go to the file of this thread, an error goes to stderr as well when
// there is no capture and the run is followed on the console
void test_log_add(enum ggml_log_level level, const char * fmt, ...) LOG_ATTRIBUTE_FORMAT(2, 3);
// print a captured log, false when it was printed in full and the file can be removed
bool test_log_dump(const std::string & path);
std::string test_log_path(const std::string & logs_dir, const std::string & model_name);
// a trace is a detail of a case, it is there to read in the file of the model
#define TLOG_ERR(...) test_log_add(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
#define TLOG_INF(...) test_log_add(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
#define TLOG_TRC(...) test_log_add(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
llama.cpp/tests/llama-context/test-runner.cpp
+117
View File
@@ -0,0 +1,117 @@
#include "test-runner.h"
#include "test-log.h"
#include <algorithm>
#include <cstdio>
#include <cstring>
#include <regex>
#include <utility>
std::vector<const ctx_test *> select_cases(const std::string & filter) {
std::vector<const ctx_test *> selected;
if (filter.empty()) {
selected.reserve(test_cases().size());
for (const auto & tc : test_cases()) {
selected.push_back(tc.get());
}
return selected;
}
const std::regex re(filter);
for (const auto & tc : test_cases()) {
if (std::regex_match(tc->name(), re)) {
selected.push_back(tc.get());
}
}
return selected;
}
std::vector<case_run> run_cases(model_run & mr, const std::vector<const ctx_test *> & cases, bool report) {
// the state cases keep the historical default of a unified KV cache when running a single
// sequence, the recurrent cases use whatever was passed on the command line
const bool unified_base = mr.params.kv_unified;
const bool unified_state = mr.params.n_parallel == 1 ? true : unified_base;
size_t name_width = 4; // "case"
for (const ctx_test * tc : cases) {
name_width = std::max(name_width, strlen(tc->name()));
}
std::vector<case_run> runs;
runs.reserve(cases.size());
for (const ctx_test * tc : cases) {
mr.params.kv_unified = tc->group() == TEST_GROUP_STATE ? unified_state : unified_base;
case_run run;
run.name = tc->name();
// the log of a model is read on its own, mark where each case begins
TLOG_INF("%s\n", run.name.c_str());
// a case that throws is a case that failed, and the run goes on with the next one
try {
if (const std::string why = tc->skip_reason(mr); !why.empty()) {
run.status = test_status::SKIP;
run.reason = why;
} else if (tc->needs_baseline() && !mr.have_baseline) {
run.status = test_status::SKIP;
run.reason = "no baseline";
} else if (!tc->uses_fill()) {
mr.fill = 0;
const case_result res = tc->run(mr);
run.status = res.status;
run.reason = res.reason;
} else {
// the recurrent cases are also run against a cache pre-filled with a known
// pattern, so that a restore reading cells it never wrote cannot pass
for (const uint8_t fill : { 0, 0x3e }) {
TLOG_INF("(fill 0x%02x)\n", fill);
mr.fill = fill;
const case_result res = tc->run(mr);
run.status = merge_status(run.status, res.status);
if (run.reason.empty()) {
run.reason = res.reason;
}
if (res.status == test_status::FAIL) {
break;
}
}
}
} catch (...) {
run.status = test_status::FAIL;
run.reason = "unhandled exception";
}
if (unified_state) {
mr.params.kv_unified = unified_base;
}
// the verdict is one word, what it means goes to the log of the model
if (!run.reason.empty()) {
TLOG_INF("%s %s: %s\n", run.name.c_str(),
run.status == test_status::FAIL ? "failed" : "skipped", run.reason.c_str());
}
if (report) {
fprintf(stderr, "%-*s %s%s%s\n", (int) name_width, run.name.c_str(),
test_status_str(run.status),
run.reason.empty() ? "" : " ", run.reason.c_str());
}
runs.push_back(std::move(run));
}
return runs;
}
llama.cpp/tests/llama-context/test-runner.h
+23
View File
@@ -0,0 +1,23 @@
#pragma once
#include "test-case.h"
#include <string>
#include <vector>
//
// the runner: the single place that decides how a case is executed
//
struct case_run {
std::string name;
test_status status = test_status::SKIP;
std::string reason;
};
// the cases selected by the filter, all of them when it is empty, in table order
std::vector<const ctx_test *> select_cases(const std::string & filter);
// run the selected cases for one model and return one verdict per case - with report, print the
// verdict of every case as it finishes, for single model mode
std::vector<case_run> run_cases(model_run & mr, const std::vector<const ctx_test *> & cases, bool report);
llama.cpp/tests/llama-context/test-util.cpp
+403
View File
@@ -0,0 +1,403 @@
#include "test-util.h"
#include "test-log.h"
#include "../../src/llama-ext.h" // llama_memory_get_buffers
#include "common.h"
#include <cmath>
#include <cstdio>
#include <cstring>
#include <limits>
#include <stdexcept>
const double NMSE_THRESHOLD = 1e-5;
//
// comparison
//
// the distance of two logits, infinite when either of them is not finite
static float logit_diff(float a, float b) {
return std::isfinite(a) && std::isfinite(b) ? std::fabs(a - b) : std::numeric_limits<float>::infinity();
}
void logits_diff::add(const float * a, const float * b, size_t n_elem) {
for (size_t i = 0; i < n_elem; i++) {
const float diff = logit_diff(a[i], b[i]);
if (diff > 0.0f && i_first < 0) {
i_first = (int64_t) (n + i);
}
if (diff > diff_max) {
diff_max = diff;
}
if (std::isfinite(a[i]) && std::isfinite(b[i])) {
const double d = (double) a[i] - (double) b[i];
mse_ab += d*d;
mse_a0 += (double) a[i] * (double) a[i];
} else {
n_nan++;
mse_ab = std::numeric_limits<double>::infinity();
mse_a0 = 1.0;
}
}
n += n_elem;
}
double logits_diff::nmse() const {
if (mse_a0 == 0.0) {
return mse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity();
}
return mse_ab / mse_a0;
}
logits_diff compare_logits(const float * a, const float * b, size_t n) {
logits_diff res;
res.add(a, b, n);
return res;
}
//
// decode
//
bool get_current_logits(llama_context * ctx, std::vector<float> & out) {
const auto * vocab = llama_model_get_vocab(llama_get_model(ctx));
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
const float * logits = llama_get_logits_ith(ctx, -1);
if (logits == nullptr) {
return false;
}
out.assign(logits, logits + n_vocab);
return true;
}
bool decode_seq(llama_context * ctx, const llama_tokens & tokens, llama_pos p_begin, llama_pos p_end,
llama_seq_id seq_id, bool last_output) {
common_batch batch(ctx);
for (llama_pos pos = p_begin; pos < p_end; ++pos) {
batch.add(tokens[pos], pos, seq_id, last_output && pos + 1 == p_end);
}
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
}
bool decode_tokens(llama_context * ctx, const llama_tokens & tokens, llama_seq_id seq_id) {
return decode_seq(ctx, tokens, 0, (llama_pos) tokens.size(), seq_id, true);
}
llama_token gen_token(int32_t n_vocab, llama_seq_id seq_id, llama_pos pos) {
return (llama_token) ((7*(uint32_t) pos + 31*(uint32_t) seq_id + 1) % (uint32_t) n_vocab);
}
bool decode_gen(llama_context * ctx, int32_t n_vocab, llama_seq_id seq_id, llama_pos p_begin, llama_pos p_end) {
common_batch batch(ctx);
for (llama_pos pos = p_begin; pos < p_end; ++pos) {
batch.add(gen_token(n_vocab, seq_id, pos), pos, seq_id, false);
}
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
}
bool decode_one(llama_context * ctx, llama_token tok, llama_pos pos, llama_seq_id seq_id) {
common_batch batch(ctx);
batch.add(tok, pos, seq_id, true);
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
}
bool generate_tokens(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id,
generation_result & result) {
common_batch batch(ctx);
// the tokens of the run as a single line - one entry per token would leave the next message
// of the log glued to the last one
std::string tokens;
for (int32_t i = 0; i < n_predict; i++) {
std::vector<float> logits;
if (!get_current_logits(ctx, logits)) {
TLOG_ERR("%s: failed to get logits\n", __func__);
return false;
}
const auto next_token = llama_sampler_sample(smpl, ctx, -1);
tokens += std::to_string(next_token) + " ";
result.tokens.push_back(next_token);
result.logits.push_back(std::move(logits));
batch.clear();
batch.add(next_token, n_past, seq_id, true);
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
TLOG_ERR("%s: failed to evaluate\n", __func__);
return false;
}
n_past++;
}
TLOG_INF("%s: %s\n", __func__, tokens.c_str());
return true;
}
bool generate_tokens_compare(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id,
const generation_result & expected, double nmse_eps) {
if (expected.tokens.size() != expected.logits.size() || expected.tokens.size() < (size_t) n_predict) {
TLOG_ERR("%s: invalid expected generation\n", __func__);
return false;
}
common_batch batch(ctx);
std::string tokens;
for (int32_t i = 0; i < n_predict; i++) {
std::vector<float> logits;
if (!get_current_logits(ctx, logits)) {
TLOG_ERR("%s: failed to get logits\n", __func__);
return false;
}
if (logits.size() != expected.logits[i].size()) {
TLOG_ERR("%s: logits size mismatch at step %d: %zu != %zu\n", __func__, i, logits.size(), expected.logits[i].size());
return false;
}
const logits_diff diff = compare_logits(expected.logits[i].data(), logits.data(), logits.size());
TLOG_TRC("%s: step %d nmse = %.6e\n", __func__, i, diff.nmse());
if (diff.nmse() > nmse_eps) {
TLOG_ERR("%s: error: NMSE at step %d is %.6e (threshold %.1e)\n", __func__, i, diff.nmse(), nmse_eps);
return false;
}
const auto next_token = llama_sampler_sample(smpl, ctx, -1);
const auto expected_token = expected.tokens[i];
tokens += std::to_string(next_token) + " ";
if (next_token != expected_token) {
TLOG_TRC("%s: sampled token %d differs from expected %d, using expected token\n", __func__, next_token, expected_token);
}
batch.clear();
batch.add(expected_token, n_past, seq_id, true);
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
TLOG_ERR("%s: failed to evaluate\n", __func__);
return false;
}
n_past++;
}
TLOG_INF("%s: %s\n", __func__, tokens.c_str());
return true;
}
//
// state
//
bool get_seq_state(llama_context * ctx, llama_seq_id seq_id, uint32_t flags, std::vector<uint8_t> & state) {
const size_t state_size = llama_state_seq_get_size_ext(ctx, seq_id, flags);
if (state_size == 0) {
TLOG_ERR("%s: sequence state is empty\n", __func__);
return false;
}
state.resize(state_size);
const size_t ncopy = llama_state_seq_get_data_ext(ctx, state.data(), state.size(), seq_id, flags);
if (ncopy != state.size()) {
TLOG_ERR("%s: sequence state length %zu does not match expected length %zu\n",
__func__, ncopy, state.size());
return false;
}
return true;
}
bool set_seq_state(llama_context * ctx, llama_seq_id seq_id, uint32_t flags, const std::vector<uint8_t> & state) {
const size_t nset = llama_state_seq_set_data_ext(ctx, state.data(), state.size(), seq_id, flags);
if (nset != state.size()) {
TLOG_ERR("%s: seq set data length %zu does not match expected length %zu\n", __func__, nset, state.size());
return false;
}
return true;
}
bool seq_rm(llama_context * ctx, llama_seq_id seq_id) {
if (!llama_memory_seq_rm(llama_get_memory(ctx), seq_id, -1, -1)) {
TLOG_ERR("%s: failed to remove sequence %d\n", __func__, seq_id);
return false;
}
return true;
}
bool load_state_replay_last(llama_context * ctx, const std::string & path, const llama_tokens & tokens, int & n_past) {
llama_tokens unused_sts(tokens.size());
size_t n_token_count_out = 0;
if (!llama_state_load_file(ctx, path.c_str(), unused_sts.data(), unused_sts.size(), &n_token_count_out)) {
TLOG_ERR("%s: failed to load state\n", __func__);
return false;
}
TLOG_TRC("%s: loaded state with %zu tokens\n", __func__, n_token_count_out);
// the session is saved before the last token, so replay it at the position it was saved at
n_past = (int) n_token_count_out - 1;
if (!common_replay_last_token(ctx, tokens.back(), n_past)) {
return false;
}
n_past++;
return true;
}
bool blobs_equal(const std::vector<uint8_t> & a, const std::vector<uint8_t> & b) {
if (a.size() == b.size() && (a.empty() || memcmp(a.data(), b.data(), a.size()) == 0)) {
return true;
}
size_t i = 0;
while (i < a.size() && i < b.size() && a[i] == b[i]) {
++i;
}
TLOG_ERR("%s: %zu bytes -> %zu bytes, first difference at byte %zu (%02x -> %02x)\n",
__func__, a.size(), b.size(), i,
i < a.size() ? a[i] : 0,
i < b.size() ? b[i] : 0);
return false;
}
bool corrupt_state(std::vector<uint8_t> & data) {
if (data.size() < 3*4096) {
TLOG_ERR("%s: state of %zu bytes is too small to corrupt\n", __func__, data.size());
return false;
}
std::fill(data.begin() + 4096, data.end() - data.size()/4, 0xff);
return true;
}
//
// context
//
// fill the buffers of the cache before a restore - a restore that reads cells it never wrote
// then produces garbage instead of silently passing
llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill) {
llama_context_ptr ctx{llama_init_from_model(model, cparams)};
if (!ctx || fill == 0) {
return ctx;
}
// Use a full ubatch so buffer discovery preserves prefill allocation sizes.
const uint32_t n_tokens = llama_n_ubatch(ctx.get());
if (!decode_tokens(ctx.get(), llama_tokens(n_tokens, 0))) {
return nullptr;
}
llama_synchronize(ctx.get());
const auto buffers = llama_memory_get_buffers(llama_get_memory(ctx.get()));
llama_memory_clear(llama_get_memory(ctx.get()), true);
if (buffers.empty()) {
TLOG_ERR("%s: no cache buffers found\n", __func__);
return nullptr;
}
for (auto * buffer : buffers) {
ggml_backend_buffer_clear(buffer, fill);
}
return ctx;
}
llama_context_ptr make_ctx(llama_model * model, const common_params & params, const ctx_opts & opts) {
auto cparams = common_context_params_to_llama(params);
if (opts.n_ctx) {
cparams.n_ctx = opts.n_ctx;
}
if (opts.n_seq_max) {
cparams.n_seq_max = opts.n_seq_max;
}
if (opts.kv_unified >= 0) {
cparams.kv_unified = opts.kv_unified != 0;
}
if (opts.n_rs_seq) {
cparams.n_rs_seq = opts.n_rs_seq;
cparams.n_batch = std::max(cparams.n_batch, (uint32_t) (opts.n_rs_seq + 1));
cparams.n_ubatch = std::max(cparams.n_ubatch, (uint32_t) (opts.n_rs_seq + 1));
}
if (opts.n_batch) {
cparams.n_batch = opts.n_batch;
}
if (opts.n_ubatch) {
cparams.n_ubatch = opts.n_ubatch;
}
if (opts.fa != LLAMA_FLASH_ATTN_TYPE_AUTO) {
cparams.flash_attn_type = opts.fa;
}
if (opts.type_k != GGML_TYPE_COUNT) {
cparams.type_k = opts.type_k;
}
if (opts.type_v != GGML_TYPE_COUNT) {
cparams.type_v = opts.type_v;
}
return init_ctx(model, cparams, opts.fill);
}
llama_sampler_ptr make_dist_sampler(uint32_t seed) {
auto sparams = llama_sampler_chain_default_params();
llama_sampler_ptr smpl{llama_sampler_chain_init(sparams)};
llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(seed));
return smpl;
}
bool arch_reserves_single_seq(const llama_model * model) {
char arch_str[64] = {};
llama_model_meta_val_str(model, "general.architecture", arch_str, sizeof(arch_str));
return strcmp(arch_str, "kimi-linear") == 0 || strcmp(arch_str, "minimax-01") == 0;
}
//
// model_run
//
llama_context_ptr model_run::ctx(const ctx_opts & opts) const {
ctx_opts effective = opts;
effective.fill = fill;
auto ctx = make_ctx(model, params, effective);
if (!ctx && !opts.quiet) {
TLOG_ERR("%s: failed to create context for '%s'\n", __func__, name.c_str());
}
return ctx;
}
llama_sampler_ptr model_run::sampler() const {
return make_dist_sampler(params.sampling.seed);
}
bool model_run::is_recurrent() const {
return llama_model_is_recurrent(model) || llama_model_is_hybrid(model);
}
bool model_run::has_vocab() const {
return llama_vocab_type(llama_model_get_vocab(model)) != LLAMA_VOCAB_TYPE_NONE;
}
llama.cpp/tests/llama-context/test-util.h
+161
View File
@@ -0,0 +1,161 @@
#pragma once
#include "llama-cpp.h"
#include "llama.h"
#include "common.h"
#include <cstdint>
#include <string>
#include <vector>
//
// shared helpers of the context state cases
//
extern const double NMSE_THRESHOLD;
//
// comparison
//
// normalized mean squared error = mse(a, b) / mse(a, 0), infinity if any logit is not finite
// accumulates logit comparisons and derives their NMSE, so that a case can compare an entire
// multi-seq batch row by row and still have a single verdict
struct logits_diff {
double mse_ab = 0.0;
double mse_a0 = 0.0;
float diff_max = 0.0f;
size_t n_nan = 0;
size_t n = 0;
int64_t i_first = -1; // flat index of the first differing element
void add(const float * a, const float * b, size_t n_elem);
double nmse() const;
};
logits_diff compare_logits(const float * a, const float * b, size_t n);
//
// generation
//
// logits produced by a run, compared against by the replay cases
struct generation_result {
llama_tokens tokens;
std::vector<std::vector<float>> logits;
bool empty() const { return tokens.empty(); }
};
//
// decode
//
bool get_current_logits(llama_context * ctx, std::vector<float> & out);
// decode tokens [p_begin, p_end) of one sequence
bool decode_seq(llama_context * ctx, const llama_tokens & tokens, llama_pos p_begin, llama_pos p_end,
llama_seq_id seq_id, bool last_output);
bool decode_tokens(llama_context * ctx, const llama_tokens & tokens, llama_seq_id seq_id = 0);
// tokens derived from (seq, pos): the recurrent cases do not use the tokenizer, so they do
// not depend on the prompt or on the prompt length
llama_token gen_token(int32_t n_vocab, llama_seq_id seq_id, llama_pos pos);
bool decode_gen(llama_context * ctx, int32_t n_vocab, llama_seq_id seq_id, llama_pos p_begin, llama_pos p_end);
bool decode_one(llama_context * ctx, llama_token tok, llama_pos pos, llama_seq_id seq_id = 0);
// greedy-free replay: sample for reporting, but feed the sampled tokens back
bool generate_tokens(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id,
generation_result & result);
// as generate_tokens, but feeds the expected tokens back so that the compared logits stay on
// the reference trajectory
bool generate_tokens_compare(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id,
const generation_result & expected, double nmse_eps);
//
// state
//
// llama_state_seq_get_size_ext + get_data_ext, with the length verified
bool get_seq_state(llama_context * ctx, llama_seq_id seq_id, uint32_t flags, std::vector<uint8_t> & state);
// llama_state_seq_set_data_ext, with the length verified
bool set_seq_state(llama_context * ctx, llama_seq_id seq_id, uint32_t flags, const std::vector<uint8_t> & state);
// drop a sequence entirely
bool seq_rm(llama_context * ctx, llama_seq_id seq_id);
// load a saved session and replay its last token, leaving n_past ready for the next token
bool load_state_replay_last(llama_context * ctx, const std::string & path, const llama_tokens & tokens, int & n_past);
// false when the sizes differ or any byte differs, the first difference is logged
bool blobs_equal(const std::vector<uint8_t> & a, const std::vector<uint8_t> & b);
// overwrite the tensor data with 0xff bytes (NaN when read as f16/f32), so that the restore fails
bool corrupt_state(std::vector<uint8_t> & data);
//
// context
//
// per-test context overrides, sentinel values mean "take the value from common_params"
struct ctx_opts {
uint32_t n_ctx = 0; // 0 = from params
uint32_t n_batch = 0; // 0 = from params
uint32_t n_ubatch = 0; // 0 = from params
uint32_t n_seq_max = 0; // 0 = from params
uint32_t n_rs_seq = 0; // 0 = from params, n_batch / n_ubatch are raised to fit it
int kv_unified = -1; // -1 = from params, 0 = off, 1 = on
llama_flash_attn_type fa = LLAMA_FLASH_ATTN_TYPE_AUTO; // AUTO = from params
ggml_type type_k = GGML_TYPE_COUNT; // COUNT = from params
ggml_type type_v = GGML_TYPE_COUNT; // COUNT = from params
uint8_t fill = 0; // fill the cache buffers with this byte, 0 = disabled
// a probe that expects the context to be refused should not report it as an error
bool quiet = false;
};
llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill);
llama_context_ptr make_ctx(llama_model * model, const common_params & params, const ctx_opts & opts);
llama_sampler_ptr make_dist_sampler(uint32_t seed);
// these archs reserve the final pp graph with n_seqs = 1, so any multi-seq graph has a
// different layout and re-reserves by design - they must not be run with a multi-seq batch
// when GGML_SCHED_DEBUG_REALLOC is enabled
// see [TAG_RESERVE_DIAG_DECAY] in llama-context.cpp
bool arch_reserves_single_seq(const llama_model * model);
//
// the state of a single model run, handed to every case
//
struct model_run {
common_params params; // per-model copy, out_file is scoped to this model
std::string name; // model file name, used for the table and for temp files
llama_model * model = nullptr;
llama_tokens tokens;
generation_result baseline;
bool have_baseline = false;
uint8_t fill = 0; // set by the runner before each case
// a context with these options, the fill of this run applied; nullptr when it failed to
// init, the reason is logged
llama_context_ptr ctx(const ctx_opts & opts = {}) const;
// a seeded distribution sampler
llama_sampler_ptr sampler() const;
bool is_recurrent() const;
bool has_vocab() const;
};