include llama.cpp since there were some edits
This commit is contained in:
zefie committed
1 parent
db7f28d087
commit
8c66990853
3918 files changed
+1640170
-1
No files matched your search
@@ -0,0 +1,139 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Cohere2 MoE (a.k.a. "North Code") parser.
|
||||
//
|
||||
// The assistant turn is fully marker-wrapped:
|
||||
// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
|
||||
// <|START_THINKING|>{reasoning}<|END_THINKING|>
|
||||
// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|>
|
||||
// OR tool calls: <|START_ACTION|>[
|
||||
// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ...
|
||||
// ]<|END_ACTION|>
|
||||
// <|END_OF_TURN_TOKEN|>
|
||||
//
|
||||
// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is
|
||||
// the template default), so the model's output continues from *inside* the thinking block. The
|
||||
// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix
|
||||
// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself,
|
||||
// regardless of whether they came from the generation prompt or the generated text.
|
||||
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
const std::string TURN_START = "<|START_OF_TURN_TOKEN|>";
|
||||
const std::string TURN_END = "<|END_OF_TURN_TOKEN|>";
|
||||
const std::string CHATBOT = "<|CHATBOT_TOKEN|>";
|
||||
const std::string USER = "<|USER_TOKEN|>";
|
||||
const std::string SYSTEM = "<|SYSTEM_TOKEN|>";
|
||||
const std::string THINK_START = "<|START_THINKING|>";
|
||||
const std::string THINK_END = "<|END_THINKING|>";
|
||||
const std::string TEXT_START = "<|START_TEXT|>";
|
||||
const std::string TEXT_END = "<|END_TEXT|>";
|
||||
const std::string ACTION_START = "<|START_ACTION|>";
|
||||
const std::string ACTION_END = "<|END_ACTION|>";
|
||||
const std::string RESULT_START = "<|START_TOOL_RESULT|>";
|
||||
const std::string RESULT_END = "<|END_TOOL_RESULT|>";
|
||||
|
||||
// Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker.
|
||||
const std::string GEN_PREFIX = TURN_START + CHATBOT;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = {THINK_END};
|
||||
data.preserved_tokens = {
|
||||
TURN_START, TURN_END, CHATBOT, USER, SYSTEM,
|
||||
THINK_START, THINK_END,
|
||||
TEXT_START, TEXT_END,
|
||||
ACTION_START, ACTION_END,
|
||||
RESULT_START, RESULT_END,
|
||||
};
|
||||
|
||||
// Declare per-role message delimiters. Tool results are rendered with the
|
||||
// system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before
|
||||
// the plain "system" one (it is a strict superset, and the role split tries delimiters in order).
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX },
|
||||
{ COMMON_CHAT_ROLE_USER, TURN_START + USER },
|
||||
{ COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + TEXT_START + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PREFIX);
|
||||
auto end = p.end();
|
||||
|
||||
// The thinking block is always present (the generation prompt forces <|START_THINKING|>).
|
||||
// When extracting reasoning, capture its body; otherwise keep the whole block (markers
|
||||
// included) inline as content, matching reasoning_format=NONE conventions.
|
||||
common_peg_parser reasoning = p.eps();
|
||||
if (extract_reasoning) {
|
||||
reasoning = p.optional(p.literal(THINK_START) +
|
||||
p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) +
|
||||
p.optional(p.literal(THINK_END)));
|
||||
} else {
|
||||
reasoning = p.optional(p.content(p.literal(THINK_START) +
|
||||
p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) +
|
||||
p.optional(p.literal(THINK_END))));
|
||||
}
|
||||
|
||||
auto text_content = has_response_format
|
||||
? p.literal(TEXT_START) +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
|
||||
p.optional(p.literal(TEXT_END))
|
||||
: p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END));
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end;
|
||||
}
|
||||
|
||||
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
|
||||
// <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|>
|
||||
auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls,
|
||||
/* force_tool_calls = */ true,
|
||||
/* name_key = */ "tool_name",
|
||||
/* args_key = */ "parameters",
|
||||
/* array_wrapped = */ true,
|
||||
/* function_is_key = */ false,
|
||||
/* call_id_key = */ "",
|
||||
/* gen_call_id_key = */ "tool_call_id",
|
||||
/* parameters_order = */ { "tool_call_id", "tool_name", "parameters" });
|
||||
|
||||
// Content and tool calls are mutually exclusive in this format.
|
||||
common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content });
|
||||
|
||||
return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
#include "parsers.h"
|
||||
#include "log.h"
|
||||
|
||||
// The DeepSeek V4 reference implementation renders consecutive tool results into a single
|
||||
// user block, ordered by the tool call order of the preceding assistant message (matched
|
||||
// by tool call id) rather than by the order they appear in the conversation.
|
||||
static json deepseek_v4_sort_tool_results(const json & messages) {
|
||||
json adjusted = messages;
|
||||
std::map<std::string, size_t> call_order;
|
||||
|
||||
for (size_t i = 0; i < adjusted.size();) {
|
||||
const auto & msg = adjusted[i];
|
||||
const auto role = msg.value("role", "");
|
||||
|
||||
if (role == "assistant" && msg.contains("tool_calls") &&
|
||||
msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
|
||||
call_order.clear();
|
||||
const auto & tool_calls = msg.at("tool_calls");
|
||||
for (size_t idx = 0; idx < tool_calls.size(); idx++) {
|
||||
auto id = tool_calls[idx].value("id", "");
|
||||
if (!id.empty()) {
|
||||
call_order[id] = idx;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (role != "user" && role != "tool") {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// collect a maximal run of user/tool messages - they render into one user block
|
||||
std::vector<size_t> tool_positions;
|
||||
size_t run_end = i;
|
||||
for (; run_end < adjusted.size(); run_end++) {
|
||||
const auto r = adjusted[run_end].value("role", "");
|
||||
if (r == "tool") {
|
||||
tool_positions.push_back(run_end);
|
||||
} else if (r != "user") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (tool_positions.size() > 1 && !call_order.empty()) {
|
||||
std::vector<json> results;
|
||||
results.reserve(tool_positions.size());
|
||||
for (auto pos : tool_positions) {
|
||||
results.push_back(adjusted[pos]);
|
||||
}
|
||||
std::stable_sort(results.begin(), results.end(), [&](const json & a, const json & b) {
|
||||
const auto order = [&](const json & m) {
|
||||
auto it = call_order.find(m.value("tool_call_id", ""));
|
||||
return it == call_order.end() ? (size_t) 0 : it->second;
|
||||
};
|
||||
return order(a) < order(b);
|
||||
});
|
||||
for (size_t k = 0; k < tool_positions.size(); k++) {
|
||||
adjusted[tool_positions[k]] = std::move(results[k]);
|
||||
}
|
||||
}
|
||||
|
||||
i = run_end;
|
||||
}
|
||||
|
||||
return adjusted;
|
||||
}
|
||||
|
||||
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
// V4 uses the same DSML markup as V3.2, but names the tool call block "tool_calls"
|
||||
// instead of "function_calls", renders tool results in tool call order and its
|
||||
// non-thinking generation prompt ends with a bare </think> instead of an empty
|
||||
// <think></think> pair.
|
||||
const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
|
||||
// V4.1: DSML tag names carry a leading space, e.g. <｜DSML｜ calls>
|
||||
const bool is_v41 = is_v4 && tmpl.source().find("｜DSML｜ calls>") != std::string::npos;
|
||||
|
||||
std::optional<json> adjusted_messages;
|
||||
if (is_v4) {
|
||||
adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
|
||||
}
|
||||
|
||||
// V4.1 can declare tools on a system message instead of top-level "tools": parse calls to those too.
|
||||
json tools = inputs.tools;
|
||||
if (is_v41 && !(tools.is_array() && !tools.empty()) && inputs.messages.is_array()) {
|
||||
for (const auto & msg : inputs.messages) {
|
||||
if (msg.value("role", "") == "system" && msg.contains("tools") && msg.at("tools").is_array()) {
|
||||
if (!tools.is_array()) {
|
||||
tools = json::array();
|
||||
}
|
||||
for (const auto & tool : msg.at("tools")) {
|
||||
tools.push_back(tool);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto has_tools = tools.is_array() && !tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
std::optional<json> additional_context;
|
||||
if (is_v4 && has_response_format) {
|
||||
additional_context = json{ { "response_format", inputs.json_schema } };
|
||||
}
|
||||
// V4.1 raises on anything but low/high/max or 1-100: map minimal -> low, xhigh -> max, medium -> default
|
||||
if (is_v41 && inputs.extra_context.is_object() && inputs.extra_context.contains("reasoning_effort") &&
|
||||
inputs.extra_context.at("reasoning_effort").is_string()) {
|
||||
const std::string effort = inputs.extra_context.at("reasoning_effort").get<std::string>();
|
||||
json mapped;
|
||||
if (effort == "minimal") {
|
||||
mapped = "low";
|
||||
} else if (effort == "xhigh") {
|
||||
mapped = "max";
|
||||
} else if (effort == "medium") {
|
||||
mapped = nullptr;
|
||||
}
|
||||
if (effort == "minimal" || effort == "xhigh" || effort == "medium") {
|
||||
LOG_DBG("%s: DeepSeek V4.1 reasoning_effort '%s' -> %s\n", __func__, effort.c_str(),
|
||||
mapped.is_null() ? "unset (template default)" : mapped.get<std::string>().c_str());
|
||||
if (!additional_context.has_value()) {
|
||||
additional_context = json::object();
|
||||
}
|
||||
(*additional_context)["reasoning_effort"] = mapped;
|
||||
}
|
||||
}
|
||||
|
||||
const std::string DSML = "｜DSML｜";
|
||||
const std::string THINK_START = "<think>";
|
||||
const std::string THINK_END = "</think>";
|
||||
const std::string TAG_SP = is_v41 ? " " : "";
|
||||
const std::string TC_BLOCK = is_v41 ? "calls" : is_v4 ? "tool_calls" : "function_calls";
|
||||
const std::string FC_START = "<" + DSML + TAG_SP + TC_BLOCK + ">";
|
||||
const std::string FC_END = "</" + DSML + TAG_SP + TC_BLOCK + ">";
|
||||
const std::string INVOKE_START = "<" + DSML + TAG_SP + "invoke";
|
||||
const std::string INVOKE_END = "</" + DSML + TAG_SP + "invoke>";
|
||||
const std::string PARAM_START = "<" + DSML + TAG_SP + "parameter";
|
||||
const std::string PARAM_END = "</" + DSML + TAG_SP + "parameter>";
|
||||
const std::string GEN_PROMPT = "<｜Assistant｜>";
|
||||
const std::string TC_SEPARATOR = "\n\n";
|
||||
|
||||
// lets the server find user turns in the prompt and place context checkpoints there
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, GEN_PROMPT },
|
||||
{ COMMON_CHAT_ROLE_USER, "<｜User｜>" },
|
||||
};
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(
|
||||
tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(
|
||||
tmpl, inputs, adjusted_messages, std::nullopt, additional_context);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = {THINK_END, FC_START};
|
||||
data.preserved_tokens = {
|
||||
DSML,
|
||||
THINK_START,
|
||||
THINK_END,
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
if (is_v4 && msg.reasoning_content.empty()) {
|
||||
data.generation_prompt = GEN_PROMPT + THINK_END;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += msg.render_content();
|
||||
}
|
||||
} else {
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PROMPT);
|
||||
auto end = p.end();
|
||||
|
||||
// build tool call section first since we might need it in reasoning
|
||||
auto tool_choice = p.choice();
|
||||
if (has_tool_calls) {
|
||||
foreach_function(tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
std::vector<common_peg_parser> required_parsers;
|
||||
std::vector<common_peg_parser> optional_parsers;
|
||||
foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
|
||||
bool is_string = param.schema->may_be_string();
|
||||
|
||||
auto arg = p.tool_arg(
|
||||
p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param.name)) +
|
||||
p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) +
|
||||
(is_string ?
|
||||
p.tool_arg_string_value(p.until(PARAM_END)) :
|
||||
p.tool_arg_json_value(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema",
|
||||
doc, *param.schema))) +
|
||||
p.tool_arg_close(p.literal(PARAM_END)));
|
||||
|
||||
auto named_arg = p.rule("tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index), arg);
|
||||
if (param.required) {
|
||||
required_parsers.push_back(named_arg);
|
||||
} else {
|
||||
optional_parsers.push_back(named_arg);
|
||||
}
|
||||
});
|
||||
|
||||
common_peg_parser args_seq = p.eps();
|
||||
for (size_t i = 0; i < required_parsers.size(); i++) {
|
||||
if (i > 0) {
|
||||
args_seq = args_seq + p.space();
|
||||
}
|
||||
args_seq = args_seq + required_parsers[i];
|
||||
}
|
||||
|
||||
if (!optional_parsers.empty()) {
|
||||
common_peg_parser any_opt = p.choice();
|
||||
for (const auto & opt : optional_parsers) {
|
||||
any_opt |= opt;
|
||||
}
|
||||
args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
|
||||
}
|
||||
|
||||
common_peg_parser invoke_body = args_seq;
|
||||
auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") +
|
||||
p.tool_name(p.literal(name)) + p.literal("\">\n")) +
|
||||
invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END)));
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), func_parser);
|
||||
});
|
||||
}
|
||||
|
||||
common_peg_parser tool_calls = p.eps();
|
||||
if (inputs.parallel_tool_calls) {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice +
|
||||
p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
|
||||
} else {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
|
||||
}
|
||||
|
||||
auto reasoning = p.eps();
|
||||
auto reasoning_with_tc = p.eps();
|
||||
auto obligatory_tool_calls = tool_calls;
|
||||
bool allow_reasoning_with_tc = false;
|
||||
|
||||
if (!require_tools) {
|
||||
tool_calls = p.optional(tool_calls);
|
||||
}
|
||||
|
||||
if (extract_reasoning && inputs.enable_thinking) {
|
||||
// V4.1 renders a continuation without reasoning as "<｜Assistant｜></think>" even in thinking mode,
|
||||
// so consume the bare THINK_END too or it leaks into the content
|
||||
reasoning = is_v41 ? p.optional((THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END) | p.literal(THINK_END))
|
||||
: p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
|
||||
reasoning_with_tc = THINK_START +
|
||||
p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
|
||||
p.space() + obligatory_tool_calls;
|
||||
allow_reasoning_with_tc = true;
|
||||
} else if (extract_reasoning) {
|
||||
// Thinking disabled but reasoning extraction requested: the generation prompt
|
||||
// contains an empty <think></think> pair (V3.2) or a bare </think> (V4) that
|
||||
// must still be consumed.
|
||||
reasoning = is_v4
|
||||
? p.optional(p.literal(THINK_END))
|
||||
: p.optional(p.literal(THINK_START) + p.until(THINK_END) + p.literal(THINK_END));
|
||||
}
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format",
|
||||
p.literal("```json") + p.space() +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
|
||||
p.space() + p.literal("```"));
|
||||
return generation_prompt + reasoning + response_format + end;
|
||||
}
|
||||
|
||||
if (!has_tool_calls) {
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
|
||||
auto content_before_tools = p.negate(p.literal(THINK_START)) +
|
||||
p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) +
|
||||
p.space();
|
||||
return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end :
|
||||
generation_prompt + reasoning + content_before_tools + tool_calls + end;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && !require_tools;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Functionary v3.2 - uses recipient-based format: >>>recipient\n{content}
|
||||
common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.preserved_tokens = {
|
||||
">>>all",
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
data.generation_prompt = "<|start_header_id|>assistant<|end_header_id|>\n\n>>>all\n" + msg.render_content();
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
// Functionary v3.2 format:
|
||||
// - Normal content: >>>all\n{content}
|
||||
// - Tool calls: >>>function_name\n{json_args}
|
||||
// Generation prompt ends with ">>>" so model outputs recipient immediately
|
||||
|
||||
// Build content parser for >>>all\n{content}
|
||||
// When tools are present, content stops before the next ">>>" (tool call)
|
||||
// When no tools, content goes until end
|
||||
auto content_until_tool = p.literal("all\n") + p.content(p.until(">>>"));
|
||||
auto content_until_end = p.literal("all\n") + p.content(p.rest());
|
||||
auto generation_prompt = p.literal("<|start_header_id|>assistant<|end_header_id|>\n\n>>>");
|
||||
|
||||
// If no tools or tool_choice is NONE, just parse content
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
// When no tools, just match the prefix and capture everything after
|
||||
return generation_prompt + content_until_end + p.end();
|
||||
}
|
||||
|
||||
// Build tool call parsers for each available function
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto schema = common_chat_tool_parameters(function);
|
||||
|
||||
// Tool format: >>>function_name\n{json_args}
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(p.tool_name(p.literal(name)) + p.literal("\n")) +
|
||||
p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", schema))
|
||||
);
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_parser);
|
||||
});
|
||||
|
||||
auto content_only = content_until_end;
|
||||
auto tools_only = p.trigger_rule("tools", p.one_or_more(tool_choice));
|
||||
auto content_and_tools = content_until_tool + tools_only;
|
||||
|
||||
auto ret = p.eps();
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
if (inputs.parallel_tool_calls) {
|
||||
ret = p.choice({ content_and_tools, tools_only }) + p.end();
|
||||
} else {
|
||||
ret = p.choice({ content_until_tool + tool_choice, tools_only }) + p.end();
|
||||
}
|
||||
} else if (inputs.parallel_tool_calls) {
|
||||
ret = p.choice({ content_and_tools, content_only, tools_only }) + p.end();
|
||||
} else {
|
||||
auto content_and_tool = content_until_tool + tool_choice;
|
||||
ret = p.choice({ content_and_tool, content_only, tool_choice }) + p.end();
|
||||
}
|
||||
return generation_prompt + ret;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
// Grammar trigger for when the model starts outputting a tool call
|
||||
// (after the initial ">>>" in the generation prompt but recipient other than "all")
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, ">>>(?!all)" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
#include "parsers.h"
|
||||
|
||||
namespace workaround {
|
||||
|
||||
// Gemma4 uses a custom tool_responses field instead of role:tool messages.
|
||||
//
|
||||
// This will transform a sequence of messages:
|
||||
// assistant(tool_call+) -> tool+ -> assistant(content)
|
||||
//
|
||||
// Into a single assistant message containing a tool_responses field:
|
||||
// assistant(content + tool_call + tool_responses)
|
||||
//
|
||||
// This is necessary for the Gemma4 chat template to properly format the prompt.
|
||||
// See https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4
|
||||
struct gemma4_model_turn_builder {
|
||||
json & messages;
|
||||
size_t pos;
|
||||
json tool_calls = json::array();
|
||||
json tool_responses = json::array();
|
||||
json content;
|
||||
json reasoning_content;
|
||||
|
||||
gemma4_model_turn_builder(json & msgs, size_t pos) : messages(msgs), pos(pos) {}
|
||||
|
||||
void collect() {
|
||||
// Collect the first assistant message
|
||||
auto & msg = messages[pos];
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
// According to the prompt formatting guide, we need to preserve reasoning_content
|
||||
// between function calls. The current chat templates do not support this, but we will do it anyway.
|
||||
reasoning_content = msg.at("reasoning_content");
|
||||
}
|
||||
for (auto & tc : msg.at("tool_calls")) {
|
||||
tool_calls.push_back(tc);
|
||||
}
|
||||
pos++;
|
||||
|
||||
// Collect tool call results
|
||||
while (pos < messages.size() && messages[pos].value("role", "") == "tool") {
|
||||
collect_result(messages[pos]);
|
||||
pos++;
|
||||
}
|
||||
|
||||
// Check if the next assistant message is the final message
|
||||
if (pos < messages.size() && messages[pos].value("role", "") == "assistant") {
|
||||
auto & next = messages[pos];
|
||||
if (!has_tool_calls(next) && has_content(next)) {
|
||||
content = next.at("content");
|
||||
pos++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void collect_result(const json & curr) {
|
||||
json response;
|
||||
if (curr.contains("content")) {
|
||||
const auto & content = curr.at("content");
|
||||
if (content.is_string()) {
|
||||
// Try to parse the content as JSON; fall back to raw string
|
||||
try {
|
||||
response = json::parse(content.get<std::string>());
|
||||
} catch (...) {
|
||||
response = content;
|
||||
}
|
||||
} else {
|
||||
response = content;
|
||||
}
|
||||
}
|
||||
|
||||
std::string name;
|
||||
|
||||
// Match name with corresponding tool call
|
||||
size_t idx = tool_responses.size();
|
||||
if (idx < tool_calls.size()) {
|
||||
auto & tc = tool_calls[idx];
|
||||
if (tc.contains("function")) {
|
||||
name = tc.at("function").value("name", "");
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to the tool call id
|
||||
if (name.empty()) {
|
||||
name = curr.value("tool_call_id", "");
|
||||
}
|
||||
|
||||
tool_responses.push_back({{"name", name}, {"response", response}});
|
||||
}
|
||||
|
||||
json build() {
|
||||
collect();
|
||||
|
||||
json msg = {
|
||||
{"role", "assistant"},
|
||||
{"tool_calls", tool_calls},
|
||||
};
|
||||
if (!tool_responses.empty()) {
|
||||
msg["tool_responses"] = tool_responses;
|
||||
}
|
||||
if (!content.is_null()) {
|
||||
msg["content"] = content;
|
||||
}
|
||||
if (!reasoning_content.is_null()) {
|
||||
msg["reasoning_content"] = reasoning_content;
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
|
||||
static bool has_content(const json & msg) {
|
||||
if (!msg.contains("content") || msg.at("content").is_null()) {
|
||||
return false;
|
||||
}
|
||||
const auto & content = msg.at("content");
|
||||
if (content.is_string() && !content.get<std::string>().empty()) {
|
||||
return true;
|
||||
}
|
||||
if (content.is_array() && !content.empty()) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool has_tool_calls(const json & msg) {
|
||||
return msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty();
|
||||
}
|
||||
};
|
||||
|
||||
void convert_tool_responses_gemma4(json & messages) {
|
||||
json result = json::array();
|
||||
size_t i = 0;
|
||||
|
||||
while (i < messages.size()) {
|
||||
auto & msg = messages[i];
|
||||
|
||||
if (msg.value("role", "") != "assistant" || !msg.contains("tool_calls") ||
|
||||
!msg.at("tool_calls").is_array() || msg.at("tool_calls").empty()) {
|
||||
result.push_back(msg);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
gemma4_model_turn_builder builder(messages, i);
|
||||
result.push_back(builder.build());
|
||||
i = builder.pos;
|
||||
}
|
||||
|
||||
messages = result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
|
||||
if (inputs.add_generation_prompt && string_ends_with(data.prompt.str(), "<turn|>\n")) {
|
||||
// This may happen if the model generates content + tool_call, the
|
||||
// template does not add the model's next turn and confuses the model
|
||||
// from emitting its proper reasoning token sequence.
|
||||
data.generation_prompt = "<|turn>model\n";
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_USER, "<|turn>user" },
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" },
|
||||
};
|
||||
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "<|channel>thought";
|
||||
data.thinking_end_tags = {"<channel|>"};
|
||||
|
||||
data.preserved_tokens = {
|
||||
"<|channel>",
|
||||
"<channel|>",
|
||||
"<|tool_call>",
|
||||
"<tool_call|>",
|
||||
"<|turn>",
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = string_ends_with(data.prompt.str(), "<turn|>\n") ? "<|turn>model\n" : "";
|
||||
data.generation_prompt += "<|channel>thought\n" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<channel|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));
|
||||
} else {
|
||||
p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>")));
|
||||
}
|
||||
|
||||
auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");
|
||||
auto thought = (p.peek(p.literal("<|channel>")) + consume_empty_channels + p.ref("thought")) | p.negate(p.literal("<|channel>"));
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.literal("```json") <<
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) <<
|
||||
p.literal("```");
|
||||
return start + p.optional(thought) + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
// Gemma4 tool calling syntax
|
||||
// Rules should match traversal logic in gemma4_to_json()
|
||||
p.rule("gemma4-string-content", p.until("<|\"|>"));
|
||||
p.rule("gemma4-string", p.literal("<|\"|>") + p.ref("gemma4-string-content") + p.literal("<|\"|>"));
|
||||
p.rule("gemma4-bool", p.json_bool());
|
||||
p.rule("gemma4-null", p.json_null());
|
||||
p.rule("gemma4-number", p.json_number());
|
||||
p.rule("gemma4-dict-key", p.rule("gemma4-dict-key-name", p.chars("[^:}]", 1, -1)) + p.literal(":"));
|
||||
p.rule("gemma4-dict-kv", p.ref("gemma4-dict-key") + p.space() + p.ref("gemma4-value"));
|
||||
p.rule("gemma4-dict", [&]() {
|
||||
auto ws = p.space();
|
||||
auto member = p.ref("gemma4-dict-kv");
|
||||
auto members = p.sequence({member, p.zero_or_more(p.sequence({p.literal(","), ws, member}))});
|
||||
return p.sequence({
|
||||
p.literal("{"), ws,
|
||||
p.choice({p.literal("}"), p.sequence({members, ws, p.literal("}")})})
|
||||
});
|
||||
});
|
||||
p.rule("gemma4-array", [&]() {
|
||||
auto ws = p.space();
|
||||
auto value = p.ref("gemma4-value");
|
||||
auto elements = p.sequence({value, p.zero_or_more(p.sequence({p.literal(","), ws, value}))});
|
||||
return p.sequence({
|
||||
p.literal("["), ws,
|
||||
p.choice({p.literal("]"), p.sequence({elements, ws, p.literal("]")})})
|
||||
});
|
||||
});
|
||||
p.rule("gemma4-value", [&]() {
|
||||
return p.choice({
|
||||
p.ref("gemma4-string"), p.ref("gemma4-dict"), p.ref("gemma4-array"),
|
||||
p.ref("gemma4-number"), p.ref("gemma4-bool"), p.ref("gemma4-null")
|
||||
});
|
||||
});
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
// TODO @aldehir : need to extend json-schema-to-grammar to produce more than JSON rules
|
||||
// const auto & params = function.at("parameters");
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.sequence({
|
||||
p.tool_open(p.tool_name(p.literal(name)) + p.peek(p.literal("{"))),
|
||||
p.tool_args(p.ref("gemma4-dict")),
|
||||
})));
|
||||
});
|
||||
|
||||
auto tool_call = p.trigger_rule("tool-call", p.repeat(
|
||||
"<|tool_call>call:" + tool_choice + "<tool_call|>",
|
||||
/* min = */ inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0,
|
||||
/* max = */ inputs.parallel_tool_calls ? -1 : 1
|
||||
));
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return start + thought + tool_call;
|
||||
}
|
||||
|
||||
auto scan_to_toolcall = p.rule("scan-to-toolcall", p.until("<|tool_call>"));
|
||||
auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>", "<|tool_call>"})));
|
||||
auto message = p.rule("message", thought + content);
|
||||
return start + p.zero_or_more(message) + scan_to_toolcall + tool_call;
|
||||
}
|
||||
|
||||
// Gemma 4 may emit an extra <|channel>thought\n<channel|> at the end of the content. It may
|
||||
// also emit a single trailing <channel|> token. Consume all complete reasoning blocks and
|
||||
// then stop at the first unmatched <channel|> token.
|
||||
auto content = p.rule("content", p.content(p.until_one_of({"<|channel>", "<channel|>"})));
|
||||
auto message = p.rule("message", thought + content);
|
||||
return start + p.one_or_more(message);
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call>" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_gigachat_v3(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = false;
|
||||
data.preserved_tokens = {
|
||||
"<|message_sep|>\n\n",
|
||||
"<|role_sep|>\n",
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
data.generation_prompt = "assistant<|role_sep|>\n" + msg.render_content();
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n";
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto ret = p.eps();
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
// Build a choice of all available tools
|
||||
auto tool_choice = p.choice();
|
||||
for (size_t i = 0; i < inputs.tools.size(); i++) {
|
||||
const auto & tool = inputs.tools[i];
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto schema = common_chat_tool_parameters(function);
|
||||
|
||||
auto tool_name = p.json_member("name", "\"" + p.tool_name(p.literal(name)) + "\"");
|
||||
auto tool_args = p.json_member("arguments", p.tool_args(p.schema(p.json(), "tool-" + std::to_string(i) + "-schema", schema)));
|
||||
|
||||
auto tool_open = p.tool_open(p.literal("{") << tool_name);
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(i), tool_open << "," << tool_args << "}");
|
||||
}
|
||||
|
||||
// Define the tool call structure
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
auto max_calls = 1; // parallel toolcalls are not supported
|
||||
auto tool_call = p.rule("tool-call", p.literal(tool_call_start_prefix) + tool_choice);
|
||||
auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(tool_call, /* min = */ min_calls, /* max = */ max_calls));
|
||||
|
||||
ret = p.content(p.until("<|message_sep|>\n\n")) << tool_calls;
|
||||
} else {
|
||||
// Content only parser
|
||||
include_grammar = false;
|
||||
ret = p.content(p.rest());
|
||||
}
|
||||
|
||||
return p.literal("assistant<|role_sep|>\n") + ret;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{COMMON_GRAMMAR_TRIGGER_TYPE_WORD, tool_call_start_prefix}
|
||||
};
|
||||
}
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
// Copy reasoning to the "thinking" field as expected by the gpt-oss template
|
||||
auto adjusted_messages = json::array();
|
||||
for (auto msg : inputs.messages) {
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
msg["thinking"] = msg.at("reasoning_content");
|
||||
if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
|
||||
msg.erase("content");
|
||||
}
|
||||
}
|
||||
adjusted_messages.push_back(msg);
|
||||
}
|
||||
|
||||
auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);
|
||||
|
||||
// Check if we need to replace the return token with end token during
|
||||
// inference and without generation prompt. For more details see:
|
||||
// https://github.com/ggml-org/llama.cpp/issues/15417
|
||||
if (inputs.is_inference && !inputs.add_generation_prompt) {
|
||||
static constexpr std::string_view return_token = "<|return|>";
|
||||
static constexpr std::string_view end_token = "<|end|>";
|
||||
prompt.replace_last(std::string(return_token), std::string(end_token));
|
||||
}
|
||||
|
||||
data.prompt = std::move(prompt);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|start|>functions" },
|
||||
};
|
||||
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
data.thinking_start_tag = "<|channel|>analysis<|message|>";
|
||||
data.thinking_end_tags = {"<|end|>"};
|
||||
|
||||
// These special tokens are required to parse properly
|
||||
data.preserved_tokens = {
|
||||
"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
|
||||
};
|
||||
|
||||
// Adjust prompt for continuation
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
auto end = p.rule("end", p.literal("<|end|>"));
|
||||
auto content = p.rule("message-content", p.until("<|end|>"));
|
||||
auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis"));
|
||||
auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);
|
||||
|
||||
// Occasionally, gpt-oss-20b will prefix channels with this commentary
|
||||
auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant")));
|
||||
auto start_analysis = stray_commentary + p.literal("<|channel|>analysis<|message|>");
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", start_analysis + p.reasoning(content) + end);
|
||||
} else {
|
||||
p.rule("analysis", p.content(start_analysis + content + end));
|
||||
}
|
||||
|
||||
auto analysis = p.ref("analysis");
|
||||
auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end);
|
||||
auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content));
|
||||
|
||||
// Consume any unsolicited tool calls, e.g. builtin functions
|
||||
auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end));
|
||||
|
||||
auto any = p.rule("any", preamble | analysis);
|
||||
|
||||
if (has_response_format) {
|
||||
auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
|
||||
auto response_format = p.rule("response-format",
|
||||
p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto tool_choice = p.choice();
|
||||
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto params = common_chat_tool_parameters(function);
|
||||
|
||||
auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name));
|
||||
auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
|
||||
auto args = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", params));
|
||||
|
||||
// recipient in role header
|
||||
// <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS
|
||||
auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + p.literal("<|message|>")) + args);
|
||||
|
||||
// recipient in channel header
|
||||
// <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS
|
||||
auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + p.literal("<|message|>")) + args);
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_in_role | tool_in_channel);
|
||||
});
|
||||
|
||||
auto tool_call = p.trigger_rule("tool-call", tool_choice);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return p.zero_or_more(start + any) + start + tool_call;
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + any) + start + (tool_call | final_msg);
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + any) + start + (final_msg | unsolicited);
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>(?:commentary|analysis)\\s+to=functions$" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(<\\|channel\\|>(?:commentary|analysis)\\s+to)" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Harmony with XML tool calls: <|start|>assistant to=NAME<|channel|>analysis <|constrain|>xml<|message|><rfl:parameter name="KEY">\nVALUE\n</rfl:parameter>
|
||||
common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },
|
||||
};
|
||||
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
data.thinking_start_tag = "<|channel|>analysis<|message|>";
|
||||
data.thinking_end_tags = {"<|end|>"};
|
||||
|
||||
// EOG text must reach the parser: an XML call has no closing tag of its own
|
||||
static const std::vector<std::string> EOG = {
|
||||
"<|call|>", "<|return|>", "<|endoftext|>", "<|im_end|>", "<|fim_pad|>", "<|repo_name|>", "<|file_sep|>",
|
||||
};
|
||||
|
||||
data.preserved_tokens = {
|
||||
"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
|
||||
};
|
||||
data.preserved_tokens.insert(data.preserved_tokens.end(), EOG.begin(), EOG.end());
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
static const std::string PARAM_OPEN = "<rfl:parameter name=\"";
|
||||
static const std::string PARAM_CLOSE = "\n</rfl:parameter>";
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
|
||||
auto eog = p.choice();
|
||||
for (const auto & tok : EOG) {
|
||||
eog |= p.literal(tok);
|
||||
}
|
||||
auto with_eog = [&](std::vector<std::string> delims) {
|
||||
delims.insert(delims.end(), EOG.begin(), EOG.end());
|
||||
return delims;
|
||||
};
|
||||
|
||||
auto analysis_open = p.literal("<|channel|>analysis<|message|>");
|
||||
auto analysis_body = p.until_one_of(with_eog({ "<|end|>", "<|constrain|><|start|>assistant", "<|start|>assistant" }));
|
||||
auto analysis_close = p.optional(p.literal("<|end|>") | p.literal("<|constrain|>") | eog);
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", analysis_open + p.reasoning(analysis_body) + analysis_close);
|
||||
} else {
|
||||
p.rule("analysis", p.atomic(p.content(analysis_open)) + p.content(analysis_body + analysis_close));
|
||||
}
|
||||
auto analysis = p.ref("analysis");
|
||||
|
||||
auto final_msg = p.rule("final",
|
||||
p.literal("<|channel|>final<|message|>") +
|
||||
p.content(p.until_one_of(with_eog({ "<|end|>" }))) +
|
||||
p.optional(p.literal("<|end|>") | eog));
|
||||
|
||||
if (has_response_format) {
|
||||
auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);
|
||||
auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type);
|
||||
auto response_format = p.rule("response-format",
|
||||
p.literal("<|channel|>final") + constraint + p.literal("<|message|>") +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
|
||||
p.optional(p.literal("<|end|>") | eog));
|
||||
|
||||
return start + p.zero_or_more(analysis + start) + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
// tolerate a missing closing tag
|
||||
std::vector<std::string> value_ends = with_eog({ PARAM_OPEN, "<|end|>" });
|
||||
std::vector<std::string> string_ends = { PARAM_CLOSE };
|
||||
auto next_open = p.choice();
|
||||
for (const auto & e : value_ends) {
|
||||
string_ends.push_back("\n" + e);
|
||||
next_open |= p.literal(e);
|
||||
}
|
||||
auto arg_close = p.tool_arg_close(p.gbnf(
|
||||
p.literal(PARAM_CLOSE) | (p.literal("\n") + p.peek(p.optional(p.literal("\n")) + next_open)),
|
||||
gbnf_format_literal("\n") + " " + gbnf_format_literal(PARAM_CLOSE.substr(1)) + "?"));
|
||||
auto arg_string = p.rule("rfl-arg-string",
|
||||
p.ac(p.tool_arg_string_value(p.until_one_of(string_ends)) + arg_close, PARAM_CLOSE));
|
||||
auto arg_sep = p.optional(p.literal("\n"));
|
||||
|
||||
std::string call_close_gbnf = gbnf_format_literal("<|end|>");
|
||||
for (const auto & tok : EOG) {
|
||||
call_close_gbnf += " | " + gbnf_format_literal(tok);
|
||||
}
|
||||
auto call_close = p.gbnf(p.literal("<|end|>") | eog | p.peek(p.literal("<|start|>assistant")),
|
||||
"(" + call_close_gbnf + ")?");
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
std::vector<common_peg_parser> arg_rules;
|
||||
foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
|
||||
auto rule_name = "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index);
|
||||
auto arg_open = p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(p.literal(param.name)) + p.literal("\">\n"));
|
||||
|
||||
auto types = param.schema->value_types();
|
||||
auto arg_value = p.eps();
|
||||
if (!types.has(common_chat_schema::TYPE_STRING) &&
|
||||
(types.has(common_chat_schema::TYPE_OBJECT) || types.has(common_chat_schema::TYPE_ARRAY))) {
|
||||
// the schema grammar fixes the key order, and the model writes keys in its own order
|
||||
arg_value = p.tool_arg_json_value(p.json()) + arg_close;
|
||||
} else if (!types.has(common_chat_schema::TYPE_STRING)) {
|
||||
arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
|
||||
} else if (types.is_only(common_chat_schema::TYPE_STRING)) {
|
||||
arg_value = arg_string;
|
||||
} else {
|
||||
// grammar keeps only the string rule; the parser tries JSON first to type the value
|
||||
auto json_value = p.choice();
|
||||
if (types.has(common_chat_schema::TYPE_OBJECT)) {
|
||||
json_value |= p.json_object();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_ARRAY)) {
|
||||
json_value |= p.json_array();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) {
|
||||
json_value |= p.json_number();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_BOOLEAN)) {
|
||||
json_value |= p.json_bool();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_NULL)) {
|
||||
json_value |= p.json_null();
|
||||
}
|
||||
arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "rfl-arg-string");
|
||||
}
|
||||
|
||||
arg_rules.push_back(p.rule(rule_name, p.tool_arg(arg_open + arg_value)));
|
||||
});
|
||||
|
||||
if (arg_rules.empty()) {
|
||||
auto any_name = p.chars("[^\"\\n<]", 1, -1);
|
||||
arg_rules.push_back(p.rule("tool-" + std::to_string(tool_index) + "-arg-any",
|
||||
p.tool_arg(p.tool_arg_open(p.literal(PARAM_OPEN) + p.tool_arg_name(any_name) + p.literal("\">\n")) + arg_string)));
|
||||
}
|
||||
|
||||
auto args = p.zero_or_more(p.choice(arg_rules) + arg_sep);
|
||||
|
||||
auto call = p.tool(
|
||||
p.tool_open(p.literal(" to=") + p.tool_name(p.literal(name)) +
|
||||
p.literal("<|channel|>analysis <|constrain|>xml<|message|>")) +
|
||||
p.tool_args(args) +
|
||||
p.tool_close(call_close));
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), call);
|
||||
});
|
||||
|
||||
auto tool_calls = inputs.parallel_tool_calls
|
||||
? tool_choice + p.zero_or_more(start + tool_choice)
|
||||
: tool_choice;
|
||||
auto tool_call = p.trigger_rule("tool-call", tool_calls);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return start + p.zero_or_more(analysis + start) + tool_call;
|
||||
}
|
||||
|
||||
return start + p.zero_or_more(analysis + start) + (tool_call | final_msg | p.content(p.rest()));
|
||||
}
|
||||
|
||||
auto unsolicited = p.rule("unsolicited", p.atomic(p.literal(" to=") + p.rest()));
|
||||
|
||||
// output without a channel header is returned as content instead of failing the request
|
||||
return start + p.zero_or_more(analysis + start) + (final_msg | unsolicited | p.content(p.rest()));
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>assistant(\\s+to)" },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// K2 Horizon format:
|
||||
// - Reasoning: <ifm|think>...</ifm|think>, or <ifm|think_fast>/<ifm|think_faster> for medium/low reasoning_effort
|
||||
// - Tool calls: <ifm|tool_calls><ifm|tool_call>...</ifm|tool_call>...</ifm|tool_calls>, one call per <ifm|tool_call>:
|
||||
// xml (default): name <ifm|arg_key>k</ifm|arg_key> [<ifm|arg_type>t</ifm|arg_type>] <ifm|arg_value>v</ifm|arg_value> ...
|
||||
// json: {"name": "...", "arguments": {...}}
|
||||
common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
// The template requires a thinking field on every assistant message
|
||||
auto messages = inputs.messages;
|
||||
for (auto & msg : messages) {
|
||||
if (msg.value("role", "") == "assistant" && !msg.contains("reasoning_content")) {
|
||||
msg["reasoning_content"] = "";
|
||||
}
|
||||
}
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, messages);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, messages);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
const std::string effort = inputs.extra_context.value("reasoning_effort", "high");
|
||||
const std::string call_format = inputs.extra_context.value("tool_call_format", "xml");
|
||||
|
||||
// Templates that handle enable_thinking disable it with an empty <ifm|think></ifm|think> block for every effort
|
||||
const bool thinking_off = !inputs.enable_thinking && tmpl.source().find("enable_thinking") != std::string::npos;
|
||||
const std::string think = thinking_off ? "ifm|think" :
|
||||
effort == "medium" ? "ifm|think_fast" :
|
||||
effort == "low" ? "ifm|think_faster" : "ifm|think";
|
||||
|
||||
const std::string GEN_PREFIX = "<|ifm|im_start|>assistant\n";
|
||||
const std::string THINK_START = "<" + think + ">";
|
||||
const std::string THINK_END = "</" + think + ">";
|
||||
const std::string SECTION_START = "<ifm|tool_calls>";
|
||||
const std::string SECTION_END = "</ifm|tool_calls>";
|
||||
const std::string CALL_START = "<ifm|tool_call>";
|
||||
const std::string CALL_END = "</ifm|tool_call>";
|
||||
const std::string ARG_KEY = "<ifm|arg_key>";
|
||||
const std::string ARG_KEY_END = "</ifm|arg_key>";
|
||||
const std::string ARG_TYPE = "<ifm|arg_type>";
|
||||
const std::string ARG_TYPE_END = "</ifm|arg_type>";
|
||||
const std::string ARG_VAL = "<ifm|arg_value>";
|
||||
const std::string ARG_VAL_END = "</ifm|arg_value>";
|
||||
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = { THINK_END };
|
||||
|
||||
data.preserved_tokens = data.thinking_end_tags;
|
||||
data.preserved_tokens.insert(data.preserved_tokens.end(), {
|
||||
THINK_START, SECTION_START, SECTION_END, CALL_START, CALL_END,
|
||||
ARG_KEY, ARG_KEY_END, ARG_TYPE, ARG_TYPE_END, ARG_VAL, ARG_VAL_END,
|
||||
});
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|ifm|im_start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|ifm|im_start|>user" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|ifm|im_start|>tool" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|ifm|im_start|>system" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PREFIX + THINK_START + "\n" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PREFIX);
|
||||
|
||||
auto think_end = p.choice();
|
||||
for (const auto & tag : data.thinking_end_tags) {
|
||||
think_end |= p.literal(tag);
|
||||
}
|
||||
auto think_body = p.until_one_of(data.thinking_end_tags);
|
||||
auto think_block = [&](const common_peg_parser & body) {
|
||||
return p.optional(THINK_START + p.space() + p.ac(body + think_end, data.thinking_end_tags));
|
||||
};
|
||||
auto reasoning = extract_reasoning ? think_block(p.reasoning(think_body)) : p.eps();
|
||||
|
||||
if (has_response_format) {
|
||||
// The answer must be bare JSON, so the think block is consumed even when it is not extracted
|
||||
auto thoughts = extract_reasoning ? reasoning : think_block(think_body);
|
||||
return generation_prompt + (thoughts << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
|
||||
}
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return generation_prompt + (reasoning << p.content(p.rest()));
|
||||
}
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
if (call_format == "json") {
|
||||
tool_choice = p.standard_json_tools(CALL_START, CALL_END, inputs.tools, false, true);
|
||||
} else {
|
||||
auto arg_close = p.tool_arg_close(p.literal(ARG_VAL_END));
|
||||
auto arg_string = p.rule("xml-arg-string", p.ac(p.tool_arg_string_value(p.until(ARG_VAL_END)) + arg_close, ARG_VAL_END));
|
||||
|
||||
// The models leave out <ifm|arg_type> even when asked for xml_typed
|
||||
auto arg_type = call_format == "xml_typed" ? p.optional(ARG_TYPE + p.until(ARG_TYPE_END) + ARG_TYPE_END + p.space()) : p.eps();
|
||||
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
std::vector<common_peg_parser> required_args;
|
||||
std::vector<common_peg_parser> optional_args;
|
||||
foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
|
||||
auto rule_name = "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index);
|
||||
auto types = param.schema->value_types();
|
||||
auto arg_value = arg_string;
|
||||
if (!types.has(common_chat_schema::TYPE_STRING)) {
|
||||
arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_STRING) && !types.is_only(common_chat_schema::TYPE_STRING)) {
|
||||
// The string alternative accepts any text, so only the parser needs the JSON alternatives.
|
||||
auto json_value = p.choice();
|
||||
if (types.has(common_chat_schema::TYPE_OBJECT)) {
|
||||
json_value |= p.json_object();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_ARRAY)) {
|
||||
json_value |= p.json_array();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) {
|
||||
json_value |= p.json_number();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_BOOLEAN)) {
|
||||
json_value |= p.json_bool();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_NULL)) {
|
||||
json_value |= p.json_null();
|
||||
}
|
||||
arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "xml-arg-string");
|
||||
}
|
||||
|
||||
auto arg = p.space() + p.tool_arg(p.tool_arg_open(ARG_KEY + p.tool_arg_name(p.literal(param.name)) + ARG_KEY_END) <<
|
||||
arg_type + ARG_VAL + arg_value);
|
||||
(param.required ? required_args : optional_args).push_back(p.rule(rule_name, arg));
|
||||
});
|
||||
|
||||
auto args = p.permute("tool-" + std::to_string(tool_index) + "-args", required_args);
|
||||
if (!optional_args.empty()) {
|
||||
args = args + p.zero_or_more(p.choice(optional_args));
|
||||
}
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(
|
||||
p.tool_open(CALL_START + p.tool_name(p.literal(name)) + "\n") + p.tool_args(args) << p.tool_close(p.literal(CALL_END))));
|
||||
});
|
||||
}
|
||||
|
||||
auto required = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
auto calls = inputs.parallel_tool_calls ? tool_choice + p.zero_or_more(p.space() + tool_choice) : tool_choice;
|
||||
auto tool_calls = p.trigger_rule("tool-calls", p.repeat(SECTION_START << calls << SECTION_END, required ? 1 : 0, 1));
|
||||
|
||||
// Keep thinking inline when required calls bypass the content parser.
|
||||
if (required && !extract_reasoning) {
|
||||
reasoning = p.content(think_block(think_body));
|
||||
}
|
||||
|
||||
// A required call follows the reasoning directly, the models otherwise keep writing content
|
||||
auto content = required ? p.eps() : p.content(p.until(SECTION_START));
|
||||
|
||||
return generation_prompt + (reasoning << content << tool_calls);
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED);
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, SECTION_START },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>
|
||||
// The ID contains both the function name and an incrementing counter
|
||||
common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.preserved_tokens = {
|
||||
"<|tool_calls_section_begin|>",
|
||||
"<|tool_calls_section_end|>",
|
||||
"<|tool_call_begin|>",
|
||||
"<|tool_call_argument_begin|>",
|
||||
"<|tool_call_end|>",
|
||||
"<think>",
|
||||
"</think>",
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
const std::string SECTION_BEGIN = "<|tool_calls_section_begin|>";
|
||||
const std::string SECTION_END = "<|tool_calls_section_end|>";
|
||||
const std::string CALL_BEGIN = "<|tool_call_begin|>";
|
||||
const std::string ARGS_BEGIN = "<|tool_call_argument_begin|>";
|
||||
const std::string CALL_END = "<|tool_call_end|>";
|
||||
|
||||
const std::string THINK_START = "<think>";
|
||||
const std::string THINK_END = "</think>";
|
||||
const std::string GEN_PROMPT = "<|im_assistant|>assistant<|im_middle|>";
|
||||
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = {THINK_END};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
// Kimi K2 Thinking format:
|
||||
// - Reasoning: <think>{reasoning}</think>
|
||||
// - Content: text after reasoning
|
||||
// - Tool calls section:
|
||||
// <|tool_calls_section_begin|>
|
||||
// <|tool_call_begin|>functions.<name>:<index><|tool_call_argument_begin|>{json_args}<|tool_call_end|>
|
||||
// ...
|
||||
// <|tool_calls_section_end|>
|
||||
// The ID format is: functions.<function_name>:<counter> where counter is 0, 1, 2, ...
|
||||
|
||||
// Tool call markers
|
||||
auto end = p.end();
|
||||
|
||||
// Note: this model is CRAZY. It can diverge from its supposed tool calling pattern in so many ways it's not funny.
|
||||
// For example, it can call tools at the end of reasoning without closing reasoning...
|
||||
auto reasoning = extract_reasoning ? p.optional(THINK_START + p.reasoning(
|
||||
p.until_one_of({ THINK_END, "<|tool_calls_section_begin|>", "<|tool_call_begin|>" })) +
|
||||
p.optional(p.literal(THINK_END))) : p.eps();
|
||||
auto generation_prompt = p.literal(GEN_PROMPT);
|
||||
|
||||
|
||||
// Content only parser (no tools)
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
|
||||
// Build tool call parsers for each available function
|
||||
// The ID format is: functions.<name>:<index>
|
||||
// We need to match: functions.<name>:<digits>
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto schema = common_chat_tool_parameters(function);
|
||||
|
||||
// Match: functions.<name>:<digits>
|
||||
// Capture the full call id (functions.<name>:<digits>) using tool_id tag
|
||||
auto tool_id = p.tool_id(p.literal("functions.") + p.tool_name(p.literal(name)) + p.literal(":") + p.chars("[0-9]", 1, -1));
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(tool_id + p.literal(ARGS_BEGIN)) +
|
||||
p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", schema)) +
|
||||
p.tool_close(p.optional((p.literal(CALL_END))))
|
||||
);
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_parser);
|
||||
});
|
||||
|
||||
// Tool calls section: <|tool_calls_section_begin|> tool_calls <|tool_calls_section_end|>
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
auto max_calls = inputs.parallel_tool_calls ? -1 : 1;
|
||||
// Use trigger_rule so grammar generator knows where to start generating rules
|
||||
auto tool_calls = p.rule("tool-calls",
|
||||
p.optional(p.literal(SECTION_BEGIN)) +
|
||||
p.trigger_rule("tool-call", p.repeat(CALL_BEGIN + tool_choice, min_calls, max_calls) +
|
||||
p.optional(p.literal(SECTION_END)))
|
||||
);
|
||||
|
||||
auto content_before_tools = p.content(p.until_one_of({ SECTION_BEGIN, CALL_BEGIN }));
|
||||
|
||||
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<|tool_call_begin|>" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Kimi K3 - XTML tagged format, built by open_tag/close_tag macros:
|
||||
// open_tag(t, attrs) = <|open|>t k="v"...<|sep|> close_tag(t) = <|close|>t<|sep|>
|
||||
// assistant := [think] [response] [tools] close_tag(message) <|end_of_msg|>
|
||||
// the generation prompt already opens the think (or response) section, so the
|
||||
// section opener is optional here - same as Kimi K2 Thinking
|
||||
common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
const std::string SEP = "<|sep|>";
|
||||
const std::string MSG_START = "<|open|>message role=\"assistant\"<|sep|>";
|
||||
const std::string THINK_START = "<|open|>think<|sep|>";
|
||||
const std::string THINK_END = "<|close|>think<|sep|>";
|
||||
const std::string RESP_START = "<|open|>response<|sep|>";
|
||||
const std::string RESP_END = "<|close|>response<|sep|>";
|
||||
const std::string TOOLS_START = "<|open|>tools<|sep|>";
|
||||
const std::string TOOLS_END = "<|close|>tools<|sep|>";
|
||||
const std::string CALL_START = "<|open|>call tool=\"";
|
||||
const std::string CALL_END = "<|close|>call<|sep|>";
|
||||
const std::string ARG_START = "<|open|>argument key=\"";
|
||||
const std::string ARG_END = "<|close|>argument<|sep|>";
|
||||
const std::string MSG_END = "<|close|>message<|sep|>";
|
||||
const std::string EOM_TOKEN = "<|end_of_msg|>";
|
||||
|
||||
// only the markers are special tokens. tag names ("think", "response", ...) are
|
||||
// normal tokens and must not be preserved, or prose with those words is broken
|
||||
data.preserved_tokens = {
|
||||
"<|open|>",
|
||||
"<|close|>",
|
||||
"<|sep|>",
|
||||
"<|end_of_msg|>",
|
||||
};
|
||||
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = { THINK_END };
|
||||
|
||||
// per-role message-start delimiters. user/assistant messages only have the role
|
||||
// attribute, so the full opener is used. system and tool messages have more
|
||||
// attributes, so those delimiters stop after the closing quote of the role
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|open|>message role=\"assistant\"<|sep|>" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|open|>message role=\"user\"<|sep|>" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|open|>message role=\"tool\"" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|open|>message role=\"system\"" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = MSG_START + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + RESP_START + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto end = p.end();
|
||||
|
||||
auto start = p.optional(p.literal(MSG_START));
|
||||
|
||||
// the think section is always consumed, even with reasoning extraction off:
|
||||
// the generation prompt ends with open_tag('think'), so it is always present.
|
||||
// reasoning stops at its own closer, or at the response opener if the model
|
||||
// skips the closer
|
||||
auto think_body = extract_reasoning ? p.reasoning(p.until_one_of({ THINK_END, RESP_START })) :
|
||||
p.content(p.until_one_of({ THINK_END, RESP_START }));
|
||||
|
||||
auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body +
|
||||
p.optional(p.literal(THINK_END)));
|
||||
|
||||
// content runs to the response closer, or to the next section if truncated
|
||||
auto response = p.optional(p.literal(RESP_START)) +
|
||||
p.content(p.until_one_of({ RESP_END, TOOLS_START, MSG_END })) +
|
||||
p.optional(p.literal(RESP_END));
|
||||
|
||||
// the EOG token after the message closer reaches the parser as text,
|
||||
// so it must be consumed or the parse stays incomplete
|
||||
auto trailer = p.optional(p.literal(MSG_END)) + p.optional(p.literal(EOM_TOKEN));
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return start + reasoning + response + trailer + end;
|
||||
}
|
||||
|
||||
auto tool_choices = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const json schema = common_chat_tool_parameters(function);
|
||||
|
||||
// arguments come one tag per key, with the JSON type in a type="..."
|
||||
// attribute. the type is taken from the tool schema instead, as it tells
|
||||
// us if the value is JSON or a literal string
|
||||
auto args = p.eps();
|
||||
if (schema.contains("properties") && !schema.at("properties").empty()) {
|
||||
auto arg_choices = p.choice();
|
||||
size_t param_index = 0;
|
||||
for (const auto & prop : schema.at("properties").items()) {
|
||||
const std::string & key = prop.key();
|
||||
|
||||
std::string type = "string";
|
||||
if (prop.value().is_object() && prop.value().contains("type") &&
|
||||
prop.value().at("type").is_string()) {
|
||||
type = prop.value().at("type").get<std::string>();
|
||||
}
|
||||
|
||||
auto value = type == "string" ? p.tool_arg_string_value(p.until(ARG_END)) :
|
||||
p.tool_arg_value(p.until(ARG_END));
|
||||
|
||||
// skip the trailing type="..." attribute: anything up to <|sep|>
|
||||
arg_choices |= p.rule("kimi-k3-arg-" + std::to_string(tool_index) + "-" + std::to_string(param_index++),
|
||||
p.tool_arg(p.tool_arg_open(p.literal(ARG_START)) +
|
||||
p.tool_arg_name(p.literal(key)) + p.literal("\"") +
|
||||
p.until(SEP) + p.literal(SEP) + value +
|
||||
p.tool_arg_close(p.literal(ARG_END))));
|
||||
}
|
||||
args = p.zero_or_more(arg_choices);
|
||||
}
|
||||
|
||||
// skip the trailing index="N" attribute the same way
|
||||
auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) + p.literal("\"") +
|
||||
p.until(SEP) + p.literal(SEP)) +
|
||||
p.tool_args(args) + p.tool_close(p.literal(CALL_END)));
|
||||
|
||||
tool_choices |= p.rule("kimi-k3-tool-" + std::to_string(tool_index), call);
|
||||
});
|
||||
|
||||
// all calls go inside one tools section, then the message is closed. the
|
||||
// message closer is part of the trigger rule, or else the lazy grammar
|
||||
// rejects it once tool calls have started
|
||||
auto tools_section =
|
||||
p.trigger_rule("kimi-k3-tool-call", p.literal(TOOLS_START) + p.one_or_more(tool_choices) +
|
||||
p.literal(TOOLS_END) + p.optional(p.literal(MSG_END)) +
|
||||
p.optional(p.literal(EOM_TOKEN)));
|
||||
|
||||
auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section :
|
||||
p.optional(tools_section);
|
||||
|
||||
return start + reasoning + response + tools + trailer + end;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOLS_START },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// LFM2 format detection: template uses <|tool_list_start|>[...]<|tool_list_end|> around the tool list
|
||||
// and <|tool_call_start|>[...]<|tool_call_end|> around each tool call
|
||||
bool is_lfm2_template(const std::string & src) {
|
||||
return src.find("<|tool_list_start|>") != std::string::npos &&
|
||||
src.find("<|tool_list_end|>") != std::string::npos;
|
||||
}
|
||||
|
||||
// LFM2/LFM2.5 parser. Tool calls are almost Python-style and parallel-capable
|
||||
// (except dotted names and JSON literals true/false/null).
|
||||
// Always wrapped in <|tool_call_start|>[name(args)]<|tool_call_end|> with optional <think> reasoning.
|
||||
// tool_list_tokens preserves LFM2 system tool-list markers.
|
||||
common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs,
|
||||
bool tool_list_tokens) {
|
||||
common_chat_params data;
|
||||
|
||||
const std::string TOOL_CALL_START = "<|tool_call_start|>";
|
||||
const std::string TOOL_CALL_END = "<|tool_call_end|>";
|
||||
const std::string TOOL_LIST_START = "<|tool_list_start|>";
|
||||
const std::string TOOL_LIST_END = "<|tool_list_end|>";
|
||||
const std::string THINK_START = "<think>";
|
||||
const std::string THINK_END = "</think>";
|
||||
const std::string GEN_PROMPT = "<|im_start|>assistant\n";
|
||||
|
||||
// Copy reasoning to the "thinking" field the template expects
|
||||
auto adjusted_messages = json::array();
|
||||
for (auto msg : inputs.messages) {
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
msg["thinking"] = msg.at("reasoning_content");
|
||||
}
|
||||
adjusted_messages.push_back(msg);
|
||||
}
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.preserved_tokens = { TOOL_CALL_START, TOOL_CALL_END, THINK_START, THINK_END };
|
||||
if (tool_list_tokens) {
|
||||
data.preserved_tokens.push_back(TOOL_LIST_START);
|
||||
data.preserved_tokens.push_back(TOOL_LIST_END);
|
||||
}
|
||||
|
||||
data.thinking_start_tag = THINK_START;
|
||||
data.thinking_end_tags = {THINK_END};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
// Gate by reasoning format and whether the template supports <think>
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE &&
|
||||
tmpl.source().find(THINK_START) != std::string::npos;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PROMPT);
|
||||
auto end = p.end();
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (extract_reasoning) {
|
||||
reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
|
||||
}
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
if (has_response_format) {
|
||||
auto response_format = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema));
|
||||
return generation_prompt + reasoning + response_format + end;
|
||||
}
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
auto tool_calls = p.rule("tool-calls",
|
||||
p.trigger_rule("tool-call",
|
||||
p.literal(TOOL_CALL_START) +
|
||||
p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) +
|
||||
p.literal(TOOL_CALL_END)
|
||||
)
|
||||
);
|
||||
|
||||
auto content = p.content(p.until(TOOL_CALL_START));
|
||||
|
||||
return generation_prompt + reasoning + content + tool_calls + end;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, TOOL_CALL_START }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// Ling 3.0 / Bailing V3 - <role>X</role> sections with tagged tool calls:
|
||||
// assistant := [<think> ... </think>] [content] {<tool_call>name
|
||||
// <arg_key>k</arg_key>\n<arg_value>v</arg_value> ...</tool_call>}
|
||||
// The generation prompt ends with "<role>ASSISTANT</role>\n<think>", so the model
|
||||
// never emits the opening think tag, and a tool call can arrive before any
|
||||
// </think>. Reasoning therefore terminates at the think close tag or at a tool
|
||||
// call start, like the Qwen3-Coder and Kimi K3 parsers. With thinking off the
|
||||
// template pre-closes the think block instead, and the model emits bare content.
|
||||
common_chat_params common_chat_params_init_ling3(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
const std::string ROLE = "<role>ASSISTANT</role>";
|
||||
const std::string THINK_START = "<think>";
|
||||
const std::string THINK_END = "</think>";
|
||||
const std::string CALL_START = "<tool_call>";
|
||||
const std::string CALL_END = "</tool_call>";
|
||||
const std::string ARG_KEY = "<arg_key>";
|
||||
const std::string ARG_KEY_END = "</arg_key>";
|
||||
const std::string ARG_VAL = "<arg_value>";
|
||||
const std::string ROLE_END = "<|role_end|>";
|
||||
const std::string ARG_VAL_END = "</arg_value>";
|
||||
|
||||
data.preserved_tokens = {
|
||||
THINK_START, THINK_END, CALL_START, CALL_END,
|
||||
ARG_KEY, ARG_KEY_END, ARG_VAL, ARG_VAL_END, ROLE_END,
|
||||
};
|
||||
|
||||
data.thinking_start_tag = THINK_START;
|
||||
// Support both </think> and <tool_call> as reasoning end sequences: a call
|
||||
// can be emitted before the think block is closed.
|
||||
data.thinking_end_tags = { THINK_END, CALL_START };
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<role>ASSISTANT</role>" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<role>HUMAN</role>" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<role>OBSERVATION</role>" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<role>SYSTEM</role>" },
|
||||
};
|
||||
|
||||
// the model may spell the end-of-turn control token out as text tokens,
|
||||
// which does not stop generation; a literal stop string catches it either
|
||||
// way (as the Laguna patch does for its </assistant> token)
|
||||
data.additional_stops = { ROLE_END };
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = ROLE + "\n" + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
// The generation prompt pre-opens the think block when thinking is on, so
|
||||
// the opening tag is optional here and reasoning runs until </think> or a
|
||||
// tool call start; with thinking off the template pre-closes the block and
|
||||
// everything the model emits is content.
|
||||
bool think_open = false;
|
||||
if (inputs.has_continuation()) {
|
||||
think_open = inputs.continue_final_message != COMMON_CHAT_CONTINUATION_CONTENT;
|
||||
} else {
|
||||
auto last_open = data.generation_prompt.rfind(THINK_START);
|
||||
auto last_close = data.generation_prompt.rfind(THINK_END);
|
||||
think_open = last_open != std::string::npos &&
|
||||
(last_close == std::string::npos || last_open > last_close);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto end = p.end();
|
||||
|
||||
// the effective parse input is generation_prompt + model output, so the
|
||||
// assistant opener is optionally consumed here
|
||||
auto opener = p.optional(p.literal(ROLE) + p.optional(p.space()));
|
||||
|
||||
// the generation prompt pre-opens the think block, so the opening tag
|
||||
// is optional; a missing close tag does not swallow a tool call
|
||||
auto body_end = think_open ? p.until_one_of({ THINK_END, CALL_START }) : p.until_one_of({ THINK_END });
|
||||
auto think_body = extract_reasoning ? p.reasoning(body_end) : p.content(body_end);
|
||||
|
||||
auto reasoning = p.optional(p.optional(p.literal(THINK_START)) + think_body +
|
||||
p.optional(p.literal(THINK_END)));
|
||||
|
||||
// content between the think block and the first tool call, plus any
|
||||
// trailing text after the last tool call, are plain content
|
||||
auto content = p.optional(p.content(p.until_one_of({ CALL_START })));
|
||||
|
||||
// a trailing end-of-turn token is consumed instead of leaking into content
|
||||
auto tail = p.optional(p.content(p.until(ROLE_END))) + p.optional(p.literal(ROLE_END));
|
||||
|
||||
// the think block must close before the JSON, so the turn cannot end inside the reasoning
|
||||
if (has_response_format) {
|
||||
auto closed_reasoning = p.literal(THINK_START) + think_body + p.literal(THINK_END);
|
||||
auto response_format = p.content(p.schema(p.json(), "response-format", inputs.json_schema));
|
||||
return opener + (closed_reasoning << response_format) + end;
|
||||
}
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return opener + reasoning + tail + end;
|
||||
}
|
||||
|
||||
auto tool_choices = p.choice();
|
||||
auto arg_close = p.tool_arg_close(p.literal(ARG_VAL_END));
|
||||
auto arg_string = p.rule("ling3-arg-string",
|
||||
p.tool_arg_string_value(p.until(ARG_VAL_END)) + arg_close);
|
||||
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
std::vector<common_peg_parser> required_args;
|
||||
std::vector<common_peg_parser> optional_args;
|
||||
|
||||
// each argument may be preceded by whitespace: the model emits
|
||||
// newlines between arguments, the template history does not
|
||||
foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
|
||||
auto rule_name = "ling3-arg-" + std::to_string(tool_index) + "-" + std::to_string(param_index);
|
||||
|
||||
auto types = param.schema->value_types();
|
||||
|
||||
// string arguments are raw text up to the closing tag, other
|
||||
// types parse as JSON per their schema; each alternative
|
||||
// consumes the closing tag itself so a JSON prefix can not
|
||||
// commit the choice before the tag matches
|
||||
auto arg_value = p.eps();
|
||||
if (!types.has(common_chat_schema::TYPE_STRING)) {
|
||||
arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
|
||||
} else if (types.is_only(common_chat_schema::TYPE_STRING)) {
|
||||
arg_value = arg_string;
|
||||
} else {
|
||||
// the parser tries the JSON alternative first to type the value
|
||||
arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close) | arg_string,
|
||||
"ling3-arg-string");
|
||||
}
|
||||
|
||||
auto arg = p.rule(rule_name,
|
||||
p.optional(p.space()) +
|
||||
p.tool_arg(p.tool_arg_open(p.literal(ARG_KEY) + p.tool_arg_name(p.literal(param.name)) +
|
||||
p.literal(ARG_KEY_END)) +
|
||||
p.optional(p.space()) + p.literal(ARG_VAL) +
|
||||
arg_value));
|
||||
|
||||
(param.required ? required_args : optional_args).push_back(arg);
|
||||
});
|
||||
|
||||
// required arguments in any order (as Qwen3-Coder does), then
|
||||
// optional ones in any order and number
|
||||
auto args = p.permute("ling3-" + std::to_string(tool_index) + "-args", required_args);
|
||||
if (!optional_args.empty()) {
|
||||
args = args + p.zero_or_more(p.choice(optional_args));
|
||||
}
|
||||
|
||||
auto call = p.tool(p.tool_open(p.literal(CALL_START) + p.tool_name(p.literal(name)) +
|
||||
p.optional(p.space())) +
|
||||
p.tool_args(args) +
|
||||
p.tool_close(p.optional(p.space()) + p.literal(CALL_END)));
|
||||
|
||||
tool_choices |= p.rule("ling3-tool-" + std::to_string(tool_index), call);
|
||||
});
|
||||
|
||||
auto calls = inputs.parallel_tool_calls ?
|
||||
tool_choices + p.zero_or_more(p.space() + tool_choices) :
|
||||
tool_choices;
|
||||
|
||||
auto tools_section = p.trigger_rule("ling3-tool-call", calls + p.space() +
|
||||
p.optional(p.content(p.until(ROLE_END))) + p.optional(p.literal(ROLE_END)));
|
||||
|
||||
auto tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? tools_section :
|
||||
p.optional(tools_section);
|
||||
|
||||
return opener + reasoning + content + tools + tail + end;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, CALL_START },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// LLM-jp-4.1: the GPT-OSS (Harmony) format with two differences
|
||||
// - the tokenizer emits a space after every special token: "<|channel|> analysis<|message|> ..."
|
||||
// - parallel tool calls are consecutive assistant messages, all but the last closed by <|end|>
|
||||
common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
// Copy reasoning to the "thinking" field as expected by the template
|
||||
auto adjusted_messages = json::array();
|
||||
for (auto msg : inputs.messages) {
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
msg["thinking"] = msg.at("reasoning_content");
|
||||
if (msg.contains("tool_calls") && msg.at("tool_calls").is_array() && !msg.at("tool_calls").empty()) {
|
||||
msg.erase("content");
|
||||
}
|
||||
}
|
||||
adjusted_messages.push_back(msg);
|
||||
}
|
||||
|
||||
auto prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);
|
||||
|
||||
// Check if we need to replace the return token with end token during
|
||||
// inference and without generation prompt. For more details see:
|
||||
// https://github.com/ggml-org/llama.cpp/issues/15417
|
||||
if (inputs.is_inference && !inputs.add_generation_prompt) {
|
||||
static constexpr std::string_view return_token = "<|return|>";
|
||||
static constexpr std::string_view end_token = "<|end|>";
|
||||
prompt.replace_last(std::string(return_token), std::string(end_token));
|
||||
}
|
||||
|
||||
data.prompt = std::move(prompt);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages);
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|start|>functions" },
|
||||
};
|
||||
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
data.thinking_start_tag = "<|channel|>analysis<|message|>";
|
||||
data.thinking_end_tags = {"<|end|>"};
|
||||
|
||||
// These special tokens are required to parse properly
|
||||
data.preserved_tokens = {
|
||||
"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",
|
||||
};
|
||||
|
||||
// Adjust prompt for continuation
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|start|>assistant<|channel|>analysis<|message|>" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<|end|><|start|>assistant<|channel|>final<|message|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
// tokenizer space after special tokens; not p.space() since GBNF `space` allows one space only
|
||||
auto sp = p.chars("[ ]", 0, -1);
|
||||
auto channel_tag = p.literal("<|channel|>") + sp;
|
||||
// one space only: keep an intentional leading space in the body
|
||||
auto message = p.literal("<|message|>") + p.optional(p.literal(" "));
|
||||
|
||||
auto start = p.rule("start", p.literal("<|start|>") + sp + p.literal("assistant"));
|
||||
auto end = p.rule("end", p.literal("<|end|>"));
|
||||
auto content = p.rule("message-content", p.until("<|end|>"));
|
||||
auto channel = channel_tag + (p.literal("commentary") | p.literal("analysis"));
|
||||
auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1);
|
||||
auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>") + sp) + constrain_type);
|
||||
|
||||
auto start_analysis = channel_tag + p.literal("analysis") + message;
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", start_analysis + p.reasoning(content) + end);
|
||||
} else {
|
||||
p.rule("analysis", p.content(start_analysis + content + end));
|
||||
}
|
||||
|
||||
auto analysis = p.ref("analysis");
|
||||
auto preamble = p.rule("preamble", channel_tag + p.literal("commentary") + message + p.content(content) + end);
|
||||
auto final_msg = p.rule("final", channel_tag + p.literal("final") + message + p.content(content));
|
||||
|
||||
auto any = p.rule("any", preamble | analysis);
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format",
|
||||
channel_tag + p.literal("final") + constraint + message +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto tool_choice = p.choice();
|
||||
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto params = common_chat_tool_parameters(function);
|
||||
|
||||
auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name));
|
||||
auto args = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", params));
|
||||
|
||||
// recipient in role header
|
||||
// <|start|>assistant to=functions.NAME<|channel|>(commentary|analysis)[constraint]<|message|>ARGS
|
||||
auto tool_in_role = p.tool(p.tool_open(func_name + channel + constraint + message) + args);
|
||||
|
||||
// recipient in channel header
|
||||
// <|channel|>(commentary|analysis) to=functions.NAME[constraint]<|message|>ARGS
|
||||
auto tool_in_channel = p.tool(p.tool_open(channel + func_name + constraint + message) + args);
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_in_role | tool_in_channel);
|
||||
});
|
||||
|
||||
// parallel calls are separated by <|end|>; inside the trigger rule so the lazy grammar covers all of them
|
||||
auto tool_calls = inputs.parallel_tool_calls
|
||||
? tool_choice + p.zero_or_more(end + start + tool_choice)
|
||||
: tool_choice;
|
||||
auto tool_call = p.trigger_rule("tool-call", tool_calls);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return p.zero_or_more(start + any) + start + tool_call;
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + any) + start + (tool_call | final_msg);
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + any) + start + final_msg;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^\\s+to$" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "^<\\|channel\\|>\\s*(?:commentary|analysis)\\s+to=functions$" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>\\s*assistant(\\s+to)" },
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|start\\|>\\s*assistant(<\\|channel\\|>\\s*(?:commentary|analysis)\\s+to)" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// MiniCPM5 format:
|
||||
// - Reasoning: <think>{reasoning}</think> (optional)
|
||||
// - Tool calls: <function name="foo"><param name="bar">value</param></function>
|
||||
common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
data.preserved_tokens = {
|
||||
"<function",
|
||||
"<param",
|
||||
"</function>",
|
||||
"</param>",
|
||||
"<think>",
|
||||
"</think>",
|
||||
};
|
||||
|
||||
data.thinking_start_tag = "<think>";
|
||||
data.thinking_end_tags = {"</think>"};
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|im_start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|im_start|>assistant\n<think>\n" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "\n</think>\n\n" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal("<|im_start|>assistant\n");
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (extract_reasoning) {
|
||||
reasoning = ("<think>" << p.reasoning(p.until("</think>")) << "</think>") + p.space();
|
||||
}
|
||||
|
||||
// Response format parser
|
||||
if (has_response_format) {
|
||||
return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema));
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
// CDATA lets a value carry characters that would otherwise close the tag (e.g.
|
||||
// </param>); capture the inner text only, excluding the CDATA markers.
|
||||
auto string_value = p.choice({
|
||||
p.literal("<![CDATA[") + p.ac(p.tool_arg_string_value(p.until("]]>")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("</param>")),
|
||||
p.negate(p.literal("< {
|
||||
const auto & function = tool.at("function");
|
||||
const std::string name = function.at("name");
|
||||
|
||||
std::vector<common_peg_parser> arg_rules;
|
||||
foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
|
||||
auto value_parser = p.eps();
|
||||
if (prop.schema->may_be_string()) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *prop.schema)
|
||||
) + p.tool_arg_close(p.literal("</param>"));
|
||||
}
|
||||
|
||||
arg_rules.push_back(p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<param name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) +
|
||||
value_parser
|
||||
));
|
||||
});
|
||||
|
||||
auto args = p.eps();
|
||||
if (!arg_rules.empty()) {
|
||||
args = p.zero_or_more(p.choice(arg_rules) + p.space());
|
||||
}
|
||||
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(p.literal("<function name=\"") + p.tool_name(p.literal(name)) + p.literal("\">"))
|
||||
<< p.tool_args(args)
|
||||
<< p.tool_close(p.literal("</function>")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_parser);
|
||||
});
|
||||
|
||||
auto max_calls = inputs.parallel_tool_calls ? -1 : 1;
|
||||
auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls));
|
||||
|
||||
auto content = p.content(p.until("<function"));
|
||||
|
||||
return generation_prompt + reasoning + content + tool_calls + p.end();
|
||||
}
|
||||
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + p.end();
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "<function" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "<mm:think>";
|
||||
data.thinking_end_tags = {"</mm:think>"};
|
||||
|
||||
// M3 prefixes every tool tag with the namespace token "]<]minimax[>[";
|
||||
// params use the parameter name as the tag (<file_path>...</file_path>).
|
||||
const std::string NS = "]<]minimax[>[";
|
||||
const std::string THINK_START = "<mm:think>";
|
||||
const std::string THINK_END = "</mm:think>";
|
||||
const std::string FC_START = NS + "<tool_call>";
|
||||
const std::string FC_END = NS + "</tool_call>";
|
||||
const std::string INVOKE_END = NS + "</invoke>";
|
||||
|
||||
data.preserved_tokens = {
|
||||
NS,
|
||||
"<tool_call>",
|
||||
"</tool_call>",
|
||||
THINK_START,
|
||||
THINK_END,
|
||||
};
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" },
|
||||
{ COMMON_CHAT_ROLE_USER, "]~b]user" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "]~b]tool" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "]~b]system" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
const std::string GEN_PROMPT = data.generation_prompt;
|
||||
|
||||
using mm3 = common_chat_peg_minimax_m3_mapper;
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += THINK_END + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);
|
||||
auto end = p.end();
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (extract_reasoning) {
|
||||
auto block = inputs.enable_thinking
|
||||
? p.literal(THINK_START) + p.space() +
|
||||
p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END)
|
||||
: p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END);
|
||||
|
||||
// A turn without reasoning is prefixed with a bare </mm:think>, written either by the
|
||||
// generation prompt (thinking_mode = "disabled") or by the model itself.
|
||||
reasoning = p.optional(p.choice({ block, p.literal(THINK_END) }));
|
||||
}
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format",
|
||||
p.literal("```json") + p.space() +
|
||||
p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) +
|
||||
p.space() + p.literal("```"));
|
||||
return generation_prompt + reasoning + response_format + end;
|
||||
}
|
||||
|
||||
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
return generation_prompt + reasoning + p.content(p.rest()) + end;
|
||||
}
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
auto params = common_chat_tool_parameters(function);
|
||||
auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params));
|
||||
|
||||
// The template expands argument values recursively in XML (see the to_xml() macro)
|
||||
std::function<common_peg_parser(const common_chat_schema &, const std::string &, const std::string &)> value_of;
|
||||
std::function<common_peg_parser(const common_chat_schema_object &, const std::string &)> members_of;
|
||||
|
||||
auto element_of = [&](const std::string & tag, const common_chat_schema & schema, const std::string & rule_name) {
|
||||
const std::string close = NS + "</" + tag + ">";
|
||||
return p.rule(rule_name,
|
||||
p.tool_arg(
|
||||
p.tool_arg_open(
|
||||
p.literal(NS + "<") +
|
||||
p.tool_arg_name(p.literal(tag)) +
|
||||
p.literal(">")) +
|
||||
value_of(schema, rule_name, close)));
|
||||
};
|
||||
|
||||
value_of = [&](const common_chat_schema & schema,
|
||||
const std::string & rule_name,
|
||||
const std::string & close) -> common_peg_parser {
|
||||
auto close_tag = p.tool_arg_close(p.literal(close));
|
||||
|
||||
// A string accepts anything, so a union with a string alternative is a string
|
||||
if (schema.may_be_string()) {
|
||||
return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close);
|
||||
}
|
||||
|
||||
if (schema.kind() == common_chat_schema::KIND_ANY_OF) {
|
||||
std::vector<common_peg_parser> choices;
|
||||
|
||||
size_t index = 0;
|
||||
for (const auto & alternative : static_cast<const common_chat_schema_any_of &>(schema).children) {
|
||||
const std::string alt_name = rule_name + "-" + std::to_string(index++);
|
||||
|
||||
// There is a risk that this breaks streaming deltas, but that's a risk we
|
||||
// assume to provide tool arg streaming.
|
||||
choices.push_back(value_of(*alternative, alt_name, close));
|
||||
}
|
||||
|
||||
return p.choice(choices);
|
||||
}
|
||||
|
||||
if (schema.kind() == common_chat_schema::KIND_OBJECT) {
|
||||
const auto & object = static_cast<const common_chat_schema_object &>(schema);
|
||||
if (!object.properties.empty()) {
|
||||
return p.tag(mm3::TOOL_ARG_OBJECT, members_of(object, rule_name)) + p.space() + close_tag;
|
||||
}
|
||||
}
|
||||
|
||||
if (schema.kind() == common_chat_schema::KIND_ARRAY) {
|
||||
const std::string item_close = NS + "</item>";
|
||||
auto item = p.rule(rule_name + "-item",
|
||||
p.tag(mm3::TOOL_ARG_ITEM,
|
||||
p.literal(NS + "<item>") +
|
||||
value_of(*static_cast<const common_chat_schema_array &>(schema).items, rule_name + "-item", item_close)));
|
||||
return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag;
|
||||
}
|
||||
|
||||
return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, schema)) + close_tag;
|
||||
};
|
||||
|
||||
// Required properties in schema order, then any number of optional ones in any order.
|
||||
members_of = [&](const common_chat_schema_object & object, const std::string & rule_prefix) -> common_peg_parser {
|
||||
std::vector<common_peg_parser> required_elements;
|
||||
std::vector<common_peg_parser> optional_elements;
|
||||
for (size_t i = 0; i < object.properties.size(); i++) {
|
||||
const auto & prop = object.properties[i];
|
||||
auto element = element_of(prop.name, *prop.schema, rule_prefix + "-" + std::to_string(i));
|
||||
(prop.required ? required_elements : optional_elements).push_back(element);
|
||||
}
|
||||
|
||||
common_peg_parser members = p.eps();
|
||||
for (size_t i = 0; i < required_elements.size(); i++) {
|
||||
if (i > 0) {
|
||||
members = members + p.space();
|
||||
}
|
||||
members = members + required_elements[i];
|
||||
}
|
||||
|
||||
if (!optional_elements.empty()) {
|
||||
common_peg_parser any_optional = p.choice();
|
||||
for (const auto & element : optional_elements) {
|
||||
any_optional |= element;
|
||||
}
|
||||
members = members + p.repeat(p.space() + any_optional, 0, -1);
|
||||
}
|
||||
|
||||
return members;
|
||||
};
|
||||
|
||||
common_peg_parser invoke_body = p.eps();
|
||||
if (doc->root->kind() == common_chat_schema::KIND_OBJECT) {
|
||||
invoke_body = members_of(static_cast<const common_chat_schema_object &>(*doc->root), "tool-" + std::to_string(tool_index) + "-arg");
|
||||
}
|
||||
|
||||
auto func_parser = p.tool(
|
||||
p.tool_open(p.literal(NS + "<invoke name=\"") +
|
||||
p.tool_name(p.literal(name)) + p.literal("\">")) +
|
||||
p.space() + invoke_body + p.space() +
|
||||
p.tool_close(p.literal(INVOKE_END)));
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), func_parser);
|
||||
});
|
||||
|
||||
auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
|
||||
common_peg_parser tool_calls = p.eps();
|
||||
if (inputs.parallel_tool_calls) {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice +
|
||||
p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END));
|
||||
} else {
|
||||
tool_calls = p.trigger_rule("tool-call",
|
||||
p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END));
|
||||
}
|
||||
|
||||
if (!require_tools) {
|
||||
tool_calls = p.optional(tool_calls);
|
||||
}
|
||||
|
||||
auto content_before_tools = p.content(p.until(FC_START));
|
||||
return generation_prompt + reasoning + content_before_tools + tool_calls + end;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
// Build up messages to follow the format: https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512/blob/main/chat_template.jinja
|
||||
auto adjusted_messages = json::array();
|
||||
for (const auto & msg : inputs.messages) {
|
||||
auto role = msg.value("role", "");
|
||||
if (role != "system" && role != "assistant") {
|
||||
// Only adjust system and assistant messages. Interestingly, the system message may contain thinking.
|
||||
adjusted_messages.push_back(msg);
|
||||
continue;
|
||||
}
|
||||
|
||||
auto content = json::array();
|
||||
|
||||
// If message contains `reasoning_content`, add it as a block of type `thinking`
|
||||
if (msg.contains("reasoning_content") && msg.at("reasoning_content").is_string()) {
|
||||
content.push_back({
|
||||
{ "type", "thinking" },
|
||||
{ "thinking", msg.at("reasoning_content").get<std::string>() },
|
||||
});
|
||||
}
|
||||
|
||||
// If message contains `content`, add it as a block of type `text`
|
||||
if (msg.contains("content")) {
|
||||
if (msg.at("content").is_string()) {
|
||||
content.push_back({
|
||||
{ "type", "text" },
|
||||
{ "text", msg.at("content").get<std::string>() },
|
||||
});
|
||||
} else if (msg.at("content").is_array()) {
|
||||
auto blocks = msg.at("content");
|
||||
content.insert(blocks);
|
||||
}
|
||||
}
|
||||
|
||||
auto adjusted = msg;
|
||||
adjusted["content"] = content;
|
||||
adjusted.erase("reasoning_content");
|
||||
adjusted_messages.push_back(adjusted);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = true;
|
||||
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "[THINK]";
|
||||
data.thinking_end_tags = {"[/THINK]"};
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.preserved_tokens = {
|
||||
"[THINK]",
|
||||
"[/THINK]",
|
||||
"[TOOL_CALLS]",
|
||||
"[ARGS]",
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "[THINK]" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "[/THINK]" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.eps();
|
||||
auto reasoning =
|
||||
extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();
|
||||
|
||||
// Response format parser
|
||||
if (has_response_format) {
|
||||
// Ministral wants to emit json surrounded by code fences
|
||||
return generation_prompt + (reasoning << "```json" << p.content(p.schema(p.json(), "response-format", inputs.json_schema)) << "```");
|
||||
}
|
||||
|
||||
// Tool call parser
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
const auto schema = common_chat_tool_parameters(function);
|
||||
|
||||
tool_choice |=
|
||||
p.rule("tool-" + std::to_string(tool_index), p.tool_open(p.tool_name(p.literal(name)) + "[ARGS]") +
|
||||
p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", schema)));
|
||||
});
|
||||
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
auto max_calls = inputs.parallel_tool_calls ? -1 : 1;
|
||||
auto tool_calls = p.trigger_rule("tool-call", p.repeat("[TOOL_CALLS]" + tool_choice, min_calls, max_calls));
|
||||
|
||||
return generation_prompt + (reasoning << p.content(p.until("[TOOL_CALLS]")) << tool_calls);
|
||||
}
|
||||
|
||||
// Content only parser
|
||||
include_grammar = false;
|
||||
return generation_prompt + (reasoning << p.content(p.rest()));
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "[TOOL_CALLS]" }
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// An assistant turn is rendered as one or more messages, each
|
||||
// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is
|
||||
// <|eom|> (more messages follow) or <|eot|> (end of turn):
|
||||
// - chain-of-thought: to=self, terminated by <|eom|>
|
||||
// - final answer: to=user, terminated by <|eot|>
|
||||
// The generation prompt is just "<|start|>assistant"; the model emits its own
|
||||
// " to=...<|message|>".
|
||||
common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = "<|start|>assistant";
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
data.preserved_tokens = {
|
||||
"<|start|>", "<|message|>", "<|eom|>", "<|eot|>",
|
||||
// ATEM tool-call markup emitted on " to=<tool>" turns.
|
||||
"<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>",
|
||||
"</atem:invoke>", "</atem:function_calls>",
|
||||
};
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|start|>tool" },
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
// Constrained grammar whenever tools are offered or a response format is requested.
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
|
||||
if (!extract_reasoning && !include_grammar) {
|
||||
return start + p.content(p.rest());
|
||||
}
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
} else {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
}
|
||||
auto analysis = p.ref("analysis");
|
||||
|
||||
auto recipient = p.optional(p.literal(" to=user"));
|
||||
auto final_msg = p.rule("final", recipient + p.literal("<|message|>") +
|
||||
p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_json = p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema));
|
||||
auto response_format = p.rule("response-format",
|
||||
recipient + p.literal("<|message|>") +
|
||||
((p.literal("```json") + p.space() + response_json + p.space() + p.literal("```")) | response_json));
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + response_format;
|
||||
}
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto string_value = p.ac(
|
||||
p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
|
||||
"</atem:parameter>");
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
const std::string name = function.at("name");
|
||||
|
||||
std::vector<common_peg_parser> arg_rules;
|
||||
foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
|
||||
auto value_parser = p.eps();
|
||||
if (prop.schema->may_be_string()) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *prop.schema))
|
||||
+ p.tool_arg_close(p.literal("</atem:parameter>"));
|
||||
}
|
||||
|
||||
arg_rules.push_back(p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) +
|
||||
value_parser));
|
||||
});
|
||||
|
||||
auto args = p.eps();
|
||||
if (!arg_rules.empty()) {
|
||||
args = p.zero_or_more(p.choice(arg_rules) + p.space());
|
||||
}
|
||||
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(p.literal(" to=") + p.until("<|message|>") +
|
||||
p.literal("<|message|><atem:function_calls>") + p.space() +
|
||||
p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())
|
||||
<< p.tool_args(args)
|
||||
<< p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_parser);
|
||||
});
|
||||
|
||||
auto tool_calls = inputs.parallel_tool_calls
|
||||
? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))
|
||||
: p.trigger_rule("tool-call", tool_choice);
|
||||
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return p.zero_or_more(start + analysis) + start + tool_calls;
|
||||
}
|
||||
auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);
|
||||
return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls));
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + analysis) + start + final_msg;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
|
||||
"(?:^|<\\|start\\|>assistant)( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
#include "parsers.h"
|
||||
|
||||
// An assistant turn is rendered as one or more messages, each
|
||||
// "<|start|>assistant to=<recipient><|message|>{content}{END}" where END is
|
||||
// <|eom|> (more messages follow) or <|eot|> (end of turn):
|
||||
// - chain-of-thought: to=self, terminated by <|eom|>
|
||||
// - user facing aside: to=commentary, terminated by <|eom|>
|
||||
// - final answer: to=user, terminated by <|eot|>
|
||||
// A commentary message carries content the user sees while the assistant keeps
|
||||
// working, typically before or between tool calls.
|
||||
// The generation prompt is just "<|start|>assistant"; the model emits its own
|
||||
// " to=...<|message|>".
|
||||
common_chat_params common_chat_params_init_muse_spark(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = "<|start|>assistant";
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
data.preserved_tokens = {
|
||||
"<|start|>", "<|message|>", "<|eom|>", "<|eot|>",
|
||||
// ATEM tool-call markup emitted on " to=<tool>" turns.
|
||||
"<atem:function_calls>", "<atem:invoke", "<atem:parameter", "</atem:parameter>",
|
||||
"</atem:invoke>", "</atem:function_calls>",
|
||||
};
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|start|>tool" },
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
// Constrained grammar whenever tools are offered.
|
||||
auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto start = p.rule("start", p.literal("<|start|>assistant"));
|
||||
|
||||
if (!extract_reasoning && !include_grammar) {
|
||||
return start + p.content(p.rest());
|
||||
}
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
} else {
|
||||
p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
}
|
||||
auto analysis = p.ref("analysis");
|
||||
|
||||
// Commentary is user facing content emitted mid turn, so it reads as
|
||||
// content and always ends on <|eom|>.
|
||||
auto preamble = p.rule("preamble", p.literal(" to=commentary<|message|>") +
|
||||
p.content(p.until("<|eom|>")) + p.literal("<|eom|>"));
|
||||
|
||||
auto aside = p.rule("aside", analysis | preamble);
|
||||
|
||||
auto recipient = p.optional(p.literal(" to=user"));
|
||||
auto final_msg = p.rule("final", recipient + p.literal("<|message|>") +
|
||||
p.content(p.until_one_of({ "<|eot|>", "<|eom|>" })));
|
||||
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto string_value = p.ac(
|
||||
p.tool_arg_string_value(p.until("</atem:parameter>")) + p.tool_arg_close(p.literal("</atem:parameter>")),
|
||||
"</atem:parameter>");
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
const std::string name = function.at("name");
|
||||
|
||||
std::vector<common_peg_parser> arg_rules;
|
||||
foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & prop, const common_chat_schema_document_ptr & doc) {
|
||||
auto value_parser = p.eps();
|
||||
if (prop.schema->may_be_string()) {
|
||||
value_parser = string_value;
|
||||
} else {
|
||||
value_parser = p.tool_arg_json_value(
|
||||
p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index) + "-schema", doc, *prop.schema))
|
||||
+ p.tool_arg_close(p.literal("</atem:parameter>"));
|
||||
}
|
||||
|
||||
arg_rules.push_back(p.tool_arg(
|
||||
p.tool_arg_open(p.literal("<atem:parameter name=\"") + p.tool_arg_name(p.literal(prop.name)) + p.literal("\">")) +
|
||||
value_parser));
|
||||
});
|
||||
|
||||
auto args = p.eps();
|
||||
if (!arg_rules.empty()) {
|
||||
args = p.zero_or_more(p.choice(arg_rules) + p.space());
|
||||
}
|
||||
|
||||
auto tool_parser = p.tool(
|
||||
p.tool_open(p.literal(" to=") + p.until("<|message|>") +
|
||||
p.literal("<|message|><atem:function_calls>") + p.space() +
|
||||
p.literal("<atem:invoke name=\"") + p.tool_name(p.literal(name)) + p.literal("\">") + p.space())
|
||||
<< p.tool_args(args)
|
||||
<< p.tool_close(p.literal("</atem:invoke>") + p.space() + p.literal("</atem:function_calls>")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), tool_parser);
|
||||
});
|
||||
|
||||
auto tool_calls = inputs.parallel_tool_calls
|
||||
? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice))
|
||||
: p.trigger_rule("tool-call", tool_choice);
|
||||
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return p.zero_or_more(start + aside) + start + tool_calls;
|
||||
}
|
||||
auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls);
|
||||
return p.zero_or_more(start + aside) + start + (tool_calls | (final_msg + trailing_calls));
|
||||
}
|
||||
|
||||
return p.zero_or_more(start + aside) + start + final_msg;
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,
|
||||
"<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!commentary<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" },
|
||||
};
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
#include "parsers.h"
|
||||
|
||||
#include "log.h"
|
||||
|
||||
void foreach_function(const json & tools, const std::function<void(size_t, const json &)> & fn) {
|
||||
for (size_t i = 0; i < tools.size(); i++) {
|
||||
const auto & tool = tools[i];
|
||||
if (!tool.contains("type") || tool.at("type") != "function" || !tool.contains("function")) {
|
||||
LOG_INF("Skipping tool without function: %s", tool.dump(2).c_str());
|
||||
continue;
|
||||
}
|
||||
fn(i, tool);
|
||||
}
|
||||
}
|
||||
|
||||
void foreach_parameter(const json & function, const std::function<void(size_t, const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn) {
|
||||
auto params = common_chat_tool_parameters(function);
|
||||
auto doc = std::make_shared<const common_chat_schema_document>(common_chat_schema_from_json(params));
|
||||
const auto * object = dynamic_cast<const common_chat_schema_object *>(doc->root.get());
|
||||
if (!object) {
|
||||
return;
|
||||
}
|
||||
for (size_t i = 0; i < object->properties.size(); i++) {
|
||||
fn(i, object->properties[i], doc);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
#pragma once
|
||||
|
||||
#include "chat.h"
|
||||
#include "chat-auto-parser.h"
|
||||
#include "chat-auto-parser-helpers.h"
|
||||
#include "chat-peg-parser.h"
|
||||
#include "common.h"
|
||||
#include "ggml.h"
|
||||
#include "json-schema-to-grammar.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using json = common_json;
|
||||
|
||||
// iterate over the function tools of an OpenAI-style tools array, passing each tool with its index in the array
|
||||
void foreach_function(const json & tools, const std::function<void(size_t, const json &)> & fn);
|
||||
|
||||
// iterate over the parameters of a function tool, passing each parameter with its index and the document that owns it
|
||||
void foreach_parameter(const json & function, const std::function<void(size_t, const common_chat_schema_property &, const common_chat_schema_document_ptr &)> & fn);
|
||||
|
||||
// render a template; the override arguments let a parser feed in messages, tools or context it has rewritten
|
||||
common_chat_prompt common_chat_template_direct_apply_impl(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs,
|
||||
const std::optional<json> & messages_override = std::nullopt,
|
||||
const std::optional<json> & tools_override = std::nullopt,
|
||||
const std::optional<json> & additional_context = std::nullopt);
|
||||
|
||||
// the suffix a template appends when add_generation_prompt is set
|
||||
std::string common_chat_template_generation_prompt_impl(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs,
|
||||
const std::optional<json> & messages_override = std::nullopt,
|
||||
const std::optional<json> & tools_override = std::nullopt,
|
||||
const std::optional<json> & additional_context = std::nullopt);
|
||||
|
||||
bool is_lfm2_template(const std::string & src);
|
||||
|
||||
namespace workaround {
|
||||
|
||||
void convert_tool_responses_gemma4(json & messages);
|
||||
|
||||
}
|
||||
|
||||
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_gemma4(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_gigachat_v3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_ling3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
// tool_list_tokens preserves the LFM2 system tool-list markers; LFM2.5 renders without them
|
||||
common_chat_params common_chat_params_init_lfm2(const common_chat_template & tmpl, const autoparser::generation_params & inputs, bool tool_list_tokens);
|
||||
|
||||
common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_ministral_3(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
common_chat_params common_chat_params_init_muse_spark(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_translate_gemma(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
@@ -0,0 +1,192 @@
|
||||
#include "parsers.h"
|
||||
|
||||
common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
const std::string GEN_PREFIX = "<|im_start|>assistant\n";
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
|
||||
auto supports_reasoning = tmpl.source().find("<think>") != std::string::npos;
|
||||
|
||||
data.supports_thinking = supports_reasoning;
|
||||
data.preserved_tokens = {
|
||||
"<tool_call>",
|
||||
"</tool_call>",
|
||||
};
|
||||
|
||||
auto is_qwen3_coder = !supports_reasoning;
|
||||
|
||||
if (supports_reasoning) {
|
||||
data.thinking_start_tag = "<think>";
|
||||
// Support both </think> and <tool_call> as reasoning end sequences.
|
||||
// The newline variant comes first so it is included in the forced message
|
||||
// <function= is omitted, as it is a workaround for Qwen3-Coder which is not a thinking model
|
||||
data.thinking_end_tags = { "\n</think>", "</think>", "<tool_call>" };
|
||||
data.preserved_tokens.insert(data.preserved_tokens.end(), { "<think>", "</think>" });
|
||||
}
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n<tool_response>" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash
|
||||
{ COMMON_CHAT_ROLE_USER, "<|im_start|>user" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" },
|
||||
};
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty();
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt = GEN_PREFIX;
|
||||
if (supports_reasoning) {
|
||||
data.generation_prompt += "<think>\n" + msg.reasoning_content;
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += "\n</think>\n\n";
|
||||
}
|
||||
}
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
data.generation_prompt += msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
std::vector<std::string> tool_call_starts = { "<tool_call>" };
|
||||
|
||||
if (is_qwen3_coder) {
|
||||
// Match complete <function=name> opener for Qwen3-Coder models that occasionally omit the
|
||||
// starting <tool_call>. The model may hallucinate a tool name, but it is preferable over
|
||||
// constraining on <function which may occur in valid content generation, e.g. #include <functional>
|
||||
foreach_function(inputs.tools, [&](size_t, const json & tool) {
|
||||
const std::string name = tool.at("function").at("name");
|
||||
tool_call_starts.push_back("<function=" + name + ">");
|
||||
});
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.literal(GEN_PREFIX);
|
||||
|
||||
auto reasoning = p.eps();
|
||||
if (supports_reasoning && extract_reasoning) {
|
||||
reasoning = p.optional("<think>" + p.space() +
|
||||
p.reasoning(p.until_one_of({ "</think>", "<tool_call>" })) +
|
||||
(p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
|
||||
}
|
||||
|
||||
// Response format parser
|
||||
if (has_response_format) {
|
||||
return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
|
||||
}
|
||||
|
||||
// Tool call parser
|
||||
if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {
|
||||
auto arg_close = p.tool_arg_close(p.literal("\n</parameter>\n"));
|
||||
auto arg_string = p.rule("xml-arg-string",
|
||||
p.ac(p.tool_arg_string_value(p.until("\n</parameter>\n")) + arg_close, "\n</parameter>\n"));
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
std::vector<common_peg_parser> required_args;
|
||||
std::vector<common_peg_parser> optional_args;
|
||||
|
||||
foreach_parameter(function, [&](size_t param_index, const common_chat_schema_property & param, const common_chat_schema_document_ptr & doc) {
|
||||
auto rule_name = "tool-" + std::to_string(tool_index) + "-arg-" + std::to_string(param_index);
|
||||
|
||||
auto arg_open = p.tool_arg_open("<parameter=" + p.tool_arg_name(p.literal(param.name)) + ">\n");
|
||||
|
||||
auto types = param.schema->value_types();
|
||||
|
||||
auto arg_value = p.eps();
|
||||
if (!types.has(common_chat_schema::TYPE_STRING)) {
|
||||
arg_value = p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", doc, *param.schema)) + arg_close;
|
||||
} else if (types.is_only(common_chat_schema::TYPE_STRING)) {
|
||||
arg_value = arg_string;
|
||||
} else {
|
||||
// The string alternative accepts any text, so the grammar only keeps the raw string
|
||||
// rule. The parser still tries the JSON alternatives first to type the value.
|
||||
auto json_value = p.choice();
|
||||
if (types.has(common_chat_schema::TYPE_OBJECT)) {
|
||||
json_value |= p.json_object();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_ARRAY)) {
|
||||
json_value |= p.json_array();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_NUMBER) || types.has(common_chat_schema::TYPE_INTEGER)) {
|
||||
json_value |= p.json_number();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_BOOLEAN)) {
|
||||
json_value |= p.json_bool();
|
||||
}
|
||||
if (types.has(common_chat_schema::TYPE_NULL)) {
|
||||
json_value |= p.json_null();
|
||||
}
|
||||
arg_value = p.gbnf(p.atomic(p.tool_arg_json_value(json_value) + arg_close) | arg_string, "xml-arg-string");
|
||||
}
|
||||
|
||||
auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value));
|
||||
|
||||
(param.required ? required_args : optional_args).push_back(arg_rule);
|
||||
});
|
||||
|
||||
// Accept required arguments in any order, as Qwen does not always adhere to the
|
||||
// order provided.
|
||||
auto args = p.permute("tool-" + std::to_string(tool_index) + "-args", required_args);
|
||||
if (!optional_args.empty()) {
|
||||
args = args + p.zero_or_more(p.choice(optional_args));
|
||||
}
|
||||
|
||||
auto func = p.tool(p.tool_open("<function=" + p.tool_name(p.literal(name)) + ">\n") +
|
||||
p.tool_args(args) +
|
||||
p.tool_close(p.literal("</function>\n")));
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), func);
|
||||
});
|
||||
|
||||
auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0;
|
||||
|
||||
auto tool_call_body = tool_choice + "</tool_call>" + p.space();
|
||||
auto tool_call = p.rule("tool-call", "<tool_call>\n" + tool_call_body);
|
||||
|
||||
// Qwen3-Coder models may occasionally omit the <tool_call> token.
|
||||
auto tool_call_first = is_qwen3_coder ?
|
||||
p.rule("tool-call-first", p.optional(p.literal("<tool_call>\n")) + tool_call_body) :
|
||||
tool_call;
|
||||
|
||||
auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first;
|
||||
auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));
|
||||
|
||||
return generation_prompt +
|
||||
(reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
|
||||
}
|
||||
|
||||
// Content only parser
|
||||
return generation_prompt + (reasoning << p.content(p.rest()));
|
||||
});
|
||||
|
||||
if (include_grammar) {
|
||||
data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;
|
||||
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
for (const auto & start : tool_call_starts) {
|
||||
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
# Specialized chat template parsers, listed explicitly so that adding or removing one re-runs CMake instead of leaving an incremental build stale.
|
||||
|
||||
set(LLAMA_CHAT_PARSERS_SOURCES
|
||||
${CMAKE_CURRENT_LIST_DIR}/parsers.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/parsers.h
|
||||
${CMAKE_CURRENT_LIST_DIR}/cohere2moe.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/deepseek.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/functionary-v3-2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gemma4.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/ling3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/lfm2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/llm-jp-harmony.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/minicpm5.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/minimax-m3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/ministral3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/muse-glimmer.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/muse-spark.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/qwen3-coder.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/translate-gemma.cpp
|
||||
)
|
||||
@@ -0,0 +1,62 @@
|
||||
#include "parsers.h"
|
||||
#include "log.h"
|
||||
|
||||
// TranslateGemma does not support tools or reasoning, it only needs user messages in its own content schema
|
||||
common_chat_params common_chat_params_init_translate_gemma(
|
||||
const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
|
||||
common_chat_params data;
|
||||
|
||||
// default to chat_template_kwargs, or en-GB if not specified
|
||||
std::string src_lang = inputs.extra_context.value("source_lang_code", "en-GB");
|
||||
std::string tgt_lang = inputs.extra_context.value("target_lang_code", "en-GB");
|
||||
for (const char * key : { "source_lang_code", "target_lang_code" }) {
|
||||
if (!inputs.extra_context.contains(key)) {
|
||||
LOG_WRN("TranslateGemma: %s not set in chat_template_kwargs, defaulting to en-GB\n", key);
|
||||
}
|
||||
}
|
||||
|
||||
json messages = inputs.messages;
|
||||
for (auto & message : messages) {
|
||||
if (message.value("role", "") != "user") {
|
||||
continue;
|
||||
}
|
||||
std::string text;
|
||||
const auto & content = message.contains("content") ? message.at("content") : json();
|
||||
if (content.is_string()) {
|
||||
text = content.get<std::string>();
|
||||
} else if (content.is_array()) {
|
||||
for (const auto & part : content) {
|
||||
if (!text.empty()) {
|
||||
text += "\n";
|
||||
}
|
||||
text += part.value("text", "");
|
||||
}
|
||||
}
|
||||
message["content"] = json::array({
|
||||
json{
|
||||
{"type", "text"},
|
||||
{"text", text},
|
||||
{"source_lang_code", src_lang},
|
||||
{"target_lang_code", tgt_lang},
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, messages);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, messages);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = false;
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
data.generation_prompt = "<start_of_turn>model\n" + inputs.continue_msg.render_content();
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
return p.literal(data.generation_prompt) << p.content(p.rest());
|
||||
});
|
||||
|
||||
return data;
|
||||
}
|
||||
Reference in new issue
Block a user