include llama.cpp since there were some edits
This commit is contained in:
zefie committed
1 parent
db7f28d087
commit
8c66990853
3918 files changed
+1640170
-1
No files matched your search
@@ -0,0 +1,9 @@
|
||||
*
|
||||
!*.*
|
||||
!snapshots/
|
||||
!fusion/
|
||||
!llama-context/
|
||||
*.o
|
||||
ggml-common.h
|
||||
**/*.swp
|
||||
!peg-parser
|
||||
@@ -0,0 +1,402 @@
|
||||
llama_add_compile_flags()
|
||||
|
||||
# Optional args:
|
||||
# - LINK: libraries to link against (defaults to llama-common, which also brings llama)
|
||||
function(llama_build source)
|
||||
set(options)
|
||||
set(oneValueArgs)
|
||||
set(multiValueArgs LINK)
|
||||
cmake_parse_arguments(LLAMA_BUILD "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
|
||||
|
||||
set(TEST_SOURCES ${source} ${LLAMA_BUILD_UNPARSED_ARGUMENTS})
|
||||
|
||||
if (NOT LLAMA_BUILD_LINK)
|
||||
set(LLAMA_BUILD_LINK llama-common)
|
||||
endif()
|
||||
|
||||
if (DEFINED LLAMA_TEST_NAME)
|
||||
set(TEST_TARGET ${LLAMA_TEST_NAME})
|
||||
else()
|
||||
get_filename_component(TEST_TARGET ${source} NAME_WE)
|
||||
endif()
|
||||
|
||||
add_executable(${TEST_TARGET} ${TEST_SOURCES})
|
||||
target_compile_features(${TEST_TARGET} PRIVATE cxx_std_17)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ${LLAMA_BUILD_LINK})
|
||||
if (LLAMA_TESTS_INSTALL)
|
||||
install(TARGETS ${TEST_TARGET} RUNTIME)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(llama_test target)
|
||||
include(CMakeParseArguments)
|
||||
set(options)
|
||||
set(oneValueArgs NAME LABEL WORKING_DIRECTORY)
|
||||
set(multiValueArgs ARGS)
|
||||
cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
|
||||
|
||||
if (NOT DEFINED LLAMA_TEST_LABEL)
|
||||
set(LLAMA_TEST_LABEL "main")
|
||||
endif()
|
||||
if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY)
|
||||
set(LLAMA_TEST_WORKING_DIRECTORY .)
|
||||
endif()
|
||||
if (DEFINED LLAMA_TEST_NAME)
|
||||
set(TEST_NAME ${LLAMA_TEST_NAME})
|
||||
else()
|
||||
set(TEST_NAME ${target})
|
||||
endif()
|
||||
|
||||
set(TEST_TARGET ${target})
|
||||
|
||||
add_test(
|
||||
NAME ${TEST_NAME}
|
||||
WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY}
|
||||
COMMAND $<TARGET_FILE:${TEST_TARGET}>
|
||||
${LLAMA_TEST_ARGS})
|
||||
|
||||
set_property(TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL})
|
||||
endfunction()
|
||||
|
||||
function(llama_test_cmd target)
|
||||
include(CMakeParseArguments)
|
||||
set(options)
|
||||
set(oneValueArgs NAME LABEL WORKING_DIRECTORY)
|
||||
set(multiValueArgs ARGS)
|
||||
cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
|
||||
|
||||
if (NOT DEFINED LLAMA_TEST_LABEL)
|
||||
set(LLAMA_TEST_LABEL "main")
|
||||
endif()
|
||||
if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY)
|
||||
set(LLAMA_TEST_WORKING_DIRECTORY .)
|
||||
endif()
|
||||
if (DEFINED LLAMA_TEST_NAME)
|
||||
set(TEST_NAME ${LLAMA_TEST_NAME})
|
||||
else()
|
||||
set(TEST_NAME ${target})
|
||||
endif()
|
||||
|
||||
add_test(
|
||||
NAME ${TEST_NAME}
|
||||
WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY}
|
||||
COMMAND ${target}
|
||||
${LLAMA_TEST_ARGS})
|
||||
|
||||
set_property(TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL})
|
||||
endfunction()
|
||||
|
||||
# Builds and runs a test source file.
|
||||
# Optional args:
|
||||
# - NAME: name of the executable & test target (defaults to the source file name without extension)
|
||||
# - LABEL: label for the test (defaults to main)
|
||||
# - ARGS: arguments to pass to the test executable
|
||||
# - WORKING_DIRECTORY
|
||||
# - LINK: libraries to link against (defaults to llama-common, which also brings llama)
|
||||
function(llama_build_and_test source)
|
||||
include(CMakeParseArguments)
|
||||
set(options)
|
||||
set(oneValueArgs NAME LABEL WORKING_DIRECTORY)
|
||||
set(multiValueArgs ARGS LINK)
|
||||
cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
|
||||
|
||||
set(TEST_SOURCES ${source} ${LLAMA_TEST_UNPARSED_ARGUMENTS})
|
||||
|
||||
if (NOT LLAMA_TEST_LINK)
|
||||
set(LLAMA_TEST_LINK llama-common)
|
||||
endif()
|
||||
|
||||
if (NOT DEFINED LLAMA_TEST_LABEL)
|
||||
set(LLAMA_TEST_LABEL "main")
|
||||
endif()
|
||||
if (NOT DEFINED LLAMA_TEST_WORKING_DIRECTORY)
|
||||
set(LLAMA_TEST_WORKING_DIRECTORY .)
|
||||
endif()
|
||||
if (DEFINED LLAMA_TEST_NAME)
|
||||
set(TEST_TARGET ${LLAMA_TEST_NAME})
|
||||
else()
|
||||
get_filename_component(TEST_TARGET ${source} NAME_WE)
|
||||
endif()
|
||||
|
||||
add_executable(${TEST_TARGET} ${TEST_SOURCES})
|
||||
target_compile_features(${TEST_TARGET} PRIVATE cxx_std_17)
|
||||
if (LLAMA_TESTS_INSTALL)
|
||||
install(TARGETS ${TEST_TARGET} RUNTIME)
|
||||
endif()
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ${LLAMA_TEST_LINK})
|
||||
|
||||
add_test(
|
||||
NAME ${TEST_TARGET}
|
||||
WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY}
|
||||
COMMAND $<TARGET_FILE:${TEST_TARGET}>
|
||||
${LLAMA_TEST_ARGS})
|
||||
|
||||
set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL})
|
||||
endfunction()
|
||||
|
||||
# build test-tokenizer-0 target once and add many tests
|
||||
llama_build(test-tokenizer-0.cpp)
|
||||
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-bert-bge LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-bert-bge.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-command-r LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-command-r.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-coder.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-llm.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-falcon.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf --check-decode)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2 LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-spm.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-mpt LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-mpt.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-phi-3 LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-phi-3.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-qwen2 LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-qwen2.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-qwen35 LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-qwen35.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-refact LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-refact.gguf)
|
||||
llama_test(test-tokenizer-0 NAME test-tokenizer-0-starcoder LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-starcoder.gguf)
|
||||
|
||||
if (NOT WIN32)
|
||||
llama_test_cmd(
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/test-tokenizers-repo.sh
|
||||
NAME test-tokenizers-ggml-vocabs
|
||||
LABEL behavior
|
||||
WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}
|
||||
ARGS https://huggingface.co/ggml-org/vocabs ${PROJECT_SOURCE_DIR}/models/ggml-vocabs
|
||||
)
|
||||
endif()
|
||||
|
||||
if (LLAMA_LLGUIDANCE)
|
||||
llama_build_and_test(test-grammar-llguidance.cpp LABEL "main behavior" ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf)
|
||||
endif ()
|
||||
|
||||
# llama_context state tests: KV and recurrent state save, load, copy, rollback
|
||||
set(LLAMA_CONTEXT_TEST_SRC
|
||||
test-llama-context.cpp
|
||||
llama-context/cases-kv.cpp
|
||||
llama-context/cases-rs.cpp
|
||||
llama-context/test-case.cpp
|
||||
llama-context/test-log.cpp
|
||||
llama-context/test-runner.cpp
|
||||
llama-context/test-util.cpp
|
||||
)
|
||||
llama_build(${LLAMA_CONTEXT_TEST_SRC})
|
||||
|
||||
if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
|
||||
# these tests are disabled on Windows because they use internal functions not exported with LLAMA_API (when building with shared libraries)
|
||||
llama_build_and_test(test-unicode.cpp LABEL behavior LINK llama)
|
||||
llama_build_and_test(test-sampling.cpp LABEL behavior LINK llama)
|
||||
llama_build_and_test(test-reasoning-budget.cpp LABEL behavior)
|
||||
llama_build_and_test(test-grammar-parser.cpp LABEL behavior LINK llama)
|
||||
llama_build_and_test(test-grammar-integration.cpp LABEL behavior)
|
||||
llama_build_and_test(test-llama-grammar.cpp LABEL behavior LINK llama)
|
||||
llama_build_and_test(test-batch-alloc.cpp LABEL behavior LINK llama)
|
||||
llama_build_and_test(test-chat.cpp LABEL behavior WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
|
||||
target_include_directories(test-chat PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
|
||||
target_link_libraries(test-chat PRIVATE server-context)
|
||||
llama_build_and_test(test-server-component.cpp LABEL behavior)
|
||||
target_include_directories(test-server-component PRIVATE ${PROJECT_SOURCE_DIR}/tools/server)
|
||||
target_link_libraries(test-server-component PRIVATE server-context)
|
||||
llama_build_and_test(test-json-schema-to-grammar.cpp LABEL behavior)
|
||||
|
||||
if (NOT GGML_BACKEND_DL)
|
||||
llama_build(test-quantize-stats.cpp)
|
||||
endif()
|
||||
|
||||
llama_build(test-gbnf-validator.cpp LINK llama)
|
||||
|
||||
# build test-tokenizer-1-bpe target once and add many tests
|
||||
llama_build(test-tokenizer-1-bpe.cpp)
|
||||
|
||||
# TODO: disabled due to slowness
|
||||
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-aquila LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-aquila.gguf)
|
||||
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-falcon LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-falcon.gguf)
|
||||
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-gpt-2 LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf)
|
||||
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-gpt-neox LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-neox.gguf)
|
||||
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-llama-bpe LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf --ignore-merges)
|
||||
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-mpt LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-mpt.gguf)
|
||||
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-refact LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-refact.gguf)
|
||||
#llama_test(test-tokenizer-1-bpe NAME test-tokenizer-1-starcoder LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-starcoder.gguf)
|
||||
|
||||
# build test-tokenizer-1-spm target once and add many tests
|
||||
llama_build(test-tokenizer-1-spm.cpp)
|
||||
|
||||
llama_test(test-tokenizer-1-spm NAME test-tokenizer-1-llama-spm LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-spm.gguf)
|
||||
#llama_test(test-tokenizer-1-spm NAME test-tokenizer-1-baichuan LABEL behavior ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-baichuan.gguf)
|
||||
|
||||
# llama_build_and_test(test-double-float.cpp) # SLOW
|
||||
|
||||
endif()
|
||||
|
||||
llama_build(test-llama-archs.cpp)
|
||||
|
||||
set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-models/")
|
||||
file(MAKE_DIRECTORY "${MODEL_DIR}")
|
||||
|
||||
llama_test(
|
||||
test-llama-archs
|
||||
NAME test-generate-models
|
||||
LABEL "main behavior backend"
|
||||
ARGS -o "${MODEL_DIR}"
|
||||
)
|
||||
set_tests_properties(test-generate-models PROPERTIES
|
||||
FIXTURES_SETUP generate-models
|
||||
)
|
||||
|
||||
# Target model for test-speculative-rejection-sampling
|
||||
set(SPEC_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-speculative-models/target")
|
||||
file(MAKE_DIRECTORY "${SPEC_TARGET_DIR}")
|
||||
llama_test(
|
||||
test-llama-archs
|
||||
NAME test-generate-speculative-target
|
||||
LABEL main
|
||||
ARGS -a llama -s 999 -o "${SPEC_TARGET_DIR}"
|
||||
)
|
||||
set_tests_properties(
|
||||
test-generate-speculative-target
|
||||
PROPERTIES FIXTURES_SETUP speculative-model
|
||||
)
|
||||
|
||||
# To avoid recreating the model use the fixture exclude any option:
|
||||
# -FA "^speculative-model$"
|
||||
llama_build_and_test(test-speculative-rejection-sampling.cpp ARGS
|
||||
-m "${SPEC_TARGET_DIR}/llama-dense.gguf")
|
||||
set_tests_properties(test-speculative-rejection-sampling PROPERTIES FIXTURES_REQUIRED speculative-model)
|
||||
|
||||
# Test context state handling across all architectures, using the generated dummy models
|
||||
llama_test(
|
||||
test-llama-context
|
||||
LABEL "main behavior backend"
|
||||
ARGS --models "${MODEL_DIR}"
|
||||
)
|
||||
# abort on graph reallocations that the initial reserve should have covered
|
||||
set_tests_properties(test-llama-context PROPERTIES
|
||||
FIXTURES_REQUIRED generate-models
|
||||
ENVIRONMENT "GGML_SCHED_DEBUG_REALLOC=1"
|
||||
)
|
||||
|
||||
llama_build(test-fusion.cpp)
|
||||
|
||||
llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple-tokenize.cpp LABEL behavior)
|
||||
llama_build_and_test(test-jinja.cpp LABEL behavior)
|
||||
llama_test(test-jinja NAME test-jinja-py ARGS -py LABEL "python behavior")
|
||||
llama_build_and_test(test-json-schema.cpp LABEL behavior)
|
||||
llama_build_and_test(test-chat-auto-parser.cpp LABEL behavior WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
|
||||
llama_build_and_test(test-chat-template.cpp LABEL behavior)
|
||||
# debug tool for chat template differential analysis (not registered as a test, run it manually)
|
||||
llama_build(test-chat-analysis.cpp)
|
||||
llama_build_and_test(test-log.cpp LABEL behavior)
|
||||
llama_build_and_test(
|
||||
test-peg-parser.cpp
|
||||
peg-parser/simple-tokenize.cpp
|
||||
peg-parser/test-basic.cpp
|
||||
peg-parser/test-gbnf-generation.cpp
|
||||
peg-parser/test-json-parser.cpp
|
||||
peg-parser/test-python-dict-parser.cpp
|
||||
peg-parser/test-unicode.cpp
|
||||
peg-parser/tests.h
|
||||
LABEL behavior
|
||||
)
|
||||
|
||||
if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x")
|
||||
set(MODEL_NAME "tinyllamas/stories15M-q4_0.gguf")
|
||||
set(MODEL_HASH "SHA256=66967fbece6dbe97886593fdbb73589584927e29119ec31f08090732d1861739")
|
||||
else()
|
||||
set(MODEL_NAME "tinyllamas/stories15M-be.Q4_0.gguf")
|
||||
set(MODEL_HASH "SHA256=9aec857937849d976f30397e97eb1cabb53eb9dcb1ce4611ba8247fb5f44c65d")
|
||||
endif()
|
||||
set(MODEL_DEST "${CMAKE_BINARY_DIR}/${MODEL_NAME}")
|
||||
|
||||
add_test(NAME test-download-model COMMAND ${CMAKE_COMMAND}
|
||||
-DDEST=${MODEL_DEST}
|
||||
-DNAME=${MODEL_NAME}
|
||||
-DHASH=${MODEL_HASH}
|
||||
-P ${PROJECT_SOURCE_DIR}/cmake/download-models.cmake
|
||||
)
|
||||
set_tests_properties(test-download-model PROPERTIES FIXTURES_SETUP test-download-model)
|
||||
|
||||
# the context state cases against a real model, the sweep above uses the generated dummy ones
|
||||
llama_test(test-llama-context NAME test-llama-context-model LABEL "model behavior backend" ARGS -m "${MODEL_DEST}")
|
||||
set_tests_properties(test-llama-context-model PROPERTIES FIXTURES_REQUIRED test-download-model)
|
||||
|
||||
llama_build_and_test(test-thread-safety.cpp LABEL "main backend" ARGS -m "${MODEL_DEST}" -ngl 99 -p "The meaning of life is" -n 128 -c 256 -ub 32 -np 4 -t 2)
|
||||
set_tests_properties(test-thread-safety PROPERTIES FIXTURES_REQUIRED test-download-model)
|
||||
|
||||
llama_build_and_test(test-arg-parser.cpp LABEL behavior)
|
||||
llama_build_and_test(test-model-resolution.cpp LABEL behavior)
|
||||
# the test serves its repos from an httplib server, and the library links it privately
|
||||
target_link_libraries(test-model-resolution PRIVATE cpp-httplib)
|
||||
|
||||
if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC)
|
||||
# TODO: repair known memory leaks
|
||||
llama_build_and_test(test-opt.cpp LABEL "main behavior backend" LINK ggml)
|
||||
endif()
|
||||
if (GGML_METAL AND NOT GGML_BACKEND_DL)
|
||||
# calls the Metal backend's graph optimizer and fusion table directly
|
||||
llama_build_and_test(test-metal-graph-optimize.cpp LABEL "main behavior backend" LINK ggml ggml-metal)
|
||||
target_include_directories(test-metal-graph-optimize PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src ${PROJECT_SOURCE_DIR}/ggml/src/ggml-metal)
|
||||
endif()
|
||||
|
||||
llama_build(test-backend-ops.cpp LINK ggml)
|
||||
|
||||
llama_build_and_test(test-tiled-mulmat.cpp LABEL "main behavior backend" LINK ggml)
|
||||
llama_build_and_test(test-model-load-cancel.cpp LABEL "model behavior")
|
||||
llama_build_and_test(test-autorelease.cpp LABEL "model behavior")
|
||||
llama_build_and_test(test-backend-sampler.cpp LABEL "model behavior backend")
|
||||
|
||||
if (APPLE)
|
||||
llama_build(test-rset-release.cpp)
|
||||
endif()
|
||||
|
||||
if (NOT GGML_BACKEND_DL AND GGML_CPU)
|
||||
# these tests use the backends directly and cannot be built with dynamic loading
|
||||
llama_build_and_test(test-barrier.cpp LABEL "main behavior" LINK ggml)
|
||||
llama_build_and_test(test-quantize-fns.cpp LABEL "main behavior" LINK ggml)
|
||||
llama_build_and_test(test-quantize-perf.cpp LABEL "main behavior backend" LINK ggml)
|
||||
llama_build_and_test(test-rope.cpp LABEL "main behavior backend" LINK ggml)
|
||||
llama_build_and_test(test-col2im-1d.cpp LABEL "main behavior backend" LINK ggml)
|
||||
llama_build_and_test(test-conv-1d.cpp LABEL "main behavior backend" LINK ggml)
|
||||
llama_build_and_test(test-gelu-quick.cpp LABEL "main behavior backend" LINK ggml)
|
||||
endif()
|
||||
|
||||
# libmtmd
|
||||
llama_build_and_test(test-mtmd-c-api.c LABEL behavior LINK mtmd)
|
||||
|
||||
llama_build_and_test(test-mtmd-impl.cpp LABEL behavior LINK mtmd)
|
||||
|
||||
# GGUF model data fetcher library for tests that need real model metadata
|
||||
# Only compile when cpp-httplib has SSL support (CPPHTTPLIB_OPENSSL_SUPPORT)
|
||||
if (TARGET cpp-httplib)
|
||||
get_target_property(_cpp_httplib_defs cpp-httplib INTERFACE_COMPILE_DEFINITIONS)
|
||||
if (_cpp_httplib_defs MATCHES "CPPHTTPLIB_OPENSSL_SUPPORT")
|
||||
add_library(gguf-model-data STATIC gguf-model-data.cpp)
|
||||
target_link_libraries(gguf-model-data PRIVATE llama-common cpp-httplib)
|
||||
target_include_directories(gguf-model-data PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
add_executable(test-gguf-model-data test-gguf-model-data.cpp)
|
||||
target_link_libraries(test-gguf-model-data PRIVATE gguf-model-data llama-common)
|
||||
llama_test(test-gguf-model-data LABEL "model behavior")
|
||||
|
||||
# test-quant-type-selection requires gguf-model-data for remote model metadata
|
||||
llama_build_and_test(test-quant-type-selection.cpp LABEL "model behavior" LINK llama)
|
||||
target_link_libraries(test-quant-type-selection PRIVATE gguf-model-data)
|
||||
target_compile_definitions(test-quant-type-selection PRIVATE
|
||||
SNAPSHOT_DIR="${CMAKE_CURRENT_SOURCE_DIR}/snapshots")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# dummy executable - not installed
|
||||
get_filename_component(TEST_TARGET test-c.c NAME_WE)
|
||||
add_executable(${TEST_TARGET} test-c.c)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE llama)
|
||||
|
||||
if (NOT LLAMA_USE_SYSTEM_GGML)
|
||||
# Needs non-public ggml-impl.h
|
||||
llama_build_and_test(test-gguf.cpp LABEL behavior LINK ggml)
|
||||
|
||||
# Needs non-public ggml{,-backend}-impl.h
|
||||
llama_build_and_test(test-alloc.cpp LABEL "behavior backend" LINK ggml)
|
||||
endif()
|
||||
|
||||
llama_build(test-export-graph-ops.cpp)
|
||||
if (TARGET gguf-model-data)
|
||||
target_link_libraries(test-export-graph-ops PRIVATE gguf-model-data)
|
||||
target_compile_definitions(test-export-graph-ops PRIVATE LLAMA_HF_FETCH)
|
||||
endif()
|
||||
@@ -0,0 +1,292 @@
|
||||
# test-fusion baseline for device MTL
|
||||
# arch ,moe ,mode ,label , count
|
||||
afmoe ,1 ,any ,MUL+ADD , 2
|
||||
afmoe ,1 ,any ,RMS_NORM+MUL , 10
|
||||
afmoe ,1 ,any ,RMS_NORM+MUL+ADD , 3
|
||||
arcee ,0 ,any ,RMS_NORM+MUL , 5
|
||||
arctic ,0 ,any ,MUL+ADD , 4
|
||||
arctic ,0 ,any ,RMS_NORM+MUL , 7
|
||||
arctic ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
baichuan ,0 ,any ,RMS_NORM+MUL , 5
|
||||
bailingmoe ,1 ,any ,ADD+ADD , 2
|
||||
bailingmoe ,1 ,any ,MUL+ADD , 4
|
||||
bailingmoe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
bailingmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2
|
||||
bailingmoe2 ,1 ,any ,ADD+ADD , 1
|
||||
bailingmoe2 ,1 ,any ,MUL+ADD , 2
|
||||
bailingmoe2 ,1 ,any ,RMS_NORM+MUL , 9
|
||||
bailingmoe3 ,1 ,any ,ADD+ADD , 1
|
||||
bailingmoe3 ,1 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
bailingmoe3 ,1 ,any ,MUL+ADD , 2
|
||||
bailingmoe3 ,1 ,any ,RMS_NORM+MUL , 8
|
||||
bailingmoe3 ,1 ,any ,RMS_NORM+SCALE , 2
|
||||
beam ,0 ,any ,MUL+ADD , 6
|
||||
beam ,0 ,any ,RMS_NORM+MUL , 25
|
||||
bloom ,0 ,any ,NORM+MUL+ADD , 6
|
||||
chatglm ,0 ,any ,RMS_NORM+MUL , 5
|
||||
codeshell ,0 ,any ,NORM+MUL+ADD , 5
|
||||
cogvlm ,0 ,any ,RMS_NORM+MUL , 5
|
||||
cohere2 ,0 ,any ,ADD+ADD , 2
|
||||
cohere2 ,0 ,any ,NORM+MUL , 3
|
||||
cohere2moe ,1 ,any ,ADD+ADD , 2
|
||||
cohere2moe ,1 ,any ,MUL+ADD , 2
|
||||
cohere2moe ,1 ,any ,RMS_NORM+MUL , 3
|
||||
command-r ,0 ,any ,NORM+MUL , 3
|
||||
dbrx ,0 ,any ,MUL+ADD , 4
|
||||
dbrx ,0 ,any ,NORM+MUL , 5
|
||||
dbrx ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
deci ,0 ,any ,RMS_NORM+MUL , 5
|
||||
deepseek ,0 ,any ,ADD+ADD , 1
|
||||
deepseek ,0 ,any ,MUL+ADD , 2
|
||||
deepseek ,0 ,any ,RMS_NORM+MUL , 5
|
||||
deepseek ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 1
|
||||
deepseek2 ,0 ,any ,ADD+ADD , 1
|
||||
deepseek2 ,0 ,any ,MUL+ADD , 2
|
||||
deepseek2 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
deepseek32 ,0 ,any ,ADD+ADD , 1
|
||||
deepseek32 ,0 ,any ,MUL+ADD , 2
|
||||
deepseek32 ,0 ,any ,NORM+MUL+ADD , 2
|
||||
deepseek32 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
deepseek4 ,0 ,any ,MUL+ADD , 8
|
||||
deepseek4 ,0 ,any ,RMS_NORM+MUL , 20
|
||||
deepseek41 ,0 ,any ,MUL+ADD , 12
|
||||
deepseek41 ,0 ,any ,RMS_NORM+MUL , 29
|
||||
dots1 ,0 ,any ,ADD+ADD , 1
|
||||
dots1 ,0 ,any ,MUL+ADD , 2
|
||||
dots1 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
dots3note ,0 ,any ,ADD+ADD , 1
|
||||
dots3note ,0 ,any ,MUL+ADD , 2
|
||||
dots3note ,0 ,any ,NORM+MUL+ADD , 1
|
||||
dots3note ,0 ,any ,RMS_NORM+MUL , 11
|
||||
dream ,0 ,any ,RMS_NORM+MUL , 5
|
||||
ernie4_5-moe ,1 ,any ,ADD+ADD , 1
|
||||
ernie4_5-moe ,1 ,any ,MUL+ADD , 2
|
||||
ernie4_5-moe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
ernie4_5 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
exaone ,0 ,any ,RMS_NORM+MUL , 5
|
||||
exaone-moe ,1 ,any ,ADD+ADD , 1
|
||||
exaone-moe ,1 ,any ,MUL+ADD , 2
|
||||
exaone-moe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
exaone4 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
exaone4 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
falcon ,0 ,any ,ADD+ADD , 2
|
||||
falcon ,0 ,any ,NORM+MUL+ADD , 5
|
||||
falcon-h1 ,0 ,any ,ADD+ADD , 2
|
||||
falcon-h1 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
flint ,0 ,any ,ADD+ADD , 1
|
||||
flint ,0 ,any ,MUL+ADD , 2
|
||||
flint ,0 ,any ,RMS_NORM+MUL , 9
|
||||
gemma ,0 ,any ,RMS_NORM+MUL , 5
|
||||
gemma2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
gemma2 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
gemma3 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
gemma3 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
glm-dsa ,0 ,any ,ADD+ADD , 1
|
||||
glm-dsa ,0 ,any ,MUL+ADD , 2
|
||||
glm-dsa ,0 ,any ,NORM+MUL+ADD , 2
|
||||
glm-dsa ,0 ,any ,RMS_NORM+MUL , 9
|
||||
glm4 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
glm4 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
glm4moe ,1 ,any ,ADD+ADD , 1
|
||||
glm4moe ,1 ,any ,MUL+ADD , 2
|
||||
glm4moe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
glm5-next ,0 ,any ,ADD+ADD+ADD , 1
|
||||
glm5-next ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
glm5-next ,0 ,any ,MUL+ADD , 2
|
||||
glm5-next ,0 ,any ,NORM+MUL+ADD , 1
|
||||
glm5-next ,0 ,any ,RMS_NORM+MUL , 8
|
||||
glm5-next ,0 ,any ,RMS_NORM+SCALE , 2
|
||||
gpt-oss ,0 ,any ,MUL+ADD , 4
|
||||
gpt-oss ,0 ,any ,RMS_NORM+MUL , 5
|
||||
gpt2 ,0 ,any ,NORM+MUL+ADD , 5
|
||||
gptneox ,0 ,any ,NORM+MUL+ADD , 5
|
||||
granite ,0 ,any ,RMS_NORM+MUL , 5
|
||||
granite ,0 ,any ,MUL+ADD , 4
|
||||
granite ,0 ,any ,RMS_NORM+MUL , 5
|
||||
granite ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
granite_swa ,0 ,any ,RMS_NORM+MUL , 5
|
||||
granitehybrid ,0 ,any ,RMS_NORM+MUL , 6
|
||||
granitemoe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
granitemoe ,1 ,any ,MUL+ADD , 4
|
||||
granitemoe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
granitemoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
grok ,0 ,any ,MUL+ADD , 4
|
||||
grok ,0 ,any ,RMS_NORM+MUL , 5
|
||||
grok ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
grok ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
grovemoe ,1 ,any ,ADD+ADD , 2
|
||||
grovemoe ,1 ,any ,MUL+ADD , 8
|
||||
grovemoe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
hunyuan-dense ,0 ,any ,RMS_NORM+MUL , 9
|
||||
hunyuan-moe ,1 ,any ,ADD+ADD , 2
|
||||
hunyuan-moe ,1 ,any ,MUL+ADD , 4
|
||||
hunyuan-moe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
hunyuan-moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
hunyuan_vl ,0 ,any ,RMS_NORM+MUL , 9
|
||||
hy_v3 ,0 ,any ,ADD+ADD , 2
|
||||
hy_v3 ,0 ,any ,MUL+ADD , 4
|
||||
hy_v3 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
hy_v4 ,0 ,any ,MUL+ADD , 2
|
||||
hy_v4 ,0 ,any ,NORM+MUL+ADD , 1
|
||||
hy_v4 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
internlm2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
jais ,0 ,any ,NORM+MUL+ADD , 5
|
||||
jais2 ,0 ,any ,NORM+MUL+ADD , 5
|
||||
jamba ,0 ,any ,RMS_NORM+MUL , 8
|
||||
k2-horizon ,0 ,any ,RMS_NORM+MUL , 4
|
||||
k2-horizon ,0 ,any ,ADD+ADD , 1
|
||||
k2-horizon ,0 ,any ,MUL+ADD , 2
|
||||
k2-horizon ,0 ,any ,RMS_NORM+MUL , 4
|
||||
kimi-k3 ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
kimi-k3 ,0 ,any ,MUL+ADD , 2
|
||||
kimi-k3 ,0 ,any ,RMS_NORM+MUL , 17
|
||||
kimi-k3 ,0 ,any ,RMS_NORM+SCALE , 2
|
||||
kimi-linear ,0 ,any ,ADD+ADD , 1
|
||||
kimi-linear ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
kimi-linear ,0 ,any ,MUL+ADD , 2
|
||||
kimi-linear ,0 ,any ,RMS_NORM+MUL , 7
|
||||
kimi-linear ,0 ,any ,RMS_NORM+SCALE , 2
|
||||
kolibri1 ,0 ,any ,MUL+ADD , 4
|
||||
kolibri1 ,0 ,any ,RMS_NORM+MUL , 10
|
||||
kolibri1 ,0 ,any ,RMS_NORM+MUL+ADD , 3
|
||||
laguna ,0 ,any ,ADD+ADD , 1
|
||||
laguna ,0 ,any ,MUL+ADD , 2
|
||||
laguna ,0 ,any ,RMS_NORM+MUL , 9
|
||||
lfm2 ,0 ,any ,RMS_NORM+MUL , 7
|
||||
lfm2moe ,1 ,any ,MUL+ADD , 2
|
||||
lfm2moe ,1 ,any ,RMS_NORM+MUL , 7
|
||||
lfm3 ,0 ,any ,RMS_NORM+MUL , 17
|
||||
llada ,0 ,any ,RMS_NORM+MUL , 5
|
||||
llada-moe ,1 ,any ,MUL+ADD , 4
|
||||
llada-moe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
llada-moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2
|
||||
llama ,0 ,any ,RMS_NORM+MUL , 5
|
||||
llama ,0 ,any ,MUL+ADD , 4
|
||||
llama ,0 ,any ,RMS_NORM+MUL , 5
|
||||
llama ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
llama4 ,0 ,any ,ADD+ADD , 2
|
||||
llama4 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
maincoder ,0 ,any ,RMS_NORM+MUL , 9
|
||||
maion-coder ,0 ,any ,MUL+ADD , 4
|
||||
maion-coder ,0 ,any ,RMS_NORM+MUL , 17
|
||||
maion-coder ,0 ,any ,RMS_NORM+MUL+ADD , 8
|
||||
maion-coder ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
mamba ,0 ,any ,RMS_NORM+MUL , 3
|
||||
mamba2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
maple ,0 ,any ,MUL+ADD , 4
|
||||
maple ,0 ,any ,RMS_NORM+MUL , 9
|
||||
maple ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
mellum ,0 ,any ,MUL+ADD , 4
|
||||
mellum ,0 ,any ,RMS_NORM+MUL , 9
|
||||
mellum ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
mimo2 ,0 ,any ,MUL+ADD , 4
|
||||
mimo2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
minicpm ,0 ,any ,RMS_NORM+MUL , 5
|
||||
minicpm ,0 ,any ,MUL+ADD , 4
|
||||
minicpm ,0 ,any ,RMS_NORM+MUL , 5
|
||||
minicpm ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
minicpm3 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
minimax-01 ,0 ,any ,MUL+ADD , 4
|
||||
minimax-01 ,0 ,any ,RMS_NORM+MUL , 6
|
||||
minimax-01 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
minimax-m2 ,0 ,any ,MUL+ADD , 4
|
||||
minimax-m2 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
minimax-m3 ,0 ,any ,ADD+ADD , 1
|
||||
minimax-m3 ,0 ,any ,MUL+ADD , 2
|
||||
minimax-m3 ,0 ,any ,RMS_NORM+MUL , 11
|
||||
mistral3 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
mistral3 ,0 ,any ,MUL+ADD , 4
|
||||
mistral3 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
mistral3 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
mistral4 ,0 ,any ,ADD+ADD , 1
|
||||
mistral4 ,0 ,any ,MUL+ADD , 2
|
||||
mistral4 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
mpt ,0 ,any ,NORM+MUL+ADD , 5
|
||||
muse-glimmer ,0 ,any ,RMS_NORM+MUL , 10
|
||||
muse-glimmer ,0 ,any ,RMS_NORM+MUL+ADD , 3
|
||||
muse-spark ,0 ,any ,MUL+ADD , 4
|
||||
muse-spark ,0 ,any ,RMS_NORM+MUL , 14
|
||||
nanbeige ,0 ,any ,RMS_NORM+MUL , 5
|
||||
nemotron ,0 ,any ,NORM+MUL+ADD , 5
|
||||
nemotron_h ,0 ,any ,RMS_NORM+MUL , 5
|
||||
nemotron_h_moe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
olmo2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
olmo2 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
olmoe ,1 ,any ,MUL+ADD , 4
|
||||
olmoe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
olmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2
|
||||
openelm ,0 ,any ,RMS_NORM+MUL , 9
|
||||
orion ,0 ,any ,NORM+MUL+ADD , 5
|
||||
paddleocr ,0 ,any ,RMS_NORM+MUL , 5
|
||||
pangu-embedded ,0 ,any ,RMS_NORM+MUL , 5
|
||||
phi2 ,0 ,any ,ADD+ADD , 2
|
||||
phi2 ,0 ,any ,NORM+MUL+ADD , 3
|
||||
phi3 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
phimoe ,1 ,any ,MUL+ADD , 4
|
||||
phimoe ,1 ,any ,RMS_NORM+MUL+ADD , 5
|
||||
phimoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
plamo ,0 ,any ,ADD+ADD , 2
|
||||
plamo ,0 ,any ,RMS_NORM+MUL , 3
|
||||
plamo2 ,0 ,any ,RMS_NORM+MUL , 10
|
||||
plamo2 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
plamo2 ,0 ,any ,SSM_CONV+UNARY , 1
|
||||
plamo3 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
plamo3 ,0 ,any ,RMS_NORM+MUL+ADD , 4
|
||||
pockettts ,0 ,any ,NORM+MUL+ADD , 5
|
||||
qwen ,0 ,any ,RMS_NORM+MUL , 5
|
||||
qwen2 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
qwen2moe ,1 ,any ,ADD+ADD , 2
|
||||
qwen2moe ,1 ,any ,MUL+ADD , 4
|
||||
qwen2moe ,1 ,any ,RMS_NORM+MUL , 5
|
||||
qwen2moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS , 2
|
||||
qwen2vl ,0 ,any ,RMS_NORM+MUL , 5
|
||||
qwen3 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
qwen35 ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
qwen35 ,0 ,any ,RMS_NORM+MUL , 8
|
||||
qwen35 ,0 ,any ,RMS_NORM+SCALE , 2
|
||||
qwen35 ,0 ,any ,SSM_CONV+UNARY , 1
|
||||
qwen35moe ,1 ,any ,ADD+ADD , 2
|
||||
qwen35moe ,1 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
qwen35moe ,1 ,any ,MUL+ADD , 4
|
||||
qwen35moe ,1 ,any ,RMS_NORM+MUL , 8
|
||||
qwen35moe ,1 ,any ,RMS_NORM+SCALE , 2
|
||||
qwen35moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
qwen35moe ,1 ,any ,SSM_CONV+UNARY , 1
|
||||
qwen3moe ,1 ,any ,MUL+ADD , 4
|
||||
qwen3moe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
qwen3moe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
qwen3next ,0 ,any ,ADD+ADD , 2
|
||||
qwen3next ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
qwen3next ,0 ,any ,MUL+ADD , 4
|
||||
qwen3next ,0 ,any ,RMS_NORM+MUL , 8
|
||||
qwen3next ,0 ,any ,RMS_NORM+SCALE , 2
|
||||
qwen3next ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
qwen3next ,0 ,any ,SSM_CONV+UNARY , 1
|
||||
qwen3tts ,0 ,any ,RMS_NORM+MUL , 9
|
||||
qwen3vl ,0 ,any ,RMS_NORM+MUL , 9
|
||||
qwen3vlmoe ,1 ,any ,MUL+ADD , 4
|
||||
qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9
|
||||
qwen3vlmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
qwen4exp ,0 ,any ,ADD+ADD+ADD , 2
|
||||
qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1
|
||||
qwen4exp ,0 ,any ,MUL+ADD , 4
|
||||
qwen4exp ,0 ,any ,RMS_NORM+MUL , 13
|
||||
qwen4exp ,0 ,any ,RMS_NORM+SCALE , 2
|
||||
qwen4exp ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
qwen4exp ,0 ,any ,SSM_CONV+UNARY , 1
|
||||
refact ,0 ,any ,RMS_NORM+MUL , 5
|
||||
refact ,0 ,any ,RMS_NORM+MUL , 5
|
||||
rnd1 ,0 ,any ,MUL+ADD , 4
|
||||
rnd1 ,0 ,any ,RMS_NORM+MUL , 9
|
||||
rnd1 ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
|
||||
seed_oss ,0 ,any ,RMS_NORM+MUL , 5
|
||||
smallthinker ,0 ,any ,MUL+ADD , 4
|
||||
smallthinker ,0 ,any ,RMS_NORM+MUL , 5
|
||||
smollm3 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
spark2_5 ,0 ,any ,RMS_NORM+MUL , 5
|
||||
stablelm ,0 ,any ,NORM+MUL , 4
|
||||
stablelm ,0 ,any ,NORM+MUL+ADD , 5
|
||||
starcoder ,0 ,any ,NORM+MUL+ADD , 5
|
||||
starcoder2 ,0 ,any ,NORM+MUL+ADD , 5
|
||||
talkie ,0 ,any ,ADD+ADD , 2
|
||||
xverse ,0 ,any ,RMS_NORM+MUL , 5
|
||||
|
@@ -0,0 +1,26 @@
|
||||
# Fusion baselines
|
||||
|
||||
Per-device baselines for `test-fusion`, one CSV per backend (e.g. `MTL.csv`). Rows are
|
||||
`arch,moe,mode,label,count`. Regenerate a CSV whenever fusion patterns change.
|
||||
|
||||
## Update a baseline
|
||||
|
||||
```sh
|
||||
cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_METAL=ON # enable the target backend
|
||||
cmake --build build --config Release --target test-llama-archs --target test-fusion -j
|
||||
|
||||
rm -rf build-ci-models && mkdir -p build-ci-models
|
||||
./build/bin/test-llama-archs -o build-ci-models
|
||||
|
||||
./build/bin/test-fusion --models build-ci-models --device MTL0 --record MTL.csv
|
||||
```
|
||||
|
||||
## Validate
|
||||
|
||||
```sh
|
||||
./build/bin/test-fusion --models build-ci-models --device MTL0 --check MTL.csv
|
||||
```
|
||||
|
||||
Non-zero exit means a row differs from the baseline. Use `--model FILE` to run a single
|
||||
architecture. `--check` also reports baseline rows that the run no longer produces, so a fusion
|
||||
that stops matching fails the run instead of silently dropping out of the CSV.
|
||||
@@ -0,0 +1,743 @@
|
||||
// GGUF binary parser adapted from the huggingface/gguf package.
|
||||
// Reference: https://github.com/huggingface/huggingface.js
|
||||
|
||||
#include "gguf-model-data.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "ggml-cpp.h"
|
||||
#include "gguf.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
|
||||
#include "http.h"
|
||||
#define JSON_ASSERT GGML_ASSERT
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
// Equivalent of RangeView
|
||||
struct gguf_buf_reader {
|
||||
const char * data;
|
||||
size_t size;
|
||||
size_t pos;
|
||||
|
||||
gguf_buf_reader(const std::vector<char> & buf) : data(buf.data()), size(buf.size()), pos(0) {}
|
||||
|
||||
bool has_n_bytes(size_t n) const {
|
||||
return pos + n <= size;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool read_val(T & out) {
|
||||
if (!has_n_bytes(sizeof(T))) {
|
||||
return false;
|
||||
}
|
||||
memcpy(&out, data + pos, sizeof(T));
|
||||
pos += sizeof(T);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool read_str(std::string & out) {
|
||||
uint64_t len;
|
||||
if (!read_val(len)) {
|
||||
return false;
|
||||
}
|
||||
if (!has_n_bytes((size_t)len)) {
|
||||
return false;
|
||||
}
|
||||
out.assign(data + pos, (size_t)len);
|
||||
pos += (size_t)len;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool skip(size_t n) {
|
||||
if (!has_n_bytes(n)) {
|
||||
return false;
|
||||
}
|
||||
pos += n;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
static size_t gguf_val_type_size(int32_t vtype) {
|
||||
switch (vtype) {
|
||||
case GGUF_TYPE_UINT8: return 1;
|
||||
case GGUF_TYPE_INT8: return 1;
|
||||
case GGUF_TYPE_UINT16: return 2;
|
||||
case GGUF_TYPE_INT16: return 2;
|
||||
case GGUF_TYPE_UINT32: return 4;
|
||||
case GGUF_TYPE_INT32: return 4;
|
||||
case GGUF_TYPE_FLOAT32: return 4;
|
||||
case GGUF_TYPE_BOOL: return 1;
|
||||
case GGUF_TYPE_UINT64: return 8;
|
||||
case GGUF_TYPE_INT64: return 8;
|
||||
case GGUF_TYPE_FLOAT64: return 8;
|
||||
default: return 0; // string/array handled separately
|
||||
}
|
||||
}
|
||||
|
||||
// Equivalent of readMetadataValue(), skips unused values rather than storing
|
||||
static bool gguf_skip_value(gguf_buf_reader & r, int32_t vtype) {
|
||||
if (vtype == GGUF_TYPE_STRING) {
|
||||
std::string tmp;
|
||||
return r.read_str(tmp);
|
||||
}
|
||||
if (vtype == GGUF_TYPE_ARRAY) {
|
||||
int32_t elem_type;
|
||||
uint64_t count;
|
||||
if (!r.read_val(elem_type)) {
|
||||
return false;
|
||||
}
|
||||
if (!r.read_val(count)) {
|
||||
return false;
|
||||
}
|
||||
if (elem_type == GGUF_TYPE_STRING) {
|
||||
for (uint64_t i = 0; i < count; i++) {
|
||||
std::string tmp;
|
||||
if (!r.read_str(tmp)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (elem_type == GGUF_TYPE_ARRAY) {
|
||||
// nested arrays - recurse
|
||||
for (uint64_t i = 0; i < count; i++) {
|
||||
if (!gguf_skip_value(r, GGUF_TYPE_ARRAY)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
size_t elem_sz = gguf_val_type_size(elem_type);
|
||||
if (elem_sz == 0) {
|
||||
return false;
|
||||
}
|
||||
return r.skip((size_t)count * elem_sz);
|
||||
}
|
||||
size_t sz = gguf_val_type_size(vtype);
|
||||
if (sz == 0) {
|
||||
return false;
|
||||
}
|
||||
return r.skip(sz);
|
||||
}
|
||||
|
||||
static bool gguf_read_uint32_val(gguf_buf_reader & r, int32_t vtype, uint32_t & out) {
|
||||
// Handle array-valued fields (e.g. per-layer head counts in hybrid models)
|
||||
// by reading the first element as a representative value.
|
||||
if (vtype == GGUF_TYPE_ARRAY) {
|
||||
int32_t elem_type;
|
||||
uint64_t count;
|
||||
if (!r.read_val(elem_type)) {
|
||||
return false;
|
||||
}
|
||||
if (!r.read_val(count)) {
|
||||
return false;
|
||||
}
|
||||
if (count == 0) {
|
||||
return false;
|
||||
}
|
||||
// Read first element, skip the rest
|
||||
if (!gguf_read_uint32_val(r, elem_type, out)) {
|
||||
return false;
|
||||
}
|
||||
for (uint64_t i = 1; i < count; i++) {
|
||||
size_t sz = gguf_val_type_size(elem_type);
|
||||
if (sz == 0) {
|
||||
return false;
|
||||
}
|
||||
if (!r.skip(sz)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (vtype == GGUF_TYPE_UINT8) {
|
||||
uint8_t v;
|
||||
if (!r.read_val(v)) {
|
||||
return false;
|
||||
}
|
||||
out = v;
|
||||
return true;
|
||||
}
|
||||
if (vtype == GGUF_TYPE_INT8) {
|
||||
int8_t v;
|
||||
if (!r.read_val(v)) {
|
||||
return false;
|
||||
}
|
||||
out = (uint32_t)v;
|
||||
return true;
|
||||
}
|
||||
if (vtype == GGUF_TYPE_UINT16) {
|
||||
uint16_t v;
|
||||
if (!r.read_val(v)) {
|
||||
return false;
|
||||
}
|
||||
out = v;
|
||||
return true;
|
||||
}
|
||||
if (vtype == GGUF_TYPE_INT16) {
|
||||
int16_t v;
|
||||
if (!r.read_val(v)) {
|
||||
return false;
|
||||
}
|
||||
out = (uint32_t)v;
|
||||
return true;
|
||||
}
|
||||
if (vtype == GGUF_TYPE_UINT32) {
|
||||
uint32_t v;
|
||||
if (!r.read_val(v)) {
|
||||
return false;
|
||||
}
|
||||
out = v;
|
||||
return true;
|
||||
}
|
||||
if (vtype == GGUF_TYPE_INT32) {
|
||||
int32_t v;
|
||||
if (!r.read_val(v)) {
|
||||
return false;
|
||||
}
|
||||
out = (uint32_t)v;
|
||||
return true;
|
||||
}
|
||||
if (vtype == GGUF_TYPE_UINT64) {
|
||||
uint64_t v;
|
||||
if (!r.read_val(v)) {
|
||||
return false;
|
||||
}
|
||||
out = (uint32_t)v;
|
||||
return true;
|
||||
}
|
||||
if (vtype == GGUF_TYPE_INT64) {
|
||||
int64_t v;
|
||||
if (!r.read_val(v)) {
|
||||
return false;
|
||||
}
|
||||
out = (uint32_t)v;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Follows the same header -> KV -> tensor parsing sequence as gguf() huggingface/gguf
|
||||
static std::optional<gguf_remote_model> gguf_parse_meta(const std::vector<char> & buf) {
|
||||
gguf_buf_reader r(buf);
|
||||
|
||||
// Header: magic(4) + version(4) + tensor_count(8) + kv_count(8) = 24 bytes minimum
|
||||
uint32_t magic_raw;
|
||||
if (!r.read_val(magic_raw)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
if (memcmp(&magic_raw, "GGUF", 4) != 0) {
|
||||
fprintf(stderr, "gguf_parse_meta: invalid magic\n");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
uint32_t version;
|
||||
if (!r.read_val(version)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
if (version < 2 || version > 3) {
|
||||
fprintf(stderr, "gguf_parse_meta: unsupported version %u\n", version);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
int64_t tensor_count_raw;
|
||||
int64_t kv_count_raw;
|
||||
if (!r.read_val(tensor_count_raw)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
if (!r.read_val(kv_count_raw)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
uint64_t tensor_count = (uint64_t)tensor_count_raw;
|
||||
uint64_t kv_count = (uint64_t)kv_count_raw;
|
||||
|
||||
gguf_remote_model model;
|
||||
|
||||
std::string arch_prefix;
|
||||
|
||||
// Parse KV pairs
|
||||
for (uint64_t i = 0; i < kv_count; i++) {
|
||||
std::string key;
|
||||
if (!r.read_str(key)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
int32_t vtype;
|
||||
if (!r.read_val(vtype)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
if (key == "general.architecture" && vtype == GGUF_TYPE_STRING) {
|
||||
if (!r.read_str(model.architecture)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
arch_prefix = model.architecture + ".";
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract split.count for proper handling of split files
|
||||
if (key == "split.count") {
|
||||
uint32_t v;
|
||||
if (!gguf_read_uint32_val(r, vtype, v)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
model.n_split = (uint16_t)v;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract split.tensors.count so we can verify we have all tensors
|
||||
if (key == "split.tensors.count") {
|
||||
uint32_t v;
|
||||
if (!gguf_read_uint32_val(r, vtype, v)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
model.n_split_tensors = v;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!arch_prefix.empty()) {
|
||||
uint32_t * target = nullptr;
|
||||
|
||||
if (key == arch_prefix + "embedding_length") { target = &model.n_embd; }
|
||||
else if (key == arch_prefix + "feed_forward_length") { target = &model.n_ff; }
|
||||
else if (key == arch_prefix + "block_count") { target = &model.n_layer; }
|
||||
else if (key == arch_prefix + "attention.head_count") { target = &model.n_head; }
|
||||
else if (key == arch_prefix + "attention.head_count_kv") { target = &model.n_head_kv; }
|
||||
else if (key == arch_prefix + "expert_count") { target = &model.n_expert; }
|
||||
else if (key == arch_prefix + "attention.key_length") { target = &model.n_embd_head_k; }
|
||||
else if (key == arch_prefix + "attention.value_length") { target = &model.n_embd_head_v; }
|
||||
|
||||
if (target) {
|
||||
if (!gguf_read_uint32_val(r, vtype, *target)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (!gguf_skip_value(r, vtype)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
// Parse tensor info entries
|
||||
model.tensors.reserve((size_t)tensor_count);
|
||||
for (uint64_t i = 0; i < tensor_count; i++) {
|
||||
gguf_remote_tensor t;
|
||||
|
||||
if (!r.read_str(t.name)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
if (!r.read_val(t.n_dims)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
if (t.n_dims > 4) {
|
||||
fprintf(stderr, "gguf_parse_meta: tensor '%s' has %u dims (max 4)\n", t.name.c_str(), t.n_dims);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
for (uint32_t d = 0; d < t.n_dims; d++) {
|
||||
if (!r.read_val(t.ne[d])) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
int32_t type_raw;
|
||||
if (!r.read_val(type_raw)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
t.type = (ggml_type)type_raw;
|
||||
|
||||
uint64_t offset;
|
||||
if (!r.read_val(offset)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// Infer n_vocab from token_embd.weight
|
||||
if (t.name == "token_embd.weight") {
|
||||
model.n_vocab = (uint32_t)t.ne[1];
|
||||
}
|
||||
|
||||
model.tensors.push_back(std::move(t));
|
||||
}
|
||||
|
||||
return model;
|
||||
}
|
||||
|
||||
// cache handling for local download
|
||||
static std::filesystem::path get_default_cache_dir() {
|
||||
return fs_get_cache_directory() / "gguf-headers";
|
||||
}
|
||||
|
||||
static std::string sanitize_for_path(const std::string & s) {
|
||||
std::string out = s;
|
||||
for (char & c : out) {
|
||||
if (c == '/' || c == '\\' || c == ':') {
|
||||
c = '_';
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static bool read_file(const std::filesystem::path & path, std::vector<char> & out) {
|
||||
std::ifstream f(path, std::ios::binary | std::ios::ate);
|
||||
if (!f.good()) {
|
||||
return false;
|
||||
}
|
||||
auto sz = f.tellg();
|
||||
if (sz <= 0) {
|
||||
return false;
|
||||
}
|
||||
out.resize((size_t)sz);
|
||||
f.seekg(0);
|
||||
f.read(out.data(), sz);
|
||||
return f.good();
|
||||
}
|
||||
|
||||
static bool write_file(const std::filesystem::path & path, const std::vector<char> & data) {
|
||||
std::ofstream f(path, std::ios::binary | std::ios::trunc);
|
||||
if (!f.good()) {
|
||||
return false;
|
||||
}
|
||||
f.write(data.data(), (std::streamsize)data.size());
|
||||
return f.good();
|
||||
}
|
||||
|
||||
// HuggingFace file auto-detection and HTTP download
|
||||
static std::pair<long, std::vector<char>> gguf_http_get(
|
||||
const std::string & url,
|
||||
const httplib::Headers & headers = {},
|
||||
int timeout_sec = 60) {
|
||||
try {
|
||||
auto [cli, parts] = common_http_client(url);
|
||||
|
||||
if (timeout_sec > 0) {
|
||||
cli.set_read_timeout(timeout_sec, 0);
|
||||
cli.set_write_timeout(timeout_sec, 0);
|
||||
}
|
||||
cli.set_connection_timeout(30, 0);
|
||||
|
||||
std::vector<char> body;
|
||||
auto res = cli.Get(parts.path, headers,
|
||||
[&](const char * data, size_t len) {
|
||||
body.insert(body.end(), data, data + len);
|
||||
return true;
|
||||
}, nullptr);
|
||||
|
||||
if (!res) {
|
||||
fprintf(stderr, "gguf_fetch: HTTP request failed for %s (error %d)\n",
|
||||
url.c_str(), (int)res.error());
|
||||
return {-1, {}};
|
||||
}
|
||||
return {res->status, std::move(body)};
|
||||
} catch (const std::exception & e) {
|
||||
fprintf(stderr, "gguf_fetch: HTTP error: %s\n", e.what());
|
||||
return {-1, {}};
|
||||
}
|
||||
}
|
||||
|
||||
// Find the filename for given repo/quant.
|
||||
// For split models, returns the first shard (the one containing "00001-of-")
|
||||
// split_prefix is set to the portion before "-00001-of-XXXXX.gguf" when a split file is found
|
||||
static std::string detect_gguf_filename(const std::string & repo, const std::string & quant,
|
||||
std::string & split_prefix) {
|
||||
split_prefix.clear();
|
||||
std::string api_url = "https://huggingface.co/api/models/" + repo;
|
||||
|
||||
auto [code, body] = gguf_http_get(api_url, {}, 30);
|
||||
if (code != 200 || body.empty()) {
|
||||
fprintf(stderr, "gguf_fetch: failed to query HF API for %s (HTTP %ld)\n", repo.c_str(), code);
|
||||
return "";
|
||||
}
|
||||
|
||||
nlohmann::json j;
|
||||
try {
|
||||
j = nlohmann::json::parse(body.begin(), body.end());
|
||||
} catch (...) {
|
||||
fprintf(stderr, "gguf_fetch: failed to parse HF API response\n");
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!j.contains("siblings") || !j["siblings"].is_array()) {
|
||||
fprintf(stderr, "gguf_fetch: unexpected HF API response format\n");
|
||||
return "";
|
||||
}
|
||||
|
||||
std::vector<std::string> matches;
|
||||
std::string quant_upper = quant;
|
||||
for (char & c : quant_upper) { c = (char)toupper(c); }
|
||||
|
||||
for (const auto & sibling : j["siblings"]) {
|
||||
if (!sibling.contains("rfilename")) { continue; }
|
||||
std::string fname = sibling["rfilename"].get<std::string>();
|
||||
if (fname.size() < 5 || fname.substr(fname.size() - 5) != ".gguf") {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string fname_upper = fname;
|
||||
for (char & c : fname_upper) { c = (char)toupper(c); }
|
||||
if (fname_upper.find(quant_upper) != std::string::npos) {
|
||||
matches.push_back(fname);
|
||||
}
|
||||
}
|
||||
|
||||
if (matches.empty()) {
|
||||
fprintf(stderr, "gguf_fetch: no .gguf files matching '%s' in %s\n", quant.c_str(), repo.c_str());
|
||||
return "";
|
||||
}
|
||||
|
||||
std::sort(matches.begin(), matches.end());
|
||||
|
||||
// Prefer non-split, non-supplementary file
|
||||
for (const auto & m : matches) {
|
||||
if (m.find("-of-") == std::string::npos && m.find("mmproj") == std::string::npos) {
|
||||
return m;
|
||||
}
|
||||
}
|
||||
|
||||
// Return the first shard (00001-of-) and extract the prefix
|
||||
for (const auto & m : matches) {
|
||||
auto pos = m.find("-00001-of-");
|
||||
if (pos != std::string::npos) {
|
||||
split_prefix = m.substr(0, pos);
|
||||
return m;
|
||||
}
|
||||
}
|
||||
|
||||
return matches[0];
|
||||
}
|
||||
|
||||
static std::optional<gguf_remote_model> fetch_and_parse(
|
||||
const std::string & repo,
|
||||
const std::string & filename,
|
||||
const std::filesystem::path & cache_path,
|
||||
bool verbose) {
|
||||
std::string url = "https://huggingface.co/" + repo + "/resolve/main/" + filename;
|
||||
|
||||
// Progressive download inspired by RangeView.fetchChunk()
|
||||
// Start at 2MB, double each time, cap at 64MB
|
||||
size_t chunk_size = 2 * 1024 * 1024;
|
||||
const size_t max_chunk = 64 * 1024 * 1024;
|
||||
|
||||
while (chunk_size <= max_chunk) {
|
||||
if (verbose) {
|
||||
fprintf(stderr, "gguf_fetch: downloading %zu bytes from %s\n", chunk_size, filename.c_str());
|
||||
}
|
||||
|
||||
char range_buf[64];
|
||||
snprintf(range_buf, sizeof(range_buf), "bytes=0-%zu", chunk_size - 1);
|
||||
httplib::Headers headers = {{"Range", range_buf}};
|
||||
|
||||
auto [code, body] = gguf_http_get(url, headers, 120);
|
||||
if (code != 200 && code != 206) {
|
||||
fprintf(stderr, "gguf_fetch: HTTP %ld fetching %s\n", code, url.c_str());
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
if (body.empty()) {
|
||||
fprintf(stderr, "gguf_fetch: empty response\n");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto result = gguf_parse_meta(body);
|
||||
if (result.has_value()) {
|
||||
write_file(cache_path, body);
|
||||
return result;
|
||||
}
|
||||
|
||||
if (code == 200) {
|
||||
fprintf(stderr, "gguf_fetch: server returned full response but metadata parse failed\n");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// Parse failed, try larger chunk
|
||||
chunk_size *= 2;
|
||||
}
|
||||
|
||||
fprintf(stderr, "gguf_fetch: metadata exceeds 64MB, giving up\n");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
static std::filesystem::path get_cache_file_path(const std::filesystem::path & cdir, const std::string & repo_part, const std::string & filename) {
|
||||
std::string fname_part = sanitize_for_path(filename);
|
||||
return cdir / std::filesystem::u8path(repo_part + "--" + fname_part + ".partial");
|
||||
}
|
||||
|
||||
// Try cache first, then fetch and parse a single GGUF shard.
|
||||
static std::optional<gguf_remote_model> fetch_or_cached(
|
||||
const std::string & repo,
|
||||
const std::string & filename,
|
||||
const std::filesystem::path & cdir,
|
||||
const std::string & repo_part,
|
||||
bool verbose) {
|
||||
std::filesystem::path cache_path = get_cache_file_path(cdir, repo_part, filename);
|
||||
|
||||
{
|
||||
std::vector<char> cached;
|
||||
if (std::filesystem::exists(cache_path) && read_file(cache_path, cached)) {
|
||||
auto result = gguf_parse_meta(cached);
|
||||
if (result.has_value()) {
|
||||
if (verbose) {
|
||||
fprintf(stderr, "gguf_fetch: loaded from cache: %s\n", fs_path_to_utf8(cache_path).c_str());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::error_code ec;
|
||||
std::filesystem::create_directories(cdir, ec);
|
||||
return fetch_and_parse(repo, filename, cache_path, verbose);
|
||||
}
|
||||
|
||||
std::optional<gguf_remote_model> gguf_fetch_model_meta(
|
||||
const std::string & repo,
|
||||
const std::string & quant,
|
||||
const std::string & cache_dir,
|
||||
bool verbose) {
|
||||
const std::filesystem::path cdir = cache_dir.empty() ? get_default_cache_dir() : std::filesystem::u8path(cache_dir);
|
||||
std::string repo_part = sanitize_for_path(repo);
|
||||
|
||||
std::string split_prefix;
|
||||
std::string filename = detect_gguf_filename(repo, quant, split_prefix);
|
||||
if (filename.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto model_opt = fetch_or_cached(repo, filename, cdir, repo_part, verbose);
|
||||
if (!model_opt.has_value()) {
|
||||
fprintf(stderr, "gguf_fetch: failed to fetch %s\n", filename.c_str());
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto & model = model_opt.value();
|
||||
|
||||
// If the model is split across multiple files we need to fetch the remaining shards metadata
|
||||
if (model.n_split > 1) {
|
||||
if (split_prefix.empty()) {
|
||||
fprintf(stderr, "gguf_fetch: model reports %u splits but filename has no split pattern\n", model.n_split);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
fprintf(stderr, "gguf_fetch: split model with %u shards, fetching remaining %u...\n",
|
||||
model.n_split, model.n_split - 1);
|
||||
}
|
||||
|
||||
for (int i = 2; i <= model.n_split; i++) {
|
||||
char buf_num[32];
|
||||
char buf_tot[32];
|
||||
snprintf(buf_num, sizeof(buf_num), "%05d", i);
|
||||
snprintf(buf_tot, sizeof(buf_tot), "%05d", (int)model.n_split);
|
||||
std::string shard_name = split_prefix + "-" + buf_num + "-of-" + buf_tot + ".gguf";
|
||||
|
||||
auto shard = fetch_or_cached(repo, shard_name, cdir, repo_part, verbose);
|
||||
if (!shard.has_value()) {
|
||||
fprintf(stderr, "gguf_fetch: failed to fetch shard %d: %s\n", i, shard_name.c_str());
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
model.tensors.insert(model.tensors.end(),
|
||||
std::make_move_iterator(shard->tensors.begin()),
|
||||
std::make_move_iterator(shard->tensors.end()));
|
||||
}
|
||||
|
||||
if (model.n_split_tensors > 0 && model.tensors.size() != model.n_split_tensors) {
|
||||
fprintf(stderr, "gguf_fetch: WARNING: expected %u tensors from split.tensors.count, got %zu\n",
|
||||
model.n_split_tensors, model.tensors.size());
|
||||
}
|
||||
}
|
||||
|
||||
return model_opt;
|
||||
}
|
||||
|
||||
gguf_context_ptr gguf_fetch_gguf_ctx(
|
||||
const std::string & repo,
|
||||
const std::string & quant,
|
||||
const std::string & cache_dir,
|
||||
bool verbose) {
|
||||
const std::filesystem::path cdir = cache_dir.empty() ? get_default_cache_dir() : std::filesystem::u8path(cache_dir);
|
||||
std::string repo_part = sanitize_for_path(repo);
|
||||
|
||||
std::string split_prefix;
|
||||
std::string filename = detect_gguf_filename(repo, quant, split_prefix);
|
||||
|
||||
if (filename.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto model_opt = fetch_or_cached(repo, filename, cdir, repo_part, verbose);
|
||||
if (!model_opt.has_value()) {
|
||||
fprintf(stderr, "gguf_fetch: failed to fetch %s\n", filename.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto & model = model_opt.value();
|
||||
|
||||
const std::filesystem::path cache_path = get_cache_file_path(cdir, repo_part, filename);
|
||||
|
||||
ggml_context_ptr ggml_ctx_ptr;
|
||||
ggml_context * ggml_ctx{};
|
||||
gguf_init_params params{true, &ggml_ctx};
|
||||
gguf_context_ptr ctx{gguf_init_from_file(fs_path_to_utf8(cache_path).c_str(), params)};
|
||||
ggml_ctx_ptr.reset(ggml_ctx);
|
||||
|
||||
if (ctx == nullptr) {
|
||||
fprintf(stderr, "gguf_fetch: gguf_init_from_file failed\n");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// If the model is split across multiple files we need to fetch the remaining shards metadata
|
||||
if (model.n_split > 1) {
|
||||
if (split_prefix.empty()) {
|
||||
fprintf(stderr, "gguf_fetch: model reports %u splits but filename has no split pattern\n", model.n_split);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
fprintf(stderr, "gguf_fetch: split model with %u shards, fetching remaining %u...\n",
|
||||
model.n_split, model.n_split - 1);
|
||||
}
|
||||
|
||||
for (int i = 2; i <= model.n_split; i++) {
|
||||
char buf_num[32];
|
||||
char buf_tot[32];
|
||||
snprintf(buf_num, sizeof(buf_num), "%05d", i);
|
||||
snprintf(buf_tot, sizeof(buf_tot), "%05d", (int)model.n_split);
|
||||
std::string shard_name = split_prefix + "-" + buf_num + "-of-" + buf_tot + ".gguf";
|
||||
|
||||
auto shard = fetch_or_cached(repo, shard_name, cdir, repo_part, verbose);
|
||||
if (!shard.has_value()) {
|
||||
fprintf(stderr, "gguf_fetch: failed to fetch shard %d: %s\n", i, shard_name.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Load tensors from shard and add to main gguf_context
|
||||
const std::filesystem::path shard_path = get_cache_file_path(cdir, repo_part, shard_name);
|
||||
ggml_context_ptr shard_ggml_ctx_ptr;
|
||||
ggml_context * shard_ggml_ctx{};
|
||||
gguf_init_params shard_params{true, &shard_ggml_ctx};
|
||||
gguf_context_ptr shard_ctx{gguf_init_from_file(fs_path_to_utf8(shard_path).c_str(), shard_params)};
|
||||
shard_ggml_ctx_ptr.reset(shard_ggml_ctx);
|
||||
|
||||
if (shard_ctx == nullptr) {
|
||||
fprintf(stderr, "gguf_fetch: shard gguf_init_from_file failed\n");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(shard_ggml_ctx); t; t = ggml_get_next_tensor(shard_ggml_ctx, t)) {
|
||||
gguf_add_tensor(ctx.get(), t);
|
||||
}
|
||||
}
|
||||
|
||||
gguf_set_val_u16(ctx.get(), "split.count", 1);
|
||||
}
|
||||
|
||||
return ctx;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
#pragma once
|
||||
|
||||
#include "ggml-cpp.h"
|
||||
#include "gguf.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct gguf_remote_tensor {
|
||||
std::string name;
|
||||
ggml_type type = GGML_TYPE_F32;
|
||||
int64_t ne[4] = {1, 1, 1, 1}; // dimensions, unused dims = 1
|
||||
uint32_t n_dims = 0;
|
||||
};
|
||||
|
||||
struct gguf_remote_model {
|
||||
// Selected KV metadata
|
||||
std::string architecture; // general.architecture
|
||||
uint32_t n_embd = 0; // <arch>.embedding_length
|
||||
uint32_t n_ff = 0; // <arch>.feed_forward_length
|
||||
uint32_t n_vocab = 0; // inferred from token_embd.weight ne[1]
|
||||
uint32_t n_layer = 0; // <arch>.block_count
|
||||
uint32_t n_head = 0; // <arch>.attention.head_count
|
||||
uint32_t n_head_kv = 0; // <arch>.attention.head_count_kv
|
||||
uint32_t n_expert = 0; // <arch>.expert_count (0 if absent)
|
||||
uint32_t n_embd_head_k = 0; // <arch>.attention.key_length
|
||||
uint32_t n_embd_head_v = 0; // <arch>.attention.value_length
|
||||
uint16_t n_split = 0; // split.count (0 = not split)
|
||||
uint32_t n_split_tensors = 0; // split.tensors.count (0 if not split)
|
||||
|
||||
std::vector<gguf_remote_tensor> tensors;
|
||||
};
|
||||
|
||||
// Fetch model metadata from HuggingFace with local caching.
|
||||
// repo: e.g., "ggml-org/Qwen3-32B-GGUF"
|
||||
// quant: e.g., "Q8_0" -- auto-detects filename (including first shard of split models)
|
||||
// Returns nullopt if download fails or network is unavailable.
|
||||
std::optional<gguf_remote_model> gguf_fetch_model_meta(
|
||||
const std::string & repo,
|
||||
const std::string & quant = "Q8_0",
|
||||
const std::string & cache_dir = "", // empty = default
|
||||
bool verbose = true);
|
||||
|
||||
gguf_context_ptr gguf_fetch_gguf_ctx(
|
||||
const std::string & repo,
|
||||
const std::string & quant = "Q8_0",
|
||||
const std::string & cache_dir = "",
|
||||
bool verbose = true);
|
||||
@@ -0,0 +1,787 @@
|
||||
// KV cache state cases
|
||||
//
|
||||
#include "test-case.h"
|
||||
|
||||
#include "test-log.h"
|
||||
#include "test-util.h"
|
||||
|
||||
#include "common.h"
|
||||
|
||||
#include "../../src/llama-ext.h" // llama_model_set_attn_rot
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <iterator>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
//
|
||||
// reference run
|
||||
//
|
||||
|
||||
// - decode all but the last token, saving the state to disk
|
||||
// - decode the last token
|
||||
// - generate n_predict tokens
|
||||
// the logits it produces are the reference of every case with needs_baseline()
|
||||
struct case_baseline : ctx_test {
|
||||
const char * name() const override { return "baseline"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "reference generation over the whole prompt, saves the state before the last token";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
ctx_opts opts;
|
||||
opts.n_seq_max = 2;
|
||||
|
||||
auto ctx = mr.ctx(opts);
|
||||
auto smpl = mr.sampler();
|
||||
if (!ctx) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
auto n_past = 0;
|
||||
if (!common_prompt_batch_decode(ctx.get(), mr.tokens, (int) mr.tokens.size(), n_past, mr.params.n_batch, mr.params.out_file, true)) {
|
||||
TLOG_ERR("%s: failed to decode the prompt\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
generation_result result;
|
||||
if (!generate_tokens(ctx.get(), smpl.get(), n_past, mr.params.n_predict, 0, result)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
if (result.empty()) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
mr.baseline = std::move(result);
|
||||
mr.have_baseline = true;
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// sequences
|
||||
//
|
||||
|
||||
// - decode the same prefix into two sequences
|
||||
// - remove sequence 0
|
||||
// - verify that sequence 1 remains unchanged
|
||||
struct case_seq_rm_isolated : ctx_test {
|
||||
const char * name() const override { return "seq_rm"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "removing one sequence leaves the state of the other sequences untouched";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
ctx_opts opts;
|
||||
opts.n_ctx = 256;
|
||||
opts.n_seq_max = 2;
|
||||
opts.kv_unified = 1;
|
||||
|
||||
auto ctx = mr.ctx(opts);
|
||||
if (!ctx) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const size_t n_tokens = std::min<size_t>(mr.tokens.size(), 128);
|
||||
for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) {
|
||||
if (!decode_seq(ctx.get(), mr.tokens, 0, (llama_pos) n_tokens, seq_id, true)) {
|
||||
TLOG_ERR("%s: failed to decode the prompt for sequence %d\n", __func__, seq_id);
|
||||
return case_result::fail();
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<uint8_t> before;
|
||||
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, before)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
if (!seq_rm(ctx.get(), 0)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
std::vector<uint8_t> after;
|
||||
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, after)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
CTX_REQUIRE(blobs_equal(before, after), "removing seq 0 changed the state of seq 1");
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
// - load the state saved by the baseline case
|
||||
// - replay its last token
|
||||
// - generate again and compare against the baseline logits
|
||||
struct case_state_load : ctx_test {
|
||||
const char * name() const override { return "state_load"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "a saved session replays to the same logits as the run that saved it";
|
||||
}
|
||||
|
||||
bool needs_baseline() const override { return true; }
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
ctx_opts opts;
|
||||
opts.n_seq_max = 2;
|
||||
|
||||
auto ctx = mr.ctx(opts);
|
||||
auto smpl = mr.sampler();
|
||||
if (!ctx) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
int n_past = 0;
|
||||
if (!load_state_replay_last(ctx.get(), mr.params.out_file, mr.tokens, n_past)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
if (!generate_tokens_compare(ctx.get(), smpl.get(), n_past, mr.params.n_predict, 0, mr.baseline, NMSE_THRESHOLD)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
// - load the state saved by the baseline case and replay its last token
|
||||
// - copy sequence 0 to sequence 1 over the given io path, clearing the cache in between
|
||||
// - continue on sequence 1 and compare against the baseline logits
|
||||
struct case_seq_cp : ctx_test {
|
||||
explicit case_seq_cp(bool on_device_) : on_device(on_device_) {}
|
||||
|
||||
const char * name() const override { return on_device ? "cp_d" : "cp_h"; }
|
||||
|
||||
const char * description() const override {
|
||||
return on_device ? "copying a sequence over the device io path keeps its state"
|
||||
: "copying a sequence over the host io path keeps its state";
|
||||
}
|
||||
|
||||
bool needs_baseline() const override { return true; }
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
ctx_opts opts;
|
||||
opts.n_seq_max = 2;
|
||||
|
||||
auto ctx = mr.ctx(opts);
|
||||
auto smpl = mr.sampler();
|
||||
if (!ctx) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const uint32_t flags = on_device ? LLAMA_STATE_SEQ_FLAGS_ON_DEVICE : LLAMA_STATE_SEQ_FLAGS_NONE;
|
||||
TLOG_INF("io path: %s\n", on_device ? "device" : "host");
|
||||
|
||||
int n_past = 0;
|
||||
if (!load_state_replay_last(ctx.get(), mr.params.out_file, mr.tokens, n_past)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
{
|
||||
std::vector<uint8_t> seq_store;
|
||||
if (!get_seq_state(ctx.get(), 0, flags, seq_store)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
TLOG_TRC("%s: seq 0 copied, %zu bytes\n", __func__, seq_store.size());
|
||||
|
||||
llama_memory_clear(llama_get_memory(ctx.get()), true);
|
||||
TLOG_TRC("%s: kv cache cleared\n", __func__);
|
||||
|
||||
if (!set_seq_state(ctx.get(), 1, flags, seq_store)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
TLOG_TRC("%s: seq 1 restored, %zu bytes\n", __func__, seq_store.size());
|
||||
}
|
||||
|
||||
if (!generate_tokens_compare(ctx.get(), smpl.get(), n_past, mr.params.n_predict, 1, mr.baseline, NMSE_THRESHOLD)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
|
||||
private:
|
||||
bool on_device;
|
||||
};
|
||||
|
||||
// - decode the same prefix on two sequences, interleaving seq 0 cells between the seq 1 cells
|
||||
// - save the seq 1 state, free the interleaved seq 0 cells, and restore via the given io path
|
||||
// - the restore destination is non-contiguous: scatter reads are batched per contiguous run
|
||||
// - save again on the host and compare the two blobs byte for byte
|
||||
struct case_seq_cp_scatter : ctx_test {
|
||||
explicit case_seq_cp_scatter(bool on_device_) : on_device(on_device_) {}
|
||||
|
||||
const char * name() const override { return on_device ? "cp_d_s" : "cp_h_s"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "saving and restoring a sequence with a fragmented (scattered) cell range";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
ctx_opts opts;
|
||||
opts.n_ctx = 256;
|
||||
opts.n_seq_max = 2;
|
||||
opts.kv_unified = 1;
|
||||
|
||||
auto ctx = mr.ctx(opts);
|
||||
if (!ctx) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const uint32_t flags = on_device ? LLAMA_STATE_SEQ_FLAGS_ON_DEVICE : LLAMA_STATE_SEQ_FLAGS_NONE;
|
||||
TLOG_INF("io path: %s\n", on_device ? "device" : "host");
|
||||
|
||||
// seq 0 cells 0,1,4 interleave the seq 1 cells 2,3,5
|
||||
if (!decode_one(ctx.get(), mr.tokens[0], 0, 0) ||
|
||||
!decode_one(ctx.get(), mr.tokens[1], 1, 0) ||
|
||||
!decode_one(ctx.get(), mr.tokens[0], 0, 1) ||
|
||||
!decode_one(ctx.get(), mr.tokens[1], 1, 1) ||
|
||||
!decode_one(ctx.get(), mr.tokens[2], 2, 0) ||
|
||||
!decode_one(ctx.get(), mr.tokens[2], 2, 1)) {
|
||||
TLOG_ERR("%s: failed to build the interleaved state\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
// host blob: contains the KV data, used for the byte-for-byte comparison
|
||||
std::vector<uint8_t> state_before;
|
||||
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_before)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
// save via the io path under test
|
||||
std::vector<uint8_t> state_save;
|
||||
if (!get_seq_state(ctx.get(), 1, flags, state_save)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
TLOG_TRC("%s: seq 1 saved via %s, %zu bytes\n", __func__, on_device ? "device" : "host", state_save.size());
|
||||
|
||||
// free seq 0's cells so the ring is fragmented: the restore destination (seq 1's interleaved cells) stays non-contiguous
|
||||
if (!seq_rm(ctx.get(), 0)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
// restore via the io path under test
|
||||
if (!set_seq_state(ctx.get(), 1, flags, state_save)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
TLOG_TRC("%s: seq 1 restored via %s, %zu bytes\n", __func__, on_device ? "device" : "host", state_save.size());
|
||||
|
||||
std::vector<uint8_t> state_after;
|
||||
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_after)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
// the blob is serialized in sequence cell order, so identical bytes iff the restore wrote the same KV
|
||||
CTX_REQUIRE(blobs_equal(state_before, state_after), "the restored KV state is not byte-identical to the saved state");
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
|
||||
private:
|
||||
bool on_device;
|
||||
};
|
||||
|
||||
//
|
||||
// sessions
|
||||
//
|
||||
|
||||
// - save a sequence, erase it, restore it, save it again
|
||||
// - the two blobs must be identical
|
||||
// compares blobs rather than generated text: a partially restored cell still decodes to plausible tokens
|
||||
struct case_state_roundtrip : ctx_test {
|
||||
const char * name() const override { return "rt"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "save, erase, restore, save again - the two blobs must be identical";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
auto ctx = mr.ctx();
|
||||
if (!ctx) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
common_batch batch = common_batch_get_one(ctx.get(), mr.tokens);
|
||||
if (llama_process(ctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
TLOG_ERR("%s: failed to decode the prompt\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
std::vector<uint8_t> blob_a;
|
||||
if (!get_seq_state(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, blob_a)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
if (!seq_rm(ctx.get(), 0)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
if (!set_seq_state(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, blob_a)) {
|
||||
TLOG_ERR("%s: failed to restore seq 0\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
std::vector<uint8_t> blob_b;
|
||||
if (!get_seq_state(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, blob_b)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
CTX_REQUIRE(blobs_equal(blob_a, blob_b), "the state changed across a save / erase / restore cycle");
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
// the partial state holds what seq_rm cannot roll back: recurrent states and SWA windows
|
||||
struct case_state_partial_size : ctx_test {
|
||||
const char * name() const override { return "ps"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "the partial state is not larger than the full state and, without SWA, does not grow with the tokens";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
auto ctx = mr.ctx();
|
||||
if (!ctx) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const llama_pos n_tokens = mr.tokens.size();
|
||||
const llama_pos n_half = n_tokens / 2;
|
||||
|
||||
if (!decode_seq(ctx.get(), mr.tokens, 0, n_half, 0, true)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const size_t partial_a = llama_state_seq_get_size_ext(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
const size_t full_a = llama_state_seq_get_size_ext(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE);
|
||||
|
||||
if (!decode_seq(ctx.get(), mr.tokens, n_half, n_tokens, 0, true)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const size_t partial_b = llama_state_seq_get_size_ext(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY);
|
||||
const size_t full_b = llama_state_seq_get_size_ext(ctx.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE);
|
||||
|
||||
TLOG_INF("%s: partial %zu -> %zu, full %zu -> %zu\n", __func__, partial_a, partial_b, full_a, full_b);
|
||||
|
||||
CTX_REQUIRE(partial_a <= full_a && partial_b <= full_b, "the partial state is larger than the full state");
|
||||
|
||||
if (llama_model_n_swa(mr.model) == 0) {
|
||||
CTX_REQUIRE(partial_a == partial_b, "the partial state grows with the tokens");
|
||||
}
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
// a failed restore must leave the sequence empty and must not change the logits of other sequences
|
||||
struct case_state_restore_failure : ctx_test {
|
||||
const char * name() const override { return "rf"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "restoring a corrupted state fails and leaves the other sequences untouched";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
ctx_opts opts;
|
||||
opts.n_ctx = 256;
|
||||
opts.n_seq_max = 4;
|
||||
opts.kv_unified = 1;
|
||||
|
||||
// without flash attention, corrupted data left behind by the restore shows up as NaN logits on the other sequences
|
||||
opts.fa = LLAMA_FLASH_ATTN_TYPE_DISABLED;
|
||||
|
||||
auto ctx = mr.ctx(opts);
|
||||
if (!ctx) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
llama_memory_t mem = llama_get_memory(ctx.get());
|
||||
if (mem == nullptr) {
|
||||
TLOG_INF("no memory to test\n");
|
||||
return case_result::pass();
|
||||
}
|
||||
|
||||
const auto decode = [&](const llama_tokens & inp, llama_seq_id seq_id, std::vector<float> * logits_out) {
|
||||
if (!decode_tokens(ctx.get(), inp, seq_id)) {
|
||||
TLOG_ERR("%s: failed to decode on sequence %d\n", __func__, seq_id);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (logits_out && !get_current_logits(ctx.get(), *logits_out)) {
|
||||
TLOG_ERR("%s: failed to get logits\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
const llama_tokens tokens_save (mr.tokens.begin(), mr.tokens.begin() + std::min<size_t>(24, mr.tokens.size()));
|
||||
const llama_tokens tokens_verify(mr.tokens.end() - std::min<size_t>(8, mr.tokens.size()), mr.tokens.end());
|
||||
|
||||
// the cases share a working directory, so the state file is named after the model
|
||||
const std::string path = "state-restore-failure." + mr.name + ".tmp.bin";
|
||||
|
||||
llama_memory_clear(mem, true);
|
||||
|
||||
std::vector<float> baseline;
|
||||
if (!decode(tokens_verify, 1, &baseline)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const std::vector<std::pair<const char *, std::function<bool()>>> sub_cases = {
|
||||
{ "buffer", [&]() {
|
||||
std::vector<uint8_t> state(llama_state_seq_get_size(ctx.get(), 0));
|
||||
GGML_ASSERT(llama_state_seq_get_data(ctx.get(), state.data(), state.size(), 0) == state.size());
|
||||
llama_memory_seq_rm(mem, 0, -1, -1);
|
||||
|
||||
if (!corrupt_state(state)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return llama_state_seq_set_data(ctx.get(), state.data(), state.size(), 0) == 0;
|
||||
}},
|
||||
{ "file", [&]() {
|
||||
GGML_ASSERT(llama_state_seq_save_file(ctx.get(), path.c_str(), 0, tokens_save.data(), tokens_save.size()) > 0);
|
||||
llama_memory_seq_rm(mem, 0, -1, -1);
|
||||
|
||||
std::vector<uint8_t> data;
|
||||
{
|
||||
std::ifstream f(path, std::ios::binary);
|
||||
data.assign(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>());
|
||||
}
|
||||
|
||||
if (!corrupt_state(data)) {
|
||||
std::remove(path.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
{
|
||||
std::ofstream f(path, std::ios::binary);
|
||||
f.write((const char *) data.data(), data.size());
|
||||
}
|
||||
|
||||
llama_tokens tokens_out(tokens_save.size());
|
||||
size_t n_token_count = 0;
|
||||
const size_t nread = llama_state_seq_load_file(ctx.get(), path.c_str(), 0, tokens_out.data(), tokens_out.size(), &n_token_count);
|
||||
std::remove(path.c_str());
|
||||
|
||||
return nread == 0;
|
||||
}},
|
||||
};
|
||||
|
||||
for (const auto & [name, restore_failed] : sub_cases) {
|
||||
llama_memory_clear(mem, true);
|
||||
|
||||
if (!decode(tokens_save, 0, nullptr)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
CTX_REQUIRE(restore_failed(), std::string(name) + ": restoring a corrupted state did not fail");
|
||||
|
||||
CTX_REQUIRE(llama_memory_seq_pos_max(mem, 0) == -1,
|
||||
std::string(name) + ": the sequence is not empty after a failed restore");
|
||||
|
||||
std::vector<float> logits;
|
||||
if (!decode(tokens_verify, 1, &logits)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const logits_diff diff = compare_logits(baseline.data(), logits.data(), logits.size());
|
||||
|
||||
CTX_REQUIRE(diff.n_nan == 0 && diff.diff_max <= 1e-6f,
|
||||
std::string(name) + ": the logits changed after a failed restore (max diff = " +
|
||||
std::to_string(diff.diff_max) + ", nan = " + std::to_string(diff.n_nan) + ")");
|
||||
|
||||
TLOG_TRC("%s: %s: logits match (max diff = %g)\n", __func__, name, (double) diff.diff_max);
|
||||
}
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
// a KV state saved with attention rotation enabled must restore only into a context with the
|
||||
// same setting
|
||||
// note: rotation is only active for quantized KV caches with a head size that is a multiple of
|
||||
// 64, for other models the restore into the rotation-disabled context is valid and the
|
||||
// test passes vacuously
|
||||
struct case_state_rotation : ctx_test {
|
||||
const char * name() const override { return "rot"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "a state saved with attention rotation restores only into a matching context";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
const auto make_context = [&](ggml_type type_k, ggml_type type_v, bool disable_rotation) {
|
||||
// a setting of the model, not of the process - a sweep that runs several models at a
|
||||
// time must not let this leak into the contexts of another model
|
||||
llama_model_set_attn_rot(mr.model, !disable_rotation);
|
||||
|
||||
ctx_opts opts;
|
||||
opts.n_ctx = 32;
|
||||
opts.n_batch = 1;
|
||||
opts.n_ubatch = 1;
|
||||
opts.type_k = type_k;
|
||||
opts.type_v = type_v;
|
||||
opts.fa = LLAMA_FLASH_ATTN_TYPE_ENABLED;
|
||||
|
||||
// probing whether this KV type is supported is expected to fail for some models
|
||||
opts.quiet = true;
|
||||
|
||||
return mr.ctx(opts);
|
||||
};
|
||||
|
||||
std::vector<std::pair<ggml_type, ggml_type>> type_pairs;
|
||||
for (const auto & types : { std::pair{GGML_TYPE_Q8_0, GGML_TYPE_Q8_0} }) {
|
||||
if (make_context(types.first, types.second, false)) {
|
||||
type_pairs.push_back(types);
|
||||
}
|
||||
}
|
||||
if (type_pairs.empty()) {
|
||||
return case_result::skip("no supported quantized KV cache type combination");
|
||||
}
|
||||
|
||||
bool success = true;
|
||||
for (const auto & types : type_pairs) {
|
||||
auto src = make_context(types.first, types.second, false);
|
||||
if (!src) {
|
||||
TLOG_ERR("%s: failed to create the source context\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
llama_token token = 0;
|
||||
if (llama_decode(src.get(), llama_batch_get_one(&token, 1))) {
|
||||
TLOG_ERR("%s: failed to decode a token\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> state;
|
||||
if (!get_seq_state(src.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, state)) {
|
||||
if (state.empty()) {
|
||||
continue; // no KV state to test
|
||||
}
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
auto matching = make_context(types.first, types.second, false);
|
||||
if (!matching || !set_seq_state(matching.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, state)) {
|
||||
TLOG_ERR("%s: failed to restore a matching rotation\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
|
||||
auto mismatched = make_context(types.first, types.second, true);
|
||||
if (!mismatched) {
|
||||
TLOG_ERR("%s: failed to create a mismatched rotation context\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
// a model that does not use attention rotation accepts the mismatched restore
|
||||
if (set_seq_state(mismatched.get(), 0, LLAMA_STATE_SEQ_FLAGS_NONE, state)) {
|
||||
TLOG_TRC("%s: state restored into rotation-disabled context, model does not use attention rotation\n", __func__);
|
||||
}
|
||||
}
|
||||
|
||||
llama_model_set_attn_rot(mr.model, true); // the default, the cases that follow rely on it
|
||||
|
||||
if (!success) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
// restore a sequence into a fragmented cache: the slots freed by the removed sequence are
|
||||
// scattered between live cells of the other sequences, so the restore cannot claim one
|
||||
// contiguous block
|
||||
// the restored state must come back byte for byte, and the sequence must continue to behave
|
||||
// exactly as the same sequence in a context where it was never removed
|
||||
// ref: https://github.com/ggml-org/llama.cpp/issues/17527
|
||||
struct case_state_restore_fragmented : ctx_test {
|
||||
const char * name() const override { return "frag"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "restore a sequence into a cache fragmented by the removal of another sequence";
|
||||
}
|
||||
|
||||
std::string skip_reason(model_run & mr) const override {
|
||||
if (arch_reserves_single_seq(mr.model)) {
|
||||
return "the interleaved batch is a multi-seq graph";
|
||||
}
|
||||
|
||||
return ctx_test::skip_reason(mr);
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(mr.model));
|
||||
|
||||
ctx_opts opts;
|
||||
opts.n_ctx = 256;
|
||||
opts.n_seq_max = 3;
|
||||
opts.kv_unified = 1;
|
||||
opts.n_batch = 3*70;
|
||||
opts.n_ubatch = 3*70; // the interleaved prompt is a single multi-seq batch
|
||||
|
||||
// ctx removes and restores seq 1, ctx_keep keeps it untouched as the reference
|
||||
llama_context_ptr ctx = mr.ctx(opts);
|
||||
llama_context_ptr ctx_keep = mr.ctx(opts);
|
||||
if (!ctx || !ctx_keep) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const llama_pos n_tokens = (llama_pos) std::min<size_t>(70, mr.tokens.size());
|
||||
|
||||
// interleave the 3 sequences: 01201230123... - seq 1 owns every third cell
|
||||
const auto decode_interleaved = [&](llama_context * cur) {
|
||||
common_batch batch(cur);
|
||||
for (llama_pos i = 0; i < n_tokens; i++) {
|
||||
for (llama_seq_id s = 0; s < 3; ++s) {
|
||||
batch.add(mr.tokens[i], i, s, false);
|
||||
}
|
||||
}
|
||||
batch.set_output(batch.size() - 1, true);
|
||||
|
||||
return llama_process(cur, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
|
||||
};
|
||||
|
||||
if (!decode_interleaved(ctx.get()) || !decode_interleaved(ctx_keep.get())) {
|
||||
TLOG_ERR("%s: failed to decode the interleaved prompt\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
TLOG_INF("%s: processed the prompt on seq 0, 1, 2 (%d tokens each)\n", __func__, n_tokens);
|
||||
|
||||
std::vector<uint8_t> state_before;
|
||||
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_before)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
// clearing seq 1 leaves holes where its cells were - no contiguous block large enough
|
||||
// for the seq 1 state is left in the cache
|
||||
llama_memory_seq_rm(llama_get_memory(ctx.get()), 1, -1, -1);
|
||||
|
||||
if (!set_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_before)) {
|
||||
TLOG_ERR("%s: failed to restore the seq state into a fragmented cache\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
TLOG_INF("%s: restored seq 1 into a fragmented cache, %zu bytes\n", __func__, state_before.size());
|
||||
|
||||
// the restore has to be lossless - the cells the sequence ends up in are irrelevant, its
|
||||
// state has to come back exactly as it was saved
|
||||
std::vector<uint8_t> state_after;
|
||||
if (!get_seq_state(ctx.get(), 1, LLAMA_STATE_SEQ_FLAGS_NONE, state_after)) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
CTX_REQUIRE(blobs_equal(state_before, state_after), "restoring into a fragmented cache changed the state of seq 1");
|
||||
|
||||
// the restored sequence must continue exactly as the untouched one - decode the same token
|
||||
// on seq 1 in both contexts and compare the logits they produce
|
||||
const llama_token token = gen_token(n_vocab, 1, n_tokens);
|
||||
|
||||
if (!decode_one(ctx.get(), token, n_tokens, 1) || !decode_one(ctx_keep.get(), token, n_tokens, 1)) {
|
||||
TLOG_ERR("%s: failed to decode with the restored state\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const float * logits = llama_get_logits(ctx.get());
|
||||
const float * logits_keep = llama_get_logits(ctx_keep.get());
|
||||
if (!logits || !logits_keep) {
|
||||
TLOG_ERR("%s: missing logits after the restore\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const logits_diff diff = compare_logits(logits, logits_keep, n_vocab);
|
||||
|
||||
CTX_REQUIRE(diff.nmse() <= NMSE_THRESHOLD,
|
||||
"the logits of the restored seq 1 differ from the reference: nmse " + std::to_string(diff.nmse()));
|
||||
|
||||
TLOG_INF("%s: restored seq 1 continued identically to the reference, nmse %g\n", __func__, diff.nmse());
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// no_alloc
|
||||
//
|
||||
|
||||
// - load a second instance of the model with no_alloc, so the tensor data stays on disk
|
||||
// - create a context from it, which reserves the compute graphs over the dummy buffers
|
||||
// - the memory use must still be reported, and nothing real may be allocated for the memory
|
||||
struct case_noalloc : ctx_test {
|
||||
const char * name() const override { return "noalloc"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "with no_alloc the memory use is simulated: non-zero breakdown, null buffer bases";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
const std::string & path = mr.params.model.path;
|
||||
CTX_REQUIRE(!path.empty(), "the model path is not known");
|
||||
|
||||
llama_model_params mparams = llama_model_default_params();
|
||||
mparams.no_alloc = true;
|
||||
mparams.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
|
||||
llama_model_ptr model(llama_model_load_from_file(path.c_str(), mparams));
|
||||
CTX_REQUIRE(model, "failed to load the model with no_alloc");
|
||||
|
||||
auto ctx = init_ctx(model.get(), common_context_params_to_llama(mr.params), 0);
|
||||
CTX_REQUIRE(ctx, "failed to create the context with no_alloc");
|
||||
|
||||
size_t n_ctx_mem = 0;
|
||||
for (const auto & [_, data] : llama_get_memory_breakdown(ctx.get())) {
|
||||
n_ctx_mem += data.context;
|
||||
}
|
||||
|
||||
// models without a memory module (BERT, diffusion) have a null memory
|
||||
if (llama_get_memory(ctx.get()) != nullptr) {
|
||||
CTX_REQUIRE(n_ctx_mem > 0, "the simulated memory breakdown is zero");
|
||||
} else {
|
||||
CTX_REQUIRE(n_ctx_mem == 0, "a model without memory reported context memory");
|
||||
}
|
||||
|
||||
for (ggml_backend_buffer_t buf : llama_memory_get_buffers(llama_get_memory(ctx.get()))) {
|
||||
CTX_REQUIRE(ggml_backend_buffer_get_base(buf) == nullptr, "a memory buffer was really allocated");
|
||||
}
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<ctx_test_ptr> make_kv_cases() {
|
||||
std::vector<ctx_test_ptr> cases;
|
||||
|
||||
cases.push_back(std::make_unique<case_baseline>());
|
||||
cases.push_back(std::make_unique<case_seq_rm_isolated>());
|
||||
cases.push_back(std::make_unique<case_state_load>());
|
||||
cases.push_back(std::make_unique<case_seq_cp>(false));
|
||||
cases.push_back(std::make_unique<case_seq_cp>(true));
|
||||
cases.push_back(std::make_unique<case_seq_cp_scatter>(false));
|
||||
cases.push_back(std::make_unique<case_seq_cp_scatter>(true));
|
||||
cases.push_back(std::make_unique<case_state_roundtrip>());
|
||||
cases.push_back(std::make_unique<case_state_partial_size>());
|
||||
cases.push_back(std::make_unique<case_state_restore_failure>());
|
||||
cases.push_back(std::make_unique<case_state_rotation>());
|
||||
cases.push_back(std::make_unique<case_state_restore_fragmented>());
|
||||
cases.push_back(std::make_unique<case_noalloc>());
|
||||
|
||||
return cases;
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
// recurrent state cases
|
||||
|
||||
#include "test-case.h"
|
||||
|
||||
#include "test-log.h"
|
||||
#include "test-util.h"
|
||||
|
||||
#include "common.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// roll back multiple sequences, then replay them in a single batch whose per-seq token count
|
||||
// exceeds n_ubatch: each seq's replay spans several ubatches while its rollback restore is
|
||||
// still pending. Compared against a reference context that never advanced past the rollback
|
||||
// point and decodes the identical replay batch.
|
||||
struct case_multi_seq_split_replay : ctx_test {
|
||||
test_group group() const override { return TEST_GROUP_RS; }
|
||||
|
||||
const char * name() const override { return "replay"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "replay rolled-back sequences in a batch wider than n_ubatch, against a reference context";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(mr.model));
|
||||
|
||||
constexpr uint32_t n_seqs = 2;
|
||||
constexpr uint32_t n_ubatch = 16;
|
||||
constexpr uint32_t n_prompt = 20;
|
||||
constexpr uint32_t n_rollback = 3;
|
||||
constexpr uint32_t n_replay = 40; // > n_ubatch so each seq spans multiple ubatches
|
||||
constexpr llama_pos p0 = n_prompt - n_rollback;
|
||||
|
||||
// both contexts have the same layout, only what they decode differs
|
||||
const auto make_ctx_multi = [&]() {
|
||||
ctx_opts opts;
|
||||
opts.n_ctx = 256;
|
||||
opts.n_batch = 256;
|
||||
opts.n_ubatch = n_ubatch;
|
||||
opts.n_seq_max = n_seqs;
|
||||
opts.n_rs_seq = 8;
|
||||
opts.kv_unified = 0;
|
||||
return mr.ctx(opts);
|
||||
};
|
||||
|
||||
llama_context_ptr ctx_roll = make_ctx_multi();
|
||||
llama_context_ptr ctx_ref = make_ctx_multi();
|
||||
if (!ctx_roll || !ctx_ref) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
if (llama_n_rs_seq(ctx_roll.get()) < n_rollback) {
|
||||
return case_result::skip("n_rs_seq is too small");
|
||||
}
|
||||
|
||||
const auto tok = [&](uint32_t seq, llama_pos pos) {
|
||||
return gen_token(n_vocab, (llama_seq_id) seq, pos);
|
||||
};
|
||||
|
||||
bool ok = true;
|
||||
|
||||
// keep the anchor at p0 - 1 when rolling back the tail
|
||||
for (uint32_t s = 0; s < n_seqs && ok; ++s) {
|
||||
ok = ok && decode_gen(ctx_roll.get(), n_vocab, (llama_seq_id) s, 0, (llama_pos) p0 - 1);
|
||||
ok = ok && decode_gen(ctx_ref.get(), n_vocab, (llama_seq_id) s, 0, (llama_pos) p0 - 1);
|
||||
|
||||
ok = ok && decode_gen(ctx_roll.get(), n_vocab, (llama_seq_id) s, (llama_pos) p0 - 1, (llama_pos) n_prompt);
|
||||
ok = ok && decode_gen(ctx_ref.get(), n_vocab, (llama_seq_id) s, (llama_pos) p0 - 1, (llama_pos) p0);
|
||||
|
||||
ok = ok && llama_memory_seq_rm(llama_get_memory(ctx_roll.get()), (llama_seq_id) s, p0, -1);
|
||||
|
||||
// a second partial removal while one is pending must be refused
|
||||
ok = ok && !llama_memory_seq_rm(llama_get_memory(ctx_roll.get()), (llama_seq_id) s, p0 - 1, -1);
|
||||
}
|
||||
CTX_REQUIRE(ok, "the multi-seq prefill / rollback failed");
|
||||
|
||||
// all seqs replay in a single batch
|
||||
const auto decode_replay = [&](llama_context * ctx) {
|
||||
common_batch batch(ctx);
|
||||
for (uint32_t s = 0; s < n_seqs; ++s) {
|
||||
for (uint32_t i = 0; i < n_replay; ++i) {
|
||||
const llama_pos pos = p0 + (llama_pos) i;
|
||||
batch.add(tok(s, pos), pos, (llama_seq_id) s, true);
|
||||
}
|
||||
}
|
||||
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
|
||||
};
|
||||
|
||||
ok = decode_replay(ctx_roll.get()) && decode_replay(ctx_ref.get());
|
||||
CTX_REQUIRE(ok, "the multi-seq replay decode failed");
|
||||
|
||||
// both contexts decode identical batches, so the logits should match;
|
||||
// random dummy models can still drift up to ~1.7e-5, so the bound is 1e-4
|
||||
constexpr float nmse_eps = 1e-4f;
|
||||
|
||||
logits_diff diff;
|
||||
for (uint32_t i = 0; i < n_seqs*n_replay; ++i) {
|
||||
const float * l_roll = llama_get_logits_ith(ctx_roll.get(), i);
|
||||
const float * l_ref = llama_get_logits_ith(ctx_ref.get(), i);
|
||||
if (l_roll == nullptr || l_ref == nullptr) {
|
||||
TLOG_ERR("%s: missing multi-seq logits at index %u\n", __func__, i);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
diff.add(l_roll, l_ref, n_vocab);
|
||||
}
|
||||
|
||||
CTX_REQUIRE(diff.nmse() <= nmse_eps,
|
||||
"multi-seq split replay logits mismatch (max diff " + std::to_string(diff.diff_max) +
|
||||
", nmse " + std::to_string(diff.nmse()) + ", first difference at index " + std::to_string(diff.i_first) + ")");
|
||||
|
||||
TLOG_INF("%s: multi-seq split replay matched (max diff %g, nmse %g)\n", __func__, (double) diff.diff_max, diff.nmse());
|
||||
|
||||
// seq-1-only decodes must be independent of seq 0's content: diverge seq 0
|
||||
// in ctx_ref only, then compare identical seq-1-only continuations
|
||||
constexpr uint32_t n_tail = 4;
|
||||
|
||||
{
|
||||
common_batch batch_tail(ctx_ref.get());
|
||||
for (uint32_t i = 0; i < n_tail; ++i) {
|
||||
const llama_pos pos = p0 + (llama_pos) (n_replay + i);
|
||||
batch_tail.add(tok(0, pos + 7), pos, 0, false);
|
||||
}
|
||||
ok = llama_process(ctx_ref.get(), LLAMA_PROCESS_TYPE_DECODE, batch_tail.get()) == 0;
|
||||
}
|
||||
|
||||
logits_diff diff_tail;
|
||||
for (uint32_t i = 0; i < n_tail && ok; ++i) {
|
||||
const llama_pos pos = p0 + (llama_pos) (n_replay + i);
|
||||
ok = decode_one(ctx_roll.get(), tok(1, pos), pos, 1) &&
|
||||
decode_one(ctx_ref.get(), tok(1, pos), pos, 1);
|
||||
if (!ok) {
|
||||
break;
|
||||
}
|
||||
|
||||
const float * l_roll = llama_get_logits_ith(ctx_roll.get(), 0);
|
||||
const float * l_ref = llama_get_logits_ith(ctx_ref.get(), 0);
|
||||
if (l_roll == nullptr || l_ref == nullptr) {
|
||||
TLOG_ERR("%s: missing tail logits at step %u\n", __func__, i);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
diff_tail.add(l_roll, l_ref, n_vocab);
|
||||
}
|
||||
|
||||
CTX_REQUIRE(ok && diff_tail.nmse() <= nmse_eps,
|
||||
"a seq-1-only decode leaked the state of seq 0 (nmse " + std::to_string(diff_tail.nmse()) + ")");
|
||||
|
||||
TLOG_INF("%s: seq-1-only decode is independent of seq 0 (max diff %g, nmse %g)\n",
|
||||
__func__, (double) diff_tail.diff_max, diff_tail.nmse());
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
// save a rolled-back single-seq state, restore it into a fresh and into a dirty context, and
|
||||
// verify exact logit matches on replay
|
||||
struct case_rollback : ctx_test {
|
||||
test_group group() const override { return TEST_GROUP_RS; }
|
||||
|
||||
const char * name() const override { return "rollback"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "a rolled-back recurrent state restores into fresh and dirty contexts with identical replay";
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
const llama_vocab * vocab = llama_model_get_vocab(mr.model);
|
||||
const int n_vocab = llama_vocab_n_tokens(vocab);
|
||||
|
||||
const auto make_rs_ctx = [&]() {
|
||||
ctx_opts opts;
|
||||
opts.n_seq_max = 1;
|
||||
opts.n_rs_seq = 8;
|
||||
return mr.ctx(opts);
|
||||
};
|
||||
|
||||
llama_context_ptr ctx_src = make_rs_ctx();
|
||||
llama_context_ptr ctx_dst = make_rs_ctx();
|
||||
if (!ctx_src || !ctx_dst) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const uint32_t n_rs_seq = llama_n_rs_seq(ctx_src.get());
|
||||
if (n_rs_seq == 0) {
|
||||
return case_result::skip("n_rs_seq is disabled");
|
||||
}
|
||||
|
||||
constexpr uint32_t n_rollback = 3;
|
||||
if (n_rs_seq < n_rollback) {
|
||||
return case_result::skip("n_rs_seq is too small");
|
||||
}
|
||||
|
||||
llama_tokens tokens;
|
||||
if (!mr.has_vocab()) {
|
||||
tokens = { 1, 2, 3, 4, 5, 6, 7, 8, 9 };
|
||||
} else {
|
||||
tokens = common_tokenize(ctx_src.get(), "The quick brown fox jumps over the lazy dog", true);
|
||||
}
|
||||
if (tokens.empty()) {
|
||||
TLOG_ERR("%s: not enough prompt tokens\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
tokens.resize(n_rs_seq + 1, tokens.back());
|
||||
|
||||
const llama_pos rollback_pos = (llama_pos) tokens.size() - n_rollback;
|
||||
|
||||
// Decode the full prompt on the source, then roll back three positions.
|
||||
// Replaying them crosses DSV4's ratio-4 compressor boundary.
|
||||
// Rollback leaves the recurrent memory in a snapshot state (rs_idx != 0).
|
||||
if (!decode_tokens(ctx_src.get(), tokens)) {
|
||||
TLOG_ERR("%s: failed to decode the prompt\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
if (!llama_memory_seq_rm(llama_get_memory(ctx_src.get()), 0, rollback_pos, -1)) {
|
||||
TLOG_ERR("%s: rollback failed\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
// Save the rolled-back state and restore it into a fresh context.
|
||||
common_prompt_checkpoint ckpt;
|
||||
ckpt.update_tgt(ctx_src.get(), 0, 0);
|
||||
ckpt.load_tgt(ctx_dst.get(), 0, 0);
|
||||
|
||||
// exact match - the same context configuration replaying the same tokens
|
||||
constexpr float nmse_eps = 0.0;
|
||||
|
||||
std::vector<std::vector<float>> logits_src_replay(n_rollback);
|
||||
for (uint32_t i = 0; i < n_rollback; ++i) {
|
||||
const llama_pos pos = rollback_pos + i;
|
||||
if (!decode_one(ctx_src.get(), tokens[pos], pos) ||
|
||||
!decode_one(ctx_dst.get(), tokens[pos], pos)) {
|
||||
TLOG_ERR("%s: replay failed at position %d\n", __func__, pos);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const float * logits_src = llama_get_logits_ith(ctx_src.get(), 0);
|
||||
const float * logits_dst = llama_get_logits_ith(ctx_dst.get(), 0);
|
||||
if (logits_src == nullptr || logits_dst == nullptr) {
|
||||
TLOG_ERR("%s: missing replay logits at position %d\n", __func__, pos);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
logits_src_replay[i].assign(logits_src, logits_src + n_vocab);
|
||||
|
||||
const logits_diff diff = compare_logits(logits_src, logits_dst, n_vocab);
|
||||
CTX_REQUIRE(diff.nmse() <= nmse_eps,
|
||||
"replay logits mismatch at position " + std::to_string(pos) + ", nmse " + std::to_string(diff.nmse()));
|
||||
}
|
||||
|
||||
// TODO: this test is invalid because RS rollback is only correct once after a ubatch with more than n_rs_seq tokens
|
||||
// this is not the case here. add asserts and guardrails to prevent such attempts
|
||||
//if (!llama_memory_seq_rm(llama_get_memory(ctx_src.get()), 0, rollback_pos, -1) ||
|
||||
// !llama_memory_seq_rm(llama_get_memory(ctx_dst.get()), 0, rollback_pos, -1)) {
|
||||
// TLOG_ERR("%s: partial rollback failed\n", __func__);
|
||||
// return case_result::fail();
|
||||
//}
|
||||
|
||||
//constexpr llama_state_seq_flags partial_flags = LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY;
|
||||
//common_prompt_checkpoint ckpt_partial;
|
||||
//ckpt_partial.update_tgt(ctx_src.get(), 0, partial_flags);
|
||||
//ckpt_partial.load_tgt(ctx_dst.get(), 0, partial_flags);
|
||||
|
||||
// Repeat the load into a context that already has its own rollback state:
|
||||
// groups 1..n_rs_seq hold a different prompt's history, and rs_idx[0] is
|
||||
// non-zero at load time. The restore must wipe that state and still match.
|
||||
llama_context_ptr ctx_dirty = make_rs_ctx();
|
||||
if (!ctx_dirty) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
llama_tokens noise = tokens;
|
||||
for (auto & tok : noise) {
|
||||
tok = (tok + 1) % n_vocab;
|
||||
if (tok < 0) {
|
||||
tok = 0;
|
||||
}
|
||||
}
|
||||
if (!decode_tokens(ctx_dirty.get(), noise)) {
|
||||
TLOG_ERR("%s: failed to decode the dirty prompt\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty.get()), 0, rollback_pos, -1)) {
|
||||
TLOG_ERR("%s: dirty rollback failed\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
ckpt.load_tgt(ctx_dirty.get(), 0, 0);
|
||||
|
||||
for (uint32_t i = 0; i < n_rollback; ++i) {
|
||||
const llama_pos pos = rollback_pos + i;
|
||||
if (!decode_one(ctx_dirty.get(), tokens[pos], pos)) {
|
||||
TLOG_ERR("%s: dirty replay failed at position %d\n", __func__, pos);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const float * logits_dirty = llama_get_logits_ith(ctx_dirty.get(), 0);
|
||||
if (logits_dirty == nullptr) {
|
||||
TLOG_ERR("%s: missing dirty logits at position %d\n", __func__, pos);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
const logits_diff diff = compare_logits(logits_src_replay[i].data(), logits_dirty, n_vocab);
|
||||
CTX_REQUIRE(diff.nmse() <= nmse_eps,
|
||||
"dirty-ctx logits mismatch at position " + std::to_string(pos) + ", nmse " + std::to_string(diff.nmse()));
|
||||
}
|
||||
|
||||
TLOG_INF("%s: recurrent rollback checkpoint restored successfully\n", __func__);
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
// decode a prompt into seq 0, share its cells with a second sequence, then keep decoding both
|
||||
struct case_shared_seq_reserve : ctx_test {
|
||||
test_group group() const override { return TEST_GROUP_RS; }
|
||||
|
||||
const char * name() const override { return "shared"; }
|
||||
|
||||
const char * description() const override {
|
||||
return "sequences sharing cells keep decoding correctly after a seq_cp";
|
||||
}
|
||||
|
||||
std::string skip_reason(model_run & mr) const override {
|
||||
if (arch_reserves_single_seq(mr.model)) {
|
||||
return "the shared-seq batch is a multi-seq graph";
|
||||
}
|
||||
|
||||
return ctx_test::skip_reason(mr);
|
||||
}
|
||||
|
||||
case_result run(model_run & mr) const override {
|
||||
const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(mr.model));
|
||||
|
||||
constexpr uint32_t n_seqs = 2;
|
||||
constexpr uint32_t n_prompt = 128;
|
||||
constexpr uint32_t n_continue = 32;
|
||||
|
||||
ctx_opts opts;
|
||||
opts.n_ctx = 512;
|
||||
opts.n_batch = 256;
|
||||
opts.n_ubatch = 64;
|
||||
opts.n_seq_max = n_seqs;
|
||||
opts.kv_unified = 1; // only a unified cache shares cells on seq_cp
|
||||
|
||||
auto ctx = mr.ctx(opts);
|
||||
if (!ctx) {
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
if (!decode_gen(ctx.get(), n_vocab, 0, 0, (llama_pos) n_prompt)) {
|
||||
TLOG_ERR("%s: prompt decode failed\n", __func__);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
// this is what llama-batched-bench does for -pps
|
||||
llama_memory_seq_cp(llama_get_memory(ctx.get()), 0, 1, -1, -1);
|
||||
|
||||
for (uint32_t i = 0; i < n_continue; ++i) {
|
||||
const llama_pos pos = (llama_pos) (n_prompt + i);
|
||||
|
||||
common_batch batch(ctx.get());
|
||||
for (uint32_t s = 0; s < n_seqs; ++s) {
|
||||
batch.add(gen_token(n_vocab, (llama_seq_id) s, pos), pos, (llama_seq_id) s, true);
|
||||
}
|
||||
CTX_REQUIRE(llama_process(ctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0,
|
||||
"shared-seq decode failed at step " + std::to_string(i));
|
||||
|
||||
for (uint32_t s = 0; s < n_seqs; ++s) {
|
||||
const float * logits = llama_get_logits_ith(ctx.get(), (int) s);
|
||||
if (logits == nullptr) {
|
||||
TLOG_ERR("%s: missing shared-seq logits at index %u\n", __func__, s);
|
||||
return case_result::fail();
|
||||
}
|
||||
|
||||
bool finite = true;
|
||||
for (int t_ = 0; t_ < n_vocab; ++t_) {
|
||||
finite = finite && std::isfinite(logits[t_]);
|
||||
}
|
||||
|
||||
CTX_REQUIRE(finite, "non-finite shared-seq logit at step " + std::to_string(i) + ", seq " + std::to_string(s));
|
||||
}
|
||||
}
|
||||
|
||||
TLOG_INF("%s: shared-seq decode succeeded (%u tokens after seq_cp)\n", __func__, n_continue*n_seqs);
|
||||
|
||||
return case_result::pass();
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<ctx_test_ptr> make_rs_cases() {
|
||||
std::vector<ctx_test_ptr> cases;
|
||||
|
||||
cases.push_back(std::make_unique<case_rollback>());
|
||||
cases.push_back(std::make_unique<case_multi_seq_split_replay>());
|
||||
cases.push_back(std::make_unique<case_shared_seq_reserve>());
|
||||
|
||||
return cases;
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
#include "test-case.h"
|
||||
|
||||
#include <iterator>
|
||||
|
||||
const char * test_status_str(test_status status) {
|
||||
switch (status) {
|
||||
case test_status::PASS: return "\033[1;32mOK\033[0m";
|
||||
case test_status::FAIL: return "\033[1;31mFAIL\033[0m";
|
||||
case test_status::SKIP: return "\033[1;33mSKIP\033[0m";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
// the printable width of a verdict, the color escapes of test_status_str are not part of it
|
||||
size_t test_status_width(test_status status) {
|
||||
switch (status) {
|
||||
case test_status::PASS: return 2; // "OK"
|
||||
case test_status::FAIL: return 4; // "FAIL"
|
||||
case test_status::SKIP: return 4; // "SKIP"
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
test_status merge_status(test_status a, test_status b) {
|
||||
if (a == test_status::FAIL || b == test_status::FAIL) {
|
||||
return test_status::FAIL;
|
||||
}
|
||||
if (a == test_status::PASS || b == test_status::PASS) {
|
||||
return test_status::PASS;
|
||||
}
|
||||
return test_status::SKIP;
|
||||
}
|
||||
|
||||
case_result case_result::pass() {
|
||||
return { test_status::PASS, {} };
|
||||
}
|
||||
|
||||
case_result case_result::fail(const std::string & why) {
|
||||
return { test_status::FAIL, why };
|
||||
}
|
||||
|
||||
case_result case_result::skip(const std::string & why) {
|
||||
return { test_status::SKIP, why };
|
||||
}
|
||||
|
||||
std::string ctx_test::skip_reason(model_run & mr) const {
|
||||
// the recurrent cases need a recurrent cache, the KV cases work with any model
|
||||
if (group() == TEST_GROUP_RS && !mr.is_recurrent()) {
|
||||
return "not a recurrent model";
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
const std::vector<ctx_test_ptr> & test_cases() {
|
||||
static const std::vector<ctx_test_ptr> cases = [] {
|
||||
std::vector<ctx_test_ptr> res;
|
||||
|
||||
for (const auto & maker : { make_kv_cases, make_rs_cases }) {
|
||||
auto part = maker();
|
||||
res.insert(res.end(), std::make_move_iterator(part.begin()), std::make_move_iterator(part.end()));
|
||||
}
|
||||
|
||||
return res;
|
||||
}();
|
||||
|
||||
return cases;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
#pragma once
|
||||
|
||||
#include "test-util.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
//
|
||||
// the interface of a single context state case
|
||||
//
|
||||
|
||||
enum class test_status {
|
||||
PASS,
|
||||
FAIL,
|
||||
SKIP,
|
||||
};
|
||||
|
||||
// the verdict as printed in the result table, colored
|
||||
const char * test_status_str(test_status status);
|
||||
|
||||
// how many columns the verdict takes in the table - the escapes of the color are not printed
|
||||
size_t test_status_width(test_status status);
|
||||
|
||||
// a skipped case does not fail the run, a failed case always does
|
||||
test_status merge_status(test_status a, test_status b);
|
||||
|
||||
enum test_group {
|
||||
TEST_GROUP_STATE, // KV state save / load / copy
|
||||
TEST_GROUP_RS, // recurrent state, also run against a dirty cache
|
||||
};
|
||||
|
||||
struct case_result {
|
||||
test_status status = test_status::PASS;
|
||||
std::string reason; // why the case was skipped or failed, empty when it passed
|
||||
|
||||
static case_result pass();
|
||||
static case_result fail(const std::string & why = {});
|
||||
static case_result skip(const std::string & why);
|
||||
};
|
||||
|
||||
struct ctx_test {
|
||||
virtual ~ctx_test() = default;
|
||||
|
||||
// the column header of the --models table, keep it short
|
||||
virtual const char * name() const = 0;
|
||||
|
||||
// one line of what the case verifies, printed by --list
|
||||
virtual const char * description() const { return ""; }
|
||||
|
||||
virtual test_group group() const { return TEST_GROUP_STATE; }
|
||||
|
||||
// the case compares against the logits produced by the baseline case
|
||||
virtual bool needs_baseline() const { return false; }
|
||||
|
||||
// the case is also run against a cache pre-filled with a known pattern, so that a restore
|
||||
// reading cells it never wrote cannot pass
|
||||
virtual bool uses_fill() const { return group() == TEST_GROUP_RS; }
|
||||
|
||||
// non-empty means the case does not apply to this model, the text is the reported reason
|
||||
virtual std::string skip_reason(model_run & mr) const;
|
||||
|
||||
// the scenario, the checks are written with CTX_REQUIRE
|
||||
virtual case_result run(model_run & mr) const = 0;
|
||||
};
|
||||
|
||||
// record an expectation that did not hold and give up on the case - the runner reports the
|
||||
// reason, the body does not print anything
|
||||
#define CTX_REQUIRE(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
return case_result::fail(msg); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
using ctx_test_ptr = std::unique_ptr<const ctx_test>;
|
||||
|
||||
// the cases of one source file, in table order
|
||||
std::vector<ctx_test_ptr> make_kv_cases(); // cases-kv.cpp
|
||||
std::vector<ctx_test_ptr> make_rs_cases(); // cases-rs.cpp
|
||||
|
||||
// all cases, in table order
|
||||
const std::vector<ctx_test_ptr> & test_cases();
|
||||
@@ -0,0 +1,119 @@
|
||||
#include "test-log.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdarg>
|
||||
#include <cstdio>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iterator>
|
||||
|
||||
namespace {
|
||||
|
||||
// a log is printed after the row of its model, what does not fit stays in the file
|
||||
const size_t LOG_DUMP_MAX_BYTES = 64 * 1024;
|
||||
|
||||
// one worker thread runs one model and the log callback fires on the thread that logged the
|
||||
// message, so the file of a model is held by the thread that runs it
|
||||
thread_local FILE * t_file;
|
||||
|
||||
char level_char(enum ggml_log_level level) {
|
||||
switch (level) {
|
||||
case GGML_LOG_LEVEL_DEBUG: return 'D';
|
||||
case GGML_LOG_LEVEL_INFO: return 'I';
|
||||
case GGML_LOG_LEVEL_WARN: return 'W';
|
||||
case GGML_LOG_LEVEL_ERROR: return 'E';
|
||||
case GGML_LOG_LEVEL_CONT: return 'C';
|
||||
default: return '?';
|
||||
}
|
||||
}
|
||||
|
||||
// every line of a model log carries its level, so that a failure can be looked up with a grep
|
||||
void test_log_level(enum ggml_log_level level) {
|
||||
if (level != GGML_LOG_LEVEL_NONE) {
|
||||
fprintf(t_file, "%c ", level_char(level));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void test_log_capture(const std::string & path) {
|
||||
if (t_file) {
|
||||
fclose(t_file);
|
||||
t_file = nullptr;
|
||||
}
|
||||
|
||||
if (path.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
t_file = fopen(path.c_str(), "w");
|
||||
|
||||
if (!t_file) {
|
||||
fprintf(stderr, "%s: failed to open '%s'\n", __func__, path.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
void test_log_callback(enum ggml_log_level level, const char * text, void * user_data) {
|
||||
if (!t_file) {
|
||||
common_log_default_callback(level, text, user_data);
|
||||
return;
|
||||
}
|
||||
|
||||
// gate the capture the way the console would be - the debug chatter of the graph builder is
|
||||
// orders of magnitude more than everything else, a capture never holds less than info
|
||||
if (common_log_get_verbosity(level) <= std::max(common_log_get_verbosity_thold(), LOG_LEVEL_INFO)) {
|
||||
test_log_level(level);
|
||||
fputs(text, t_file);
|
||||
}
|
||||
}
|
||||
|
||||
void test_log_add(enum ggml_log_level level, const char * fmt, ...) {
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
|
||||
if (t_file) {
|
||||
test_log_level(level);
|
||||
vfprintf(t_file, fmt, args);
|
||||
} else if (level == GGML_LOG_LEVEL_ERROR) {
|
||||
// without a capture the run is followed on the console, a detail line has no place there
|
||||
vfprintf(stderr, fmt, args);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
bool test_log_dump(const std::string & path) {
|
||||
std::ifstream f(path, std::ios::binary);
|
||||
if (!f) {
|
||||
fprintf(stderr, "failed to read the captured log '%s'\n", path.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
const std::string text((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
|
||||
f.close();
|
||||
|
||||
size_t skip = 0;
|
||||
if (text.size() > LOG_DUMP_MAX_BYTES) {
|
||||
skip = text.find('\n', text.size() - LOG_DUMP_MAX_BYTES);
|
||||
skip = skip == std::string::npos ? 0 : skip + 1;
|
||||
}
|
||||
|
||||
if (skip > 0) {
|
||||
const size_t n_all = std::count(text.begin(), text.end(), '\n');
|
||||
const size_t n_shown = std::count(text.begin() + skip, text.end(), '\n');
|
||||
|
||||
fprintf(stdout, "... %zu of %zu lines omitted, full log: %s\n", n_all - n_shown, n_all, path.c_str());
|
||||
}
|
||||
|
||||
fwrite(text.data() + skip, 1, text.size() - skip, stdout);
|
||||
fflush(stdout);
|
||||
|
||||
return skip > 0;
|
||||
}
|
||||
|
||||
std::string test_log_path(const std::string & logs_dir, const std::string & model_name) {
|
||||
const std::string file = "test-llama-context." + model_name + ".log";
|
||||
|
||||
return logs_dir.empty() ? file : (std::filesystem::path(logs_dir) / file).string();
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
#pragma once
|
||||
|
||||
#include "log.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
//
|
||||
// the log of a model goes to a file of its own, so that a model which failed can be read on its
|
||||
// own. a worker thread runs one model and the log callback fires on the thread that produced the
|
||||
// message, so a capture belongs to the thread that opened it
|
||||
//
|
||||
|
||||
// open the log of this thread, an empty path closes it - a capture holds what the verbosity of
|
||||
// the process allows, never less than info, the details of a failure are looked for in the file
|
||||
void test_log_capture(const std::string & path);
|
||||
|
||||
// llama, ggml and the backends log through this: into the file of this thread when it captures,
|
||||
// into the default callback otherwise
|
||||
void test_log_callback(enum ggml_log_level level, const char * text, void * user_data);
|
||||
|
||||
// the lines of the test itself go to the file of this thread, an error goes to stderr as well when
|
||||
// there is no capture and the run is followed on the console
|
||||
void test_log_add(enum ggml_log_level level, const char * fmt, ...) LOG_ATTRIBUTE_FORMAT(2, 3);
|
||||
|
||||
// print a captured log, false when it was printed in full and the file can be removed
|
||||
bool test_log_dump(const std::string & path);
|
||||
|
||||
std::string test_log_path(const std::string & logs_dir, const std::string & model_name);
|
||||
|
||||
// a trace is a detail of a case, it is there to read in the file of the model
|
||||
#define TLOG_ERR(...) test_log_add(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
|
||||
#define TLOG_INF(...) test_log_add(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
|
||||
#define TLOG_TRC(...) test_log_add(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
|
||||
@@ -0,0 +1,117 @@
|
||||
#include "test-runner.h"
|
||||
|
||||
#include "test-log.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <regex>
|
||||
#include <utility>
|
||||
|
||||
std::vector<const ctx_test *> select_cases(const std::string & filter) {
|
||||
std::vector<const ctx_test *> selected;
|
||||
|
||||
if (filter.empty()) {
|
||||
selected.reserve(test_cases().size());
|
||||
for (const auto & tc : test_cases()) {
|
||||
selected.push_back(tc.get());
|
||||
}
|
||||
|
||||
return selected;
|
||||
}
|
||||
|
||||
const std::regex re(filter);
|
||||
|
||||
for (const auto & tc : test_cases()) {
|
||||
if (std::regex_match(tc->name(), re)) {
|
||||
selected.push_back(tc.get());
|
||||
}
|
||||
}
|
||||
|
||||
return selected;
|
||||
}
|
||||
|
||||
std::vector<case_run> run_cases(model_run & mr, const std::vector<const ctx_test *> & cases, bool report) {
|
||||
// the state cases keep the historical default of a unified KV cache when running a single
|
||||
// sequence, the recurrent cases use whatever was passed on the command line
|
||||
const bool unified_base = mr.params.kv_unified;
|
||||
const bool unified_state = mr.params.n_parallel == 1 ? true : unified_base;
|
||||
|
||||
size_t name_width = 4; // "case"
|
||||
for (const ctx_test * tc : cases) {
|
||||
name_width = std::max(name_width, strlen(tc->name()));
|
||||
}
|
||||
|
||||
std::vector<case_run> runs;
|
||||
runs.reserve(cases.size());
|
||||
|
||||
for (const ctx_test * tc : cases) {
|
||||
mr.params.kv_unified = tc->group() == TEST_GROUP_STATE ? unified_state : unified_base;
|
||||
|
||||
case_run run;
|
||||
run.name = tc->name();
|
||||
|
||||
// the log of a model is read on its own, mark where each case begins
|
||||
TLOG_INF("%s\n", run.name.c_str());
|
||||
|
||||
// a case that throws is a case that failed, and the run goes on with the next one
|
||||
try {
|
||||
if (const std::string why = tc->skip_reason(mr); !why.empty()) {
|
||||
run.status = test_status::SKIP;
|
||||
run.reason = why;
|
||||
} else if (tc->needs_baseline() && !mr.have_baseline) {
|
||||
run.status = test_status::SKIP;
|
||||
run.reason = "no baseline";
|
||||
} else if (!tc->uses_fill()) {
|
||||
mr.fill = 0;
|
||||
|
||||
const case_result res = tc->run(mr);
|
||||
|
||||
run.status = res.status;
|
||||
run.reason = res.reason;
|
||||
} else {
|
||||
// the recurrent cases are also run against a cache pre-filled with a known
|
||||
// pattern, so that a restore reading cells it never wrote cannot pass
|
||||
|
||||
for (const uint8_t fill : { 0, 0x3e }) {
|
||||
TLOG_INF("(fill 0x%02x)\n", fill);
|
||||
|
||||
mr.fill = fill;
|
||||
|
||||
const case_result res = tc->run(mr);
|
||||
|
||||
run.status = merge_status(run.status, res.status);
|
||||
if (run.reason.empty()) {
|
||||
run.reason = res.reason;
|
||||
}
|
||||
if (res.status == test_status::FAIL) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (...) {
|
||||
run.status = test_status::FAIL;
|
||||
run.reason = "unhandled exception";
|
||||
}
|
||||
|
||||
if (unified_state) {
|
||||
mr.params.kv_unified = unified_base;
|
||||
}
|
||||
|
||||
// the verdict is one word, what it means goes to the log of the model
|
||||
if (!run.reason.empty()) {
|
||||
TLOG_INF("%s %s: %s\n", run.name.c_str(),
|
||||
run.status == test_status::FAIL ? "failed" : "skipped", run.reason.c_str());
|
||||
}
|
||||
|
||||
if (report) {
|
||||
fprintf(stderr, "%-*s %s%s%s\n", (int) name_width, run.name.c_str(),
|
||||
test_status_str(run.status),
|
||||
run.reason.empty() ? "" : " ", run.reason.c_str());
|
||||
}
|
||||
|
||||
runs.push_back(std::move(run));
|
||||
}
|
||||
|
||||
return runs;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
|
||||
#include "test-case.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
//
|
||||
// the runner: the single place that decides how a case is executed
|
||||
//
|
||||
|
||||
struct case_run {
|
||||
std::string name;
|
||||
test_status status = test_status::SKIP;
|
||||
std::string reason;
|
||||
};
|
||||
|
||||
// the cases selected by the filter, all of them when it is empty, in table order
|
||||
std::vector<const ctx_test *> select_cases(const std::string & filter);
|
||||
|
||||
// run the selected cases for one model and return one verdict per case - with report, print the
|
||||
// verdict of every case as it finishes, for single model mode
|
||||
std::vector<case_run> run_cases(model_run & mr, const std::vector<const ctx_test *> & cases, bool report);
|
||||
@@ -0,0 +1,403 @@
|
||||
#include "test-util.h"
|
||||
|
||||
#include "test-log.h"
|
||||
|
||||
#include "../../src/llama-ext.h" // llama_memory_get_buffers
|
||||
|
||||
#include "common.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
|
||||
const double NMSE_THRESHOLD = 1e-5;
|
||||
|
||||
//
|
||||
// comparison
|
||||
//
|
||||
|
||||
// the distance of two logits, infinite when either of them is not finite
|
||||
static float logit_diff(float a, float b) {
|
||||
return std::isfinite(a) && std::isfinite(b) ? std::fabs(a - b) : std::numeric_limits<float>::infinity();
|
||||
}
|
||||
|
||||
void logits_diff::add(const float * a, const float * b, size_t n_elem) {
|
||||
for (size_t i = 0; i < n_elem; i++) {
|
||||
const float diff = logit_diff(a[i], b[i]);
|
||||
|
||||
if (diff > 0.0f && i_first < 0) {
|
||||
i_first = (int64_t) (n + i);
|
||||
}
|
||||
|
||||
if (diff > diff_max) {
|
||||
diff_max = diff;
|
||||
}
|
||||
|
||||
if (std::isfinite(a[i]) && std::isfinite(b[i])) {
|
||||
const double d = (double) a[i] - (double) b[i];
|
||||
|
||||
mse_ab += d*d;
|
||||
mse_a0 += (double) a[i] * (double) a[i];
|
||||
} else {
|
||||
n_nan++;
|
||||
mse_ab = std::numeric_limits<double>::infinity();
|
||||
mse_a0 = 1.0;
|
||||
}
|
||||
}
|
||||
|
||||
n += n_elem;
|
||||
}
|
||||
|
||||
double logits_diff::nmse() const {
|
||||
if (mse_a0 == 0.0) {
|
||||
return mse_ab == 0.0 ? 0.0 : std::numeric_limits<double>::infinity();
|
||||
}
|
||||
|
||||
return mse_ab / mse_a0;
|
||||
}
|
||||
|
||||
logits_diff compare_logits(const float * a, const float * b, size_t n) {
|
||||
logits_diff res;
|
||||
res.add(a, b, n);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
//
|
||||
// decode
|
||||
//
|
||||
|
||||
bool get_current_logits(llama_context * ctx, std::vector<float> & out) {
|
||||
const auto * vocab = llama_model_get_vocab(llama_get_model(ctx));
|
||||
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
|
||||
const float * logits = llama_get_logits_ith(ctx, -1);
|
||||
if (logits == nullptr) {
|
||||
return false;
|
||||
}
|
||||
out.assign(logits, logits + n_vocab);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool decode_seq(llama_context * ctx, const llama_tokens & tokens, llama_pos p_begin, llama_pos p_end,
|
||||
llama_seq_id seq_id, bool last_output) {
|
||||
common_batch batch(ctx);
|
||||
for (llama_pos pos = p_begin; pos < p_end; ++pos) {
|
||||
batch.add(tokens[pos], pos, seq_id, last_output && pos + 1 == p_end);
|
||||
}
|
||||
|
||||
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
|
||||
}
|
||||
|
||||
bool decode_tokens(llama_context * ctx, const llama_tokens & tokens, llama_seq_id seq_id) {
|
||||
return decode_seq(ctx, tokens, 0, (llama_pos) tokens.size(), seq_id, true);
|
||||
}
|
||||
|
||||
llama_token gen_token(int32_t n_vocab, llama_seq_id seq_id, llama_pos pos) {
|
||||
return (llama_token) ((7*(uint32_t) pos + 31*(uint32_t) seq_id + 1) % (uint32_t) n_vocab);
|
||||
}
|
||||
|
||||
bool decode_gen(llama_context * ctx, int32_t n_vocab, llama_seq_id seq_id, llama_pos p_begin, llama_pos p_end) {
|
||||
common_batch batch(ctx);
|
||||
for (llama_pos pos = p_begin; pos < p_end; ++pos) {
|
||||
batch.add(gen_token(n_vocab, seq_id, pos), pos, seq_id, false);
|
||||
}
|
||||
|
||||
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
|
||||
}
|
||||
|
||||
bool decode_one(llama_context * ctx, llama_token tok, llama_pos pos, llama_seq_id seq_id) {
|
||||
common_batch batch(ctx);
|
||||
batch.add(tok, pos, seq_id, true);
|
||||
|
||||
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
|
||||
}
|
||||
|
||||
bool generate_tokens(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id,
|
||||
generation_result & result) {
|
||||
common_batch batch(ctx);
|
||||
|
||||
// the tokens of the run as a single line - one entry per token would leave the next message
|
||||
// of the log glued to the last one
|
||||
std::string tokens;
|
||||
|
||||
for (int32_t i = 0; i < n_predict; i++) {
|
||||
std::vector<float> logits;
|
||||
if (!get_current_logits(ctx, logits)) {
|
||||
TLOG_ERR("%s: failed to get logits\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto next_token = llama_sampler_sample(smpl, ctx, -1);
|
||||
|
||||
tokens += std::to_string(next_token) + " ";
|
||||
result.tokens.push_back(next_token);
|
||||
result.logits.push_back(std::move(logits));
|
||||
|
||||
batch.clear();
|
||||
batch.add(next_token, n_past, seq_id, true);
|
||||
|
||||
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
TLOG_ERR("%s: failed to evaluate\n", __func__);
|
||||
return false;
|
||||
}
|
||||
n_past++;
|
||||
}
|
||||
|
||||
TLOG_INF("%s: %s\n", __func__, tokens.c_str());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool generate_tokens_compare(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id,
|
||||
const generation_result & expected, double nmse_eps) {
|
||||
if (expected.tokens.size() != expected.logits.size() || expected.tokens.size() < (size_t) n_predict) {
|
||||
TLOG_ERR("%s: invalid expected generation\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
common_batch batch(ctx);
|
||||
|
||||
std::string tokens;
|
||||
|
||||
for (int32_t i = 0; i < n_predict; i++) {
|
||||
std::vector<float> logits;
|
||||
if (!get_current_logits(ctx, logits)) {
|
||||
TLOG_ERR("%s: failed to get logits\n", __func__);
|
||||
return false;
|
||||
}
|
||||
if (logits.size() != expected.logits[i].size()) {
|
||||
TLOG_ERR("%s: logits size mismatch at step %d: %zu != %zu\n", __func__, i, logits.size(), expected.logits[i].size());
|
||||
return false;
|
||||
}
|
||||
|
||||
const logits_diff diff = compare_logits(expected.logits[i].data(), logits.data(), logits.size());
|
||||
|
||||
TLOG_TRC("%s: step %d nmse = %.6e\n", __func__, i, diff.nmse());
|
||||
if (diff.nmse() > nmse_eps) {
|
||||
TLOG_ERR("%s: error: NMSE at step %d is %.6e (threshold %.1e)\n", __func__, i, diff.nmse(), nmse_eps);
|
||||
return false;
|
||||
}
|
||||
|
||||
const auto next_token = llama_sampler_sample(smpl, ctx, -1);
|
||||
const auto expected_token = expected.tokens[i];
|
||||
|
||||
tokens += std::to_string(next_token) + " ";
|
||||
if (next_token != expected_token) {
|
||||
TLOG_TRC("%s: sampled token %d differs from expected %d, using expected token\n", __func__, next_token, expected_token);
|
||||
}
|
||||
|
||||
batch.clear();
|
||||
batch.add(expected_token, n_past, seq_id, true);
|
||||
|
||||
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
TLOG_ERR("%s: failed to evaluate\n", __func__);
|
||||
return false;
|
||||
}
|
||||
n_past++;
|
||||
}
|
||||
|
||||
TLOG_INF("%s: %s\n", __func__, tokens.c_str());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// state
|
||||
//
|
||||
|
||||
bool get_seq_state(llama_context * ctx, llama_seq_id seq_id, uint32_t flags, std::vector<uint8_t> & state) {
|
||||
const size_t state_size = llama_state_seq_get_size_ext(ctx, seq_id, flags);
|
||||
if (state_size == 0) {
|
||||
TLOG_ERR("%s: sequence state is empty\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
state.resize(state_size);
|
||||
const size_t ncopy = llama_state_seq_get_data_ext(ctx, state.data(), state.size(), seq_id, flags);
|
||||
if (ncopy != state.size()) {
|
||||
TLOG_ERR("%s: sequence state length %zu does not match expected length %zu\n",
|
||||
__func__, ncopy, state.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool set_seq_state(llama_context * ctx, llama_seq_id seq_id, uint32_t flags, const std::vector<uint8_t> & state) {
|
||||
const size_t nset = llama_state_seq_set_data_ext(ctx, state.data(), state.size(), seq_id, flags);
|
||||
if (nset != state.size()) {
|
||||
TLOG_ERR("%s: seq set data length %zu does not match expected length %zu\n", __func__, nset, state.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool seq_rm(llama_context * ctx, llama_seq_id seq_id) {
|
||||
if (!llama_memory_seq_rm(llama_get_memory(ctx), seq_id, -1, -1)) {
|
||||
TLOG_ERR("%s: failed to remove sequence %d\n", __func__, seq_id);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool load_state_replay_last(llama_context * ctx, const std::string & path, const llama_tokens & tokens, int & n_past) {
|
||||
llama_tokens unused_sts(tokens.size());
|
||||
size_t n_token_count_out = 0;
|
||||
|
||||
if (!llama_state_load_file(ctx, path.c_str(), unused_sts.data(), unused_sts.size(), &n_token_count_out)) {
|
||||
TLOG_ERR("%s: failed to load state\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
TLOG_TRC("%s: loaded state with %zu tokens\n", __func__, n_token_count_out);
|
||||
|
||||
// the session is saved before the last token, so replay it at the position it was saved at
|
||||
n_past = (int) n_token_count_out - 1;
|
||||
if (!common_replay_last_token(ctx, tokens.back(), n_past)) {
|
||||
return false;
|
||||
}
|
||||
n_past++;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool blobs_equal(const std::vector<uint8_t> & a, const std::vector<uint8_t> & b) {
|
||||
if (a.size() == b.size() && (a.empty() || memcmp(a.data(), b.data(), a.size()) == 0)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t i = 0;
|
||||
while (i < a.size() && i < b.size() && a[i] == b[i]) {
|
||||
++i;
|
||||
}
|
||||
|
||||
TLOG_ERR("%s: %zu bytes -> %zu bytes, first difference at byte %zu (%02x -> %02x)\n",
|
||||
__func__, a.size(), b.size(), i,
|
||||
i < a.size() ? a[i] : 0,
|
||||
i < b.size() ? b[i] : 0);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool corrupt_state(std::vector<uint8_t> & data) {
|
||||
if (data.size() < 3*4096) {
|
||||
TLOG_ERR("%s: state of %zu bytes is too small to corrupt\n", __func__, data.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
std::fill(data.begin() + 4096, data.end() - data.size()/4, 0xff);
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// context
|
||||
//
|
||||
|
||||
// fill the buffers of the cache before a restore - a restore that reads cells it never wrote
|
||||
// then produces garbage instead of silently passing
|
||||
llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill) {
|
||||
llama_context_ptr ctx{llama_init_from_model(model, cparams)};
|
||||
if (!ctx || fill == 0) {
|
||||
return ctx;
|
||||
}
|
||||
|
||||
// Use a full ubatch so buffer discovery preserves prefill allocation sizes.
|
||||
const uint32_t n_tokens = llama_n_ubatch(ctx.get());
|
||||
if (!decode_tokens(ctx.get(), llama_tokens(n_tokens, 0))) {
|
||||
return nullptr;
|
||||
}
|
||||
llama_synchronize(ctx.get());
|
||||
|
||||
const auto buffers = llama_memory_get_buffers(llama_get_memory(ctx.get()));
|
||||
llama_memory_clear(llama_get_memory(ctx.get()), true);
|
||||
if (buffers.empty()) {
|
||||
TLOG_ERR("%s: no cache buffers found\n", __func__);
|
||||
return nullptr;
|
||||
}
|
||||
for (auto * buffer : buffers) {
|
||||
ggml_backend_buffer_clear(buffer, fill);
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
llama_context_ptr make_ctx(llama_model * model, const common_params & params, const ctx_opts & opts) {
|
||||
auto cparams = common_context_params_to_llama(params);
|
||||
|
||||
if (opts.n_ctx) {
|
||||
cparams.n_ctx = opts.n_ctx;
|
||||
}
|
||||
if (opts.n_seq_max) {
|
||||
cparams.n_seq_max = opts.n_seq_max;
|
||||
}
|
||||
if (opts.kv_unified >= 0) {
|
||||
cparams.kv_unified = opts.kv_unified != 0;
|
||||
}
|
||||
if (opts.n_rs_seq) {
|
||||
cparams.n_rs_seq = opts.n_rs_seq;
|
||||
cparams.n_batch = std::max(cparams.n_batch, (uint32_t) (opts.n_rs_seq + 1));
|
||||
cparams.n_ubatch = std::max(cparams.n_ubatch, (uint32_t) (opts.n_rs_seq + 1));
|
||||
}
|
||||
if (opts.n_batch) {
|
||||
cparams.n_batch = opts.n_batch;
|
||||
}
|
||||
if (opts.n_ubatch) {
|
||||
cparams.n_ubatch = opts.n_ubatch;
|
||||
}
|
||||
if (opts.fa != LLAMA_FLASH_ATTN_TYPE_AUTO) {
|
||||
cparams.flash_attn_type = opts.fa;
|
||||
}
|
||||
if (opts.type_k != GGML_TYPE_COUNT) {
|
||||
cparams.type_k = opts.type_k;
|
||||
}
|
||||
if (opts.type_v != GGML_TYPE_COUNT) {
|
||||
cparams.type_v = opts.type_v;
|
||||
}
|
||||
|
||||
return init_ctx(model, cparams, opts.fill);
|
||||
}
|
||||
|
||||
llama_sampler_ptr make_dist_sampler(uint32_t seed) {
|
||||
auto sparams = llama_sampler_chain_default_params();
|
||||
llama_sampler_ptr smpl{llama_sampler_chain_init(sparams)};
|
||||
llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(seed));
|
||||
return smpl;
|
||||
}
|
||||
|
||||
bool arch_reserves_single_seq(const llama_model * model) {
|
||||
char arch_str[64] = {};
|
||||
llama_model_meta_val_str(model, "general.architecture", arch_str, sizeof(arch_str));
|
||||
|
||||
return strcmp(arch_str, "kimi-linear") == 0 || strcmp(arch_str, "minimax-01") == 0;
|
||||
}
|
||||
|
||||
//
|
||||
// model_run
|
||||
//
|
||||
|
||||
llama_context_ptr model_run::ctx(const ctx_opts & opts) const {
|
||||
ctx_opts effective = opts;
|
||||
effective.fill = fill;
|
||||
|
||||
auto ctx = make_ctx(model, params, effective);
|
||||
if (!ctx && !opts.quiet) {
|
||||
TLOG_ERR("%s: failed to create context for '%s'\n", __func__, name.c_str());
|
||||
}
|
||||
|
||||
return ctx;
|
||||
}
|
||||
|
||||
llama_sampler_ptr model_run::sampler() const {
|
||||
return make_dist_sampler(params.sampling.seed);
|
||||
}
|
||||
|
||||
bool model_run::is_recurrent() const {
|
||||
return llama_model_is_recurrent(model) || llama_model_is_hybrid(model);
|
||||
}
|
||||
|
||||
bool model_run::has_vocab() const {
|
||||
return llama_vocab_type(llama_model_get_vocab(model)) != LLAMA_VOCAB_TYPE_NONE;
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
#pragma once
|
||||
|
||||
#include "llama-cpp.h"
|
||||
#include "llama.h"
|
||||
|
||||
#include "common.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
//
|
||||
// shared helpers of the context state cases
|
||||
//
|
||||
|
||||
extern const double NMSE_THRESHOLD;
|
||||
|
||||
//
|
||||
// comparison
|
||||
//
|
||||
|
||||
// normalized mean squared error = mse(a, b) / mse(a, 0), infinity if any logit is not finite
|
||||
|
||||
// accumulates logit comparisons and derives their NMSE, so that a case can compare an entire
|
||||
// multi-seq batch row by row and still have a single verdict
|
||||
struct logits_diff {
|
||||
double mse_ab = 0.0;
|
||||
double mse_a0 = 0.0;
|
||||
float diff_max = 0.0f;
|
||||
size_t n_nan = 0;
|
||||
size_t n = 0;
|
||||
int64_t i_first = -1; // flat index of the first differing element
|
||||
|
||||
void add(const float * a, const float * b, size_t n_elem);
|
||||
|
||||
double nmse() const;
|
||||
};
|
||||
|
||||
logits_diff compare_logits(const float * a, const float * b, size_t n);
|
||||
|
||||
//
|
||||
// generation
|
||||
//
|
||||
|
||||
// logits produced by a run, compared against by the replay cases
|
||||
struct generation_result {
|
||||
llama_tokens tokens;
|
||||
std::vector<std::vector<float>> logits;
|
||||
|
||||
bool empty() const { return tokens.empty(); }
|
||||
};
|
||||
|
||||
//
|
||||
// decode
|
||||
//
|
||||
|
||||
bool get_current_logits(llama_context * ctx, std::vector<float> & out);
|
||||
|
||||
// decode tokens [p_begin, p_end) of one sequence
|
||||
bool decode_seq(llama_context * ctx, const llama_tokens & tokens, llama_pos p_begin, llama_pos p_end,
|
||||
llama_seq_id seq_id, bool last_output);
|
||||
|
||||
bool decode_tokens(llama_context * ctx, const llama_tokens & tokens, llama_seq_id seq_id = 0);
|
||||
|
||||
// tokens derived from (seq, pos): the recurrent cases do not use the tokenizer, so they do
|
||||
// not depend on the prompt or on the prompt length
|
||||
llama_token gen_token(int32_t n_vocab, llama_seq_id seq_id, llama_pos pos);
|
||||
|
||||
bool decode_gen(llama_context * ctx, int32_t n_vocab, llama_seq_id seq_id, llama_pos p_begin, llama_pos p_end);
|
||||
|
||||
bool decode_one(llama_context * ctx, llama_token tok, llama_pos pos, llama_seq_id seq_id = 0);
|
||||
|
||||
// greedy-free replay: sample for reporting, but feed the sampled tokens back
|
||||
bool generate_tokens(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id,
|
||||
generation_result & result);
|
||||
|
||||
// as generate_tokens, but feeds the expected tokens back so that the compared logits stay on
|
||||
// the reference trajectory
|
||||
bool generate_tokens_compare(llama_context * ctx, llama_sampler * smpl, int & n_past, int32_t n_predict, llama_seq_id seq_id,
|
||||
const generation_result & expected, double nmse_eps);
|
||||
|
||||
//
|
||||
// state
|
||||
//
|
||||
|
||||
// llama_state_seq_get_size_ext + get_data_ext, with the length verified
|
||||
bool get_seq_state(llama_context * ctx, llama_seq_id seq_id, uint32_t flags, std::vector<uint8_t> & state);
|
||||
|
||||
// llama_state_seq_set_data_ext, with the length verified
|
||||
bool set_seq_state(llama_context * ctx, llama_seq_id seq_id, uint32_t flags, const std::vector<uint8_t> & state);
|
||||
|
||||
// drop a sequence entirely
|
||||
bool seq_rm(llama_context * ctx, llama_seq_id seq_id);
|
||||
|
||||
// load a saved session and replay its last token, leaving n_past ready for the next token
|
||||
bool load_state_replay_last(llama_context * ctx, const std::string & path, const llama_tokens & tokens, int & n_past);
|
||||
|
||||
// false when the sizes differ or any byte differs, the first difference is logged
|
||||
bool blobs_equal(const std::vector<uint8_t> & a, const std::vector<uint8_t> & b);
|
||||
|
||||
// overwrite the tensor data with 0xff bytes (NaN when read as f16/f32), so that the restore fails
|
||||
bool corrupt_state(std::vector<uint8_t> & data);
|
||||
|
||||
//
|
||||
// context
|
||||
//
|
||||
|
||||
// per-test context overrides, sentinel values mean "take the value from common_params"
|
||||
struct ctx_opts {
|
||||
uint32_t n_ctx = 0; // 0 = from params
|
||||
uint32_t n_batch = 0; // 0 = from params
|
||||
uint32_t n_ubatch = 0; // 0 = from params
|
||||
uint32_t n_seq_max = 0; // 0 = from params
|
||||
uint32_t n_rs_seq = 0; // 0 = from params, n_batch / n_ubatch are raised to fit it
|
||||
int kv_unified = -1; // -1 = from params, 0 = off, 1 = on
|
||||
|
||||
llama_flash_attn_type fa = LLAMA_FLASH_ATTN_TYPE_AUTO; // AUTO = from params
|
||||
|
||||
ggml_type type_k = GGML_TYPE_COUNT; // COUNT = from params
|
||||
ggml_type type_v = GGML_TYPE_COUNT; // COUNT = from params
|
||||
|
||||
uint8_t fill = 0; // fill the cache buffers with this byte, 0 = disabled
|
||||
|
||||
// a probe that expects the context to be refused should not report it as an error
|
||||
bool quiet = false;
|
||||
};
|
||||
|
||||
llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill);
|
||||
llama_context_ptr make_ctx(llama_model * model, const common_params & params, const ctx_opts & opts);
|
||||
|
||||
llama_sampler_ptr make_dist_sampler(uint32_t seed);
|
||||
|
||||
// these archs reserve the final pp graph with n_seqs = 1, so any multi-seq graph has a
|
||||
// different layout and re-reserves by design - they must not be run with a multi-seq batch
|
||||
// when GGML_SCHED_DEBUG_REALLOC is enabled
|
||||
// see [TAG_RESERVE_DIAG_DECAY] in llama-context.cpp
|
||||
bool arch_reserves_single_seq(const llama_model * model);
|
||||
|
||||
//
|
||||
// the state of a single model run, handed to every case
|
||||
//
|
||||
|
||||
struct model_run {
|
||||
common_params params; // per-model copy, out_file is scoped to this model
|
||||
std::string name; // model file name, used for the table and for temp files
|
||||
llama_model * model = nullptr;
|
||||
llama_tokens tokens;
|
||||
generation_result baseline;
|
||||
bool have_baseline = false;
|
||||
uint8_t fill = 0; // set by the runner before each case
|
||||
|
||||
// a context with these options, the fill of this run applied; nullptr when it failed to
|
||||
// init, the reason is logged
|
||||
llama_context_ptr ctx(const ctx_opts & opts = {}) const;
|
||||
|
||||
// a seeded distribution sampler
|
||||
llama_sampler_ptr sampler() const;
|
||||
|
||||
bool is_recurrent() const;
|
||||
bool has_vocab() const;
|
||||
};
|
||||
@@ -0,0 +1,37 @@
|
||||
#include "simple-tokenize.h"
|
||||
|
||||
std::vector<std::string> simple_tokenize(const std::string & input) {
|
||||
std::vector<std::string> result;
|
||||
std::string current;
|
||||
|
||||
for (size_t i = 0; i < input.size(); i++) {
|
||||
switch (input[i]) {
|
||||
case ' ':
|
||||
case '\n':
|
||||
case '\t':
|
||||
case '{':
|
||||
case '}':
|
||||
case ',':
|
||||
case '[':
|
||||
case '"':
|
||||
case ']':
|
||||
case '.':
|
||||
case '<':
|
||||
case '>':
|
||||
case '=':
|
||||
case '/':
|
||||
if (!current.empty()) {
|
||||
result.push_back(current);
|
||||
current.clear();
|
||||
}
|
||||
default:;
|
||||
}
|
||||
current += input[i];
|
||||
}
|
||||
|
||||
if (!current.empty()) {
|
||||
result.push_back(current);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
std::vector<std::string> simple_tokenize(const std::string &);
|
||||
@@ -0,0 +1,471 @@
|
||||
#include "peg-parser.h"
|
||||
#include "tests.h"
|
||||
|
||||
void test_basic(testing & t) {
|
||||
t.test("chars", [](testing & t) {
|
||||
// Test common escape sequences - newline
|
||||
t.test("escape_sequence_newline", [](testing &t) {
|
||||
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("\n");
|
||||
result = common_chat_combinator_parser.parse(ctx);
|
||||
t.assert_equal("escape_sequence_newline", true, result.success());
|
||||
});
|
||||
|
||||
// Test common escape sequences - tab
|
||||
t.test("escape_sequence_tab", [](testing &t) {
|
||||
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("\t");
|
||||
result = common_chat_combinator_parser.parse(ctx);
|
||||
t.assert_equal("escape_sequence_tab", true, result.success());
|
||||
});
|
||||
|
||||
// Test common escape sequences - backslash
|
||||
t.test("escape_sequence_backslash", [](testing &t) {
|
||||
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("\\");
|
||||
result = common_chat_combinator_parser.parse(ctx);
|
||||
t.assert_equal("escape_sequence_backslash", true, result.success());
|
||||
});
|
||||
|
||||
// Test common escape sequences - space (should ())
|
||||
t.test("escape_sequence_space_fail", [](testing &t) {
|
||||
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context(" ");
|
||||
result = common_chat_combinator_parser.parse(ctx);
|
||||
t.assert_equal("escape_sequence_space_fail", true, result.fail());
|
||||
});
|
||||
|
||||
// Test escaped dash - 'a' should succeed
|
||||
t.test("escaped_dash_a", [](testing &t) {
|
||||
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("a");
|
||||
result = common_chat_combinator_parser.parse(ctx);
|
||||
t.assert_equal("escaped_dash_a", true, result.success());
|
||||
});
|
||||
|
||||
// Test escaped dash - '-' should succeed (literal dash)
|
||||
t.test("escaped_dash_literal", [](testing &t) {
|
||||
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("-");
|
||||
result = common_chat_combinator_parser.parse(ctx);
|
||||
t.assert_equal("escaped_dash_literal", true, result.success());
|
||||
});
|
||||
|
||||
// Test escaped dash - 'z' should succeed
|
||||
t.test("escaped_dash_z", [](testing &t) {
|
||||
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("z");
|
||||
result = common_chat_combinator_parser.parse(ctx);
|
||||
t.assert_equal("escaped_dash_z", true, result.success());
|
||||
});
|
||||
|
||||
// Test escaped dash - 'b' should NOT match (since \- is literal dash, not range)
|
||||
t.test("escaped_dash_b_fail", [](testing &t) {
|
||||
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("b");
|
||||
result = common_chat_combinator_parser.parse(ctx);
|
||||
t.assert_equal("escaped_dash_b_fail", true, result.fail());
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
t.test("optional", [](testing & t) {
|
||||
// Full match with optional part present
|
||||
t.test("optional_present", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("hello") + p.optional(p.literal(" world"));
|
||||
});
|
||||
|
||||
auto ctx = common_peg_parse_context("hello world");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("optional_present", true, result.success());
|
||||
t.assert_equal("optional_present_end", 11u, result.end);
|
||||
});
|
||||
|
||||
// Full match with optional part absent
|
||||
t.test("optional_absent", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("hello") + p.optional(p.literal(" world"));
|
||||
});
|
||||
|
||||
auto ctx = common_peg_parse_context("hello");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("optional_absent", true, result.success());
|
||||
t.assert_equal("optional_absent_end", 5u, result.end);
|
||||
});
|
||||
|
||||
// Partial match - waiting for more input to determine if optional matches
|
||||
t.test("partial_match_need_more", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("hello") + p.optional(p.literal(" world"));
|
||||
});
|
||||
|
||||
auto ctx = common_peg_parse_context("hello ", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("partial_match_need_more", true, result.need_more_input());
|
||||
});
|
||||
});
|
||||
|
||||
t.test("partial parsing", [](testing & t) {
|
||||
// Literals - Basic Success
|
||||
t.test("literal_success", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("hello");
|
||||
result = parser.parse(ctx);
|
||||
t.assert_equal("literal_success", true, result.success());
|
||||
});
|
||||
|
||||
// Char Classes - Basic Lowercase Success
|
||||
t.test("char_class_lowercase_success", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("a");
|
||||
result = parser.parse(ctx);
|
||||
t.assert_equal("char_class_lowercase_success", true, result.success());
|
||||
});
|
||||
|
||||
// Char Classes - Uppercase Fail
|
||||
t.test("char_class_uppercase_fail", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("A");
|
||||
result = parser.parse(ctx);
|
||||
t.assert_equal("char_class_uppercase_fail", true, result.fail());
|
||||
});
|
||||
|
||||
// Char Classes with Dash - Lowercase Success
|
||||
t.test("char_class_with_dash_lowercase", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("f");
|
||||
result = parser.parse(ctx);
|
||||
t.assert_equal("char_class_with_dash_lowercase", true, result.success());
|
||||
});
|
||||
|
||||
// Char Classes with Dash - Literal Dash Success
|
||||
t.test("char_class_with_dash_literal_dash", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("-");
|
||||
result = parser.parse(ctx);
|
||||
t.assert_equal("char_class_with_dash_literal_dash", true, result.success());
|
||||
});
|
||||
|
||||
// Char Classes with Dash - Uppercase Fail
|
||||
t.test("char_class_with_dash_uppercase_fail", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });
|
||||
|
||||
common_peg_parse_context ctx;
|
||||
common_peg_parse_result result;
|
||||
|
||||
ctx = common_peg_parse_context("A");
|
||||
result = parser.parse(ctx);
|
||||
t.assert_equal("char_class_with_dash_uppercase_fail", true, result.fail());
|
||||
});
|
||||
|
||||
// Sequences - Partial Match 1
|
||||
t.test("sequence_partial_match_1", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("<thi", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("sequence_partial_match_1", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Sequences - Partial Match 2
|
||||
t.test("sequence_partial_match_2", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("begin") + p.literal("end"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("begin", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("sequence_partial_match_2", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Sequences - Partial Match 3
|
||||
t.test("sequence_partial_match_3", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("<think></", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("sequence_partial_match_3", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Sequences - Full Match
|
||||
t.test("sequence_full_match", [&](testing & t) {
|
||||
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.literal("world"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("helloworld");
|
||||
auto result = common_chat_combinator_parser.parse(ctx);
|
||||
t.assert_equal("sequence_full_match", true, result.success());
|
||||
});
|
||||
|
||||
// Sequences - No Match
|
||||
t.test("sequence_no_match", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("<think>I am common_chat_combinator_parser", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("sequence_no_match", true, result.fail());
|
||||
});
|
||||
|
||||
// Choices - Partial Match 1
|
||||
t.test("choices_partial_match_1", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("opt", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("choices_partial_match_1", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Choices - Partial Match 2
|
||||
t.test("choices_partial_match_2", [&](testing & t) {
|
||||
auto parser =
|
||||
build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("choice", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("choices_partial_match_2", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Choices - Full Match 1
|
||||
t.test("choices_full_match_1", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("first");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("choices_full_match_1", true, result.success());
|
||||
});
|
||||
|
||||
// Choices - Full Match 2
|
||||
t.test("choices_full_match_2", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("beta");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("choices_full_match_2", true, result.success());
|
||||
});
|
||||
|
||||
// Choices - No Match
|
||||
t.test("choices_no_match", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); });
|
||||
|
||||
auto ctx = common_peg_parse_context("best");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("choices_no_match", true, result.fail());
|
||||
});
|
||||
|
||||
// Zero or More - Partial Match 1
|
||||
t.test("zero_or_more_partial_match_1", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); });
|
||||
|
||||
auto ctx = common_peg_parse_context("a", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Zero or More - Partial Match 2
|
||||
t.test("zero_or_more_partial_match_2", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); });
|
||||
|
||||
auto ctx = common_peg_parse_context("xyx", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Zero or More - Full Match
|
||||
t.test("zero_or_more_full_match", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); });
|
||||
|
||||
auto ctx = common_peg_parse_context("test");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("zero_or_more_full_match", true, result.success());
|
||||
});
|
||||
|
||||
// One or More - Partial Match 1
|
||||
t.test("one_or_more_partial_match_1", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); });
|
||||
|
||||
auto ctx = common_peg_parse_context("rep", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// One or More - Partial Match 2
|
||||
t.test("one_or_more_partial_match_2", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); });
|
||||
|
||||
auto ctx = common_peg_parse_context("aba", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// One or More - Full Match
|
||||
t.test("one_or_more_full_match", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); });
|
||||
|
||||
auto ctx = common_peg_parse_context("single");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("one_or_more_full_match", true, result.success());
|
||||
});
|
||||
|
||||
// One or More - No Match
|
||||
t.test("one_or_more_no_match", [&](testing & t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); });
|
||||
|
||||
auto ctx = common_peg_parse_context("success");
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_equal("one_or_more_no_match", true, result.fail());
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
t.test("recursive rules", [](testing &t) {
|
||||
// Test simple number
|
||||
t.test("simple_number", [](testing &t) {
|
||||
auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("number", p.chars("0-9"));
|
||||
p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
|
||||
return p.rule("value", p.ref("number") | p.ref("list"));
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx("1");
|
||||
auto result = value_parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
});
|
||||
|
||||
// Test simple list
|
||||
t.test("simple_list", [](testing &t) {
|
||||
auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("number", p.chars("0-9"));
|
||||
p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
|
||||
return p.rule("value", p.ref("number") | p.ref("list"));
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx("[1]");
|
||||
auto result = value_parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
});
|
||||
|
||||
// Test nested list
|
||||
t.test("nested_list", [](testing &t) {
|
||||
auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("number", p.chars("0-9"));
|
||||
p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
|
||||
return p.rule("value", p.ref("number") | p.ref("list"));
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx("[[2]]");
|
||||
auto result = value_parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
});
|
||||
|
||||
// Test deeply nested list
|
||||
t.test("deeply_nested_list", [](testing &t) {
|
||||
auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("number", p.chars("0-9"));
|
||||
p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
|
||||
return p.rule("value", p.ref("number") | p.ref("list"));
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx("[[[3]]]");
|
||||
auto result = value_parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
});
|
||||
|
||||
// Test need_more_input match
|
||||
t.test("need_more_input_match", [](testing &t) {
|
||||
auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("number", p.chars("0-9"));
|
||||
p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
|
||||
return p.rule("value", p.ref("number") | p.ref("list"));
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx("[[", COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = value_parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Test no match
|
||||
t.test("no_match", [](testing &t) {
|
||||
auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("number", p.chars("0-9"));
|
||||
p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));
|
||||
return p.rule("value", p.ref("number") | p.ref("list"));
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx("[a]");
|
||||
auto result = value_parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_fail", true, result.fail());
|
||||
});
|
||||
|
||||
// Test markers
|
||||
t.test("marker", [](testing &t) {
|
||||
auto bracket_parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.marker();
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx_square("[marker]");
|
||||
common_peg_parse_context ctx_sharp("<marker>");
|
||||
|
||||
auto result_square = bracket_parser.parse(ctx_square);
|
||||
auto result_sharp = bracket_parser.parse(ctx_sharp);
|
||||
|
||||
t.assert_true("result_square_is_success", result_square.success());
|
||||
t.assert_true("result_sharp_is_success", result_sharp.success());
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
#include "tests.h"
|
||||
|
||||
#include "json-schema-to-grammar.h"
|
||||
|
||||
#include <regex>
|
||||
|
||||
static std::string trim_leading_space(const std::string & s) {
|
||||
static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)");
|
||||
return std::regex_replace(s, leading_ws_re, "$1");
|
||||
}
|
||||
|
||||
static void assert_gbnf_equal(testing & t, const std::string & expected, const std::string & actual) {
|
||||
t.assert_equal("gbnf are equal", trim_leading_space(expected), trim_leading_space(actual));
|
||||
}
|
||||
|
||||
void test_gbnf_generation(testing &t) {
|
||||
t.test("literal grammar generation", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("hello");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "hello"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("char class grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.chars("[a-z]", 1, 1);
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= [a-z]
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("sequence grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("hello") + p.literal(" ") + p.literal("world");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "hello" " " "world"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("choice grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("cat") | p.literal("dog");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "cat" | "dog"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("one_or_more grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.one_or_more(p.literal("a"));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "a"+
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("zero_or_more grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.zero_or_more(p.literal("a"));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "a"*
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("optional grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("hello") + p.optional(p.literal(" world"));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "hello" " world"?
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("until grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.until("</tag>");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-0
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-0 ::= | [<] until-0-01 | [^<] until-0
|
||||
until-0-01 ::= | [<] until-0-01 | [/] until-0-02 | [^/<] until-0
|
||||
until-0-02 ::= | [<] until-0-01 | [t] until-0-03 | [^<t] until-0
|
||||
until-0-03 ::= | [<] until-0-01 | [a] until-0-04 | [^<a] until-0
|
||||
until-0-04 ::= | [<] until-0-01 | [g] until-0-05 | [^<g] until-0
|
||||
until-0-05 ::= | [<] until-0-01 | [^<>] until-0
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("until grammar overlapping delimiter", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.until("\n</parameter>\n");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-0
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-0 ::= | [\n] until-0-01 | [^\n] until-0
|
||||
until-0-01 ::= | [\n] until-0-01 | [<] until-0-02 | [^\n<] until-0
|
||||
until-0-02 ::= | [\n] until-0-01 | [/] until-0-03 | [^\n/] until-0
|
||||
until-0-03 ::= | [\n] until-0-01 | [p] until-0-04 | [^\np] until-0
|
||||
until-0-04 ::= | [\n] until-0-01 | [a] until-0-05 | [^\na] until-0
|
||||
until-0-05 ::= | [\n] until-0-01 | [r] until-0-06 | [^\nr] until-0
|
||||
until-0-06 ::= | [\n] until-0-01 | [a] until-0-07 | [^\na] until-0
|
||||
until-0-07 ::= | [\n] until-0-01 | [m] until-0-08 | [^\nm] until-0
|
||||
until-0-08 ::= | [\n] until-0-01 | [e] until-0-09 | [^\ne] until-0
|
||||
until-0-09 ::= | [\n] until-0-01 | [t] until-0-10 | [^\nt] until-0
|
||||
until-0-10 ::= | [\n] until-0-01 | [e] until-0-11 | [^\ne] until-0
|
||||
until-0-11 ::= | [\n] until-0-01 | [r] until-0-12 | [^\nr] until-0
|
||||
until-0-12 ::= | [\n] until-0-01 | [>] until-0-13 | [^\n>] until-0
|
||||
until-0-13 ::= | [^\n] until-0
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
// DeepSeek-V3.2 tag prefix. The DSML token (｜DSML｜) embeds U+FF5C,
|
||||
// so the delimiter mixes ASCII and multi-byte codepoints.
|
||||
t.test("until grammar unicode delimiter", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.until("<｜DSML｜");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-0
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-0 ::= | [<] until-0-01 | [^<] until-0
|
||||
until-0-01 ::= | [<] until-0-01 | [\uFF5C] until-0-02 | [^<\uFF5C] until-0
|
||||
until-0-02 ::= | [<] until-0-01 | [D] until-0-03 | [^<D] until-0
|
||||
until-0-03 ::= | [<] until-0-01 | [S] until-0-04 | [^<S] until-0
|
||||
until-0-04 ::= | [<] until-0-01 | [M] until-0-05 | [^<M] until-0
|
||||
until-0-05 ::= | [<] until-0-01 | [L] until-0-06 | [^<L] until-0
|
||||
until-0-06 ::= | [<] until-0-01 | [^<\uFF5C] until-0
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("until grammar multiple delimiters", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.until_one_of({"ab", "cd", "ef"});
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= until-0
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
until-0 ::= | [a] until-0-01 | [c] until-0-03 | [e] until-0-05 | [^ace] until-0
|
||||
until-0-01 ::= | [a] until-0-01 | [c] until-0-03 | [e] until-0-05 | [^abce] until-0
|
||||
until-0-03 ::= | [a] until-0-01 | [c] until-0-03 | [e] until-0-05 | [^acde] until-0
|
||||
until-0-05 ::= | [a] until-0-01 | [c] until-0-03 | [e] until-0-05 | [^acef] until-0
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("ac grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.ac(p.until("</tag>") + p.literal("</tag>"), "</tag>");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
ac-3 ::= [<] ac-3-01 | [^<] ac-3
|
||||
ac-3-01 ::= [<] ac-3-01 | [/] ac-3-02 | [^/<] ac-3
|
||||
ac-3-02 ::= [<] ac-3-01 | [t] ac-3-03 | [^<t] ac-3
|
||||
ac-3-03 ::= [<] ac-3-01 | [a] ac-3-04 | [^<a] ac-3
|
||||
ac-3-04 ::= [<] ac-3-01 | [g] ac-3-05 | [^<g] ac-3
|
||||
ac-3-05 ::= [>] | [<] ac-3-01 | [^<>] ac-3
|
||||
root ::= ac-3
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("ac grammar terminates at first delimiter", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.ac(p.until("\n</parameter>\n") + p.literal("\n</parameter>\n"), "\n</parameter>\n");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
ac-3 ::= [\n] ac-3-01 | [^\n] ac-3
|
||||
ac-3-01 ::= [\n] ac-3-01 | [<] ac-3-02 | [^\n<] ac-3
|
||||
ac-3-02 ::= [\n] ac-3-01 | [/] ac-3-03 | [^\n/] ac-3
|
||||
ac-3-03 ::= [\n] ac-3-01 | [p] ac-3-04 | [^\np] ac-3
|
||||
ac-3-04 ::= [\n] ac-3-01 | [a] ac-3-05 | [^\na] ac-3
|
||||
ac-3-05 ::= [\n] ac-3-01 | [r] ac-3-06 | [^\nr] ac-3
|
||||
ac-3-06 ::= [\n] ac-3-01 | [a] ac-3-07 | [^\na] ac-3
|
||||
ac-3-07 ::= [\n] ac-3-01 | [m] ac-3-08 | [^\nm] ac-3
|
||||
ac-3-08 ::= [\n] ac-3-01 | [e] ac-3-09 | [^\ne] ac-3
|
||||
ac-3-09 ::= [\n] ac-3-01 | [t] ac-3-10 | [^\nt] ac-3
|
||||
ac-3-10 ::= [\n] ac-3-01 | [e] ac-3-11 | [^\ne] ac-3
|
||||
ac-3-11 ::= [\n] ac-3-01 | [r] ac-3-12 | [^\nr] ac-3
|
||||
ac-3-12 ::= [\n] ac-3-01 | [>] ac-3-13 | [^\n>] ac-3
|
||||
ac-3-13 ::= [\n] | [^\n] ac-3
|
||||
root ::= ac-3
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("ac grammar multiple delimiters", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.ac(p.eps(), std::vector<std::string>{"ab", "cd", "ef"});
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
ac-1 ::= [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^ace] ac-1
|
||||
ac-1-01 ::= [b] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^abce] ac-1
|
||||
ac-1-03 ::= [d] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^acde] ac-1
|
||||
ac-1-05 ::= [f] | [a] ac-1-01 | [c] ac-1-03 | [e] ac-1-05 | [^acef] ac-1
|
||||
root ::= ac-1
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("complex expressions with parentheses", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.one_or_more(p.literal("a") | p.literal("b"));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= ("a" | "b")+
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("rule references", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
auto digit = p.rule("digit", p.chars("[0-9]", 1, 1));
|
||||
return p.one_or_more(digit);
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
digit ::= [0-9]
|
||||
root ::= digit+
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("escaping in literals", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("hello\nworld\n!");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "hello\nworld\n!"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("operator<< (whitespace insertion)", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("hello") << p.literal("world");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "hello" space "world"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("emit only reachable rules", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("orphan", p.literal("orphan"));
|
||||
return p.literal("hello") + p.rule("child", p.literal(" world"));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
child ::= " world"
|
||||
root ::= "hello" child
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("tagged choice inside sequence gets parenthesized", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("a") + p.tag("t", p.literal("b") | p.literal("c"));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "a" ("b" | "c")
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("tagged sequence inside choice gets parenthesized", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.tag("t", p.literal("a") + p.literal("b")) | p.literal("c");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "a" "b" | "c"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("atomic choice inside repetition gets parenthesized", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.one_or_more(p.atomic(p.literal("a") | p.literal("b")));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= ("a" | "b")+
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("silent parser emits nothing in gbnf", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("hello") + p.gbnf(p.literal("world"), "");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "hello"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("silent choice inside sequence emits nothing", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("a") + p.gbnf(p.literal("b") | p.literal("c"), "") + p.literal("d");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "a" "d"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("silent wrapped in tag emits nothing", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("a") + p.tag("t", p.gbnf(p.literal("b"), ""));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "a"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("gbnf parser emits custom grammar", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("a") + p.gbnf(p.literal("b"), "[a-z]+");
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "a" [a-z]+
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("nested transparent wrappers get parenthesized", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.literal("x") + p.tag("outer", p.atomic(p.literal("a") | p.literal("b")));
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= "x" ("a" | "b")
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
});
|
||||
|
||||
t.test("emit only trigger rules (and references)", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
auto rule1 = p.rule("rule-1", p.literal("a") + p.ref("rule-2"));
|
||||
p.rule("rule-2", p.literal("b") + p.ref("rule-3"), true);
|
||||
p.rule("rule-3", p.literal("c") + p.ref("rule-4"));
|
||||
p.rule("rule-4", p.literal("d"), true);
|
||||
return rule1;
|
||||
});
|
||||
|
||||
auto gbnf = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= rule-1
|
||||
rule-1 ::= "a" rule-2
|
||||
rule-2 ::= "b" rule-3
|
||||
rule-3 ::= "c" rule-4
|
||||
rule-4 ::= "d"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf);
|
||||
|
||||
auto gbnf_lazy = build_grammar([&](const common_grammar_builder & builder) {
|
||||
parser.build_grammar(builder, true);
|
||||
});
|
||||
|
||||
assert_gbnf_equal(t, R"""(
|
||||
root ::= rule-2 | rule-4
|
||||
rule-2 ::= "b" rule-3
|
||||
rule-3 ::= "c" rule-4
|
||||
rule-4 ::= "d"
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
)""", gbnf_lazy);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
#include "tests.h"
|
||||
|
||||
void test_json_parser(testing &t) {
|
||||
// Test parsing a simple JSON object
|
||||
t.test("simple JSON object parsing", [](testing &t) {
|
||||
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
|
||||
|
||||
std::string input = R"({"name": "test", "value": 42, "flag": true})";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = json.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing a JSON array with mixed types
|
||||
t.test("JSON array with mixed types", [](testing &t) {
|
||||
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
|
||||
|
||||
std::string input = R"([1, "hello", true, null, 3.14])";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = json.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing nested JSON with objects and arrays
|
||||
t.test("nested JSON with objects and arrays", [](testing &t) {
|
||||
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
|
||||
|
||||
std::string input =
|
||||
R"({"users": [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}], "count": 2, "metadata": {"version": "1.0", "tags": ["admin", "user"]}})";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = json.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test need_more_input() parsing - incomplete object
|
||||
t.test("need_more_input() parsing - incomplete object", [](testing &t) {
|
||||
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
|
||||
|
||||
std::string input = R"({"name": "test", "value": )";
|
||||
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
|
||||
auto result = json.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Test need_more_input() parsing - incomplete array
|
||||
t.test("need_more_input() parsing - incomplete array", [](testing &t) {
|
||||
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
|
||||
|
||||
std::string input = R"([1, 2, 3, )";
|
||||
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
|
||||
auto result = json.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Test need_more_input() parsing - incomplete nested structure
|
||||
t.test("need_more_input() parsing - incomplete nested structure", [](testing &t) {
|
||||
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
|
||||
|
||||
std::string input = R"({"data": {"nested": )";
|
||||
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
|
||||
auto result = json.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Test need_more_input() parsing - incomplete escape sequence in a string value
|
||||
t.test("need_more_input() parsing - incomplete escape sequence", [](testing &t) {
|
||||
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
|
||||
|
||||
std::vector<std::string> inputs {
|
||||
R"({"text": "hello\)", // dangling backslash
|
||||
R"({"text": "hello\u)", // incomplete unicode escape sequence
|
||||
R"({"text": "hello\u00)",
|
||||
};
|
||||
|
||||
for (const auto & input : inputs) {
|
||||
t.test(input, [&](testing &t) {
|
||||
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
|
||||
auto result = json.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
|
||||
|
||||
// the incomplete escape sequence is not part of the partial value
|
||||
t.assert_equal("result_end", input.find('\\'), result.end);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("object member", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.json_member("name", "\"" + p.chars("[a-z]") + "\"");
|
||||
});
|
||||
|
||||
t.test("success", [&](testing &t) {
|
||||
std::string input = R"("name": "bob")";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
});
|
||||
|
||||
t.test("partial", [&](testing &t) {
|
||||
std::string input = R"("name": "bo)";
|
||||
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("need more input", result.need_more_input());
|
||||
});
|
||||
|
||||
t.test("failed", [&](testing &t) {
|
||||
std::string input = R"([])";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("fail", result.fail());
|
||||
});
|
||||
});
|
||||
|
||||
// json-value is entered once per level
|
||||
t.test("deep JSON", [](testing &t) {
|
||||
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
|
||||
|
||||
const auto parse = [&](size_t depth) {
|
||||
std::string input = std::string(depth, '[') + std::string(depth, ']');
|
||||
common_peg_parse_context ctx(input);
|
||||
try {
|
||||
return json.parse(ctx).success() ? std::string() : std::string("fail");
|
||||
} catch (const common_peg_recursion_error & e) {
|
||||
return std::string(e.what());
|
||||
}
|
||||
};
|
||||
|
||||
t.assert_equal("64 levels", std::string(), parse(64));
|
||||
t.assert_true("65 levels", parse(65).find("nested more than 64 levels deep") != std::string::npos);
|
||||
t.assert_true("100000 levels", parse(100000).find("nested more than 64 levels deep") != std::string::npos);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
#include "tests.h"
|
||||
|
||||
void test_python_dict_parser(testing &t) {
|
||||
// Test parsing a simple Python dict object with single quotes
|
||||
t.test("simple Python dict object parsing", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'name': 'test', 'value': 42, 'flag': True}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing a Python array with mixed types
|
||||
t.test("Python array with mixed types", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "[1, 'hello', True, None, 3.14]";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing nested Python dict with objects and arrays
|
||||
t.test("nested Python dict with objects and arrays", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input =
|
||||
"{'users': [{'id': 1, 'name': 'Alice'}, {'id': 2, 'name': 'Bob'}], 'count': 2, 'metadata': {'version': '1.0', 'tags': ['admin', 'user']}}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing Python dict with escaped single quotes
|
||||
t.test("Python dict with escaped single quotes", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'message': 'It\\'s working!'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing Python dict with double quotes inside single quotes
|
||||
t.test("Python dict with double quotes inside single quotes", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'quote': 'He said \"Hello\"'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test the example from the requirements
|
||||
t.test("complex Python dict example from requirements", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{ 'obj' : { 'something': 1, 'other \"something\"' : 'foo\\'s bar' } }";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test need_more_input() parsing - incomplete object
|
||||
t.test("need_more_input() parsing - incomplete object", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'name': 'test', 'value': ";
|
||||
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Test need_more_input() parsing - incomplete single-quoted string
|
||||
t.test("need_more_input() parsing - incomplete single-quoted string", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'name': 'test";
|
||||
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Test unicode in Python dict strings
|
||||
t.test("unicode in Python dict strings", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'message': 'Hello, 世界!'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test Python dict with unicode escapes
|
||||
t.test("Python dict with unicode escapes", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'unicode': 'Hello\\u0041'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test that Python parser accepts double-quoted strings too
|
||||
t.test("Python parser accepts double-quoted strings", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{\"name\": \"test\"}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test Python parser with mixed quote styles
|
||||
t.test("Python parser with mixed quote styles", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{\"name\": 'test', 'value': \"hello\"}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test Python True/False/None
|
||||
t.test("Python True/False/None", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
t.test("True", [&](testing &t) {
|
||||
std::string input = "True";
|
||||
common_peg_parse_context ctx(input);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("False", [&](testing &t) {
|
||||
std::string input = "False";
|
||||
common_peg_parse_context ctx(input);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("None", [&](testing &t) {
|
||||
std::string input = "None";
|
||||
common_peg_parse_context ctx(input);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("rejects JSON-style true/false/null", [&](testing &t) {
|
||||
for (const auto & kw : {"true", "false", "null"}) {
|
||||
std::string input = kw;
|
||||
common_peg_parse_context ctx(input);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true(std::string("rejects ") + kw, result.fail());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Test single-quoted string content parser directly
|
||||
t.test("single-quoted string content parser", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.sequence({ p.literal("'"), p.string_content('\''), p.literal("'"), p.space() });
|
||||
});
|
||||
|
||||
t.test("simple string", [&](testing &t) {
|
||||
std::string input = "'hello'";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("string with escaped single quote", [&](testing &t) {
|
||||
std::string input = "'it\\'s'";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("string with double quotes", [&](testing &t) {
|
||||
std::string input = "'say \"hello\"'";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("incomplete string", [&](testing &t) {
|
||||
std::string input = "'hello";
|
||||
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("need_more_input", result.need_more_input());
|
||||
});
|
||||
});
|
||||
|
||||
// Test json() with pre-registered flexible json-string rule (python dict support)
|
||||
t.test("json() parser with flexible json-string rule", [](testing &t) {
|
||||
t.test("json() rejects single quotes by default", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{'name': 'test'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("fail", result.fail());
|
||||
});
|
||||
|
||||
t.test("json() accepts single quotes with pre-registered flexible json-string rule", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
// Pre-register json-string rule with both quote styles
|
||||
p.rule("json-string", [&]() {
|
||||
return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
|
||||
});
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{'name': 'test'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("json() still accepts double quotes with flexible json-string rule", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("json-string", [&]() {
|
||||
return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
|
||||
});
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{\"name\": \"test\"}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("json() accepts mixed quote styles with flexible json-string rule", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("json-string", [&]() {
|
||||
return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
|
||||
});
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{\"name\": 'test', 'value': \"hello\"}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("complex nested structure with flexible json-string rule", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("json-string", [&]() {
|
||||
return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
|
||||
});
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{ 'obj' : { 'something': 1, 'other \"something\"' : 'foo\\'s bar' } }";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
#include "tests.h"
|
||||
|
||||
#include "peg-parser.h"
|
||||
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
#include <cctype>
|
||||
|
||||
static void assert_result_equal(testing & t, common_peg_parse_result_type expected, common_peg_parse_result_type actual) {
|
||||
t.assert_equal(common_peg_parse_result_type_name(expected), common_peg_parse_result_type_name(actual));
|
||||
}
|
||||
|
||||
static std::string hex_dump(const std::string& str) {
|
||||
std::ostringstream oss;
|
||||
for (unsigned char c : str) {
|
||||
if (std::isprint(c)) {
|
||||
oss << c;
|
||||
} else {
|
||||
oss << "\\x" << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(c);
|
||||
}
|
||||
}
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
void test_unicode(testing &t) {
|
||||
struct test_case {
|
||||
std::string input;
|
||||
std::string expected_text;
|
||||
common_peg_parse_result_type expected_result;
|
||||
};
|
||||
|
||||
t.test("any", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// Valid UTF-8 sequences
|
||||
{"Hello", "Hello", COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
{std::string("Caf\xC3\xA9"), std::string("Caf\xC3\xA9"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
{std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
{std::string("\xF0\x9F\x9A\x80"), std::string("\xF0\x9F\x9A\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// Incomplete UTF-8 sequences (partial bytes at end)
|
||||
{std::string("Caf\xC3"), "Caf", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
{std::string("\xE4\xBD"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
{std::string("\xF0\x9F\x9A"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
|
||||
// Invalid/malformed UTF-8 sequences
|
||||
{std::string("\xFF\xFE"), "", COMMON_PEG_PARSE_RESULT_FAIL},
|
||||
{std::string("Hello\x80World"), "Hello", COMMON_PEG_PARSE_RESULT_FAIL},
|
||||
{std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL},
|
||||
};
|
||||
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.sequence({p.one_or_more(p.any()), p.end()});
|
||||
});
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
// Assert result type matches
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
|
||||
// Assert matched text if success or need_more_input
|
||||
if (result.success() || result.need_more_input()) {
|
||||
std::string matched = tc.input.substr(result.start, result.end - result.start);
|
||||
t.assert_equal(tc.expected_text, matched);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("char classes", [](testing &t) {
|
||||
t.test("unicode range U+4E00-U+9FFF (CJK)", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// Within range - CJK Unified Ideographs
|
||||
{std::string("\xE4\xB8\x80"), std::string("\xE4\xB8\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4E00
|
||||
{std::string("\xE4\xBD\xA0"), std::string("\xE4\xBD\xA0"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4F60
|
||||
{std::string("\xE5\xA5\xBD"), std::string("\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+597D
|
||||
{std::string("\xE9\xBF\xBF"), std::string("\xE9\xBF\xBF"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+9FFF
|
||||
|
||||
// Outside range - should fail
|
||||
{"a", "", COMMON_PEG_PARSE_RESULT_FAIL}, // ASCII
|
||||
{std::string("\xE4\xB7\xBF"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+4DFF (before range)
|
||||
{std::string("\xEA\x80\x80"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+A000 (after range)
|
||||
|
||||
// Incomplete sequences in range
|
||||
{std::string("\xE4\xB8"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete U+4E00
|
||||
{std::string("\xE5\xA5"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete U+597D
|
||||
};
|
||||
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.sequence({p.chars(R"([\u4E00-\u9FFF])"), p.end()});
|
||||
});
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
// Assert result type matches
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
|
||||
// Assert matched text if success or need_more_input
|
||||
if (result.success() || result.need_more_input()) {
|
||||
std::string matched = tc.input.substr(result.start, result.end - result.start);
|
||||
t.assert_equal(tc.expected_text, matched);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("unicode range U+1F600-U+1F64F (emoticons)", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// Within range - Emoticons (all 4-byte UTF-8)
|
||||
{std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+1F600
|
||||
{std::string("\xF0\x9F\x98\x81"), std::string("\xF0\x9F\x98\x81"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+1F601
|
||||
{std::string("\xF0\x9F\x99\x8F"), std::string("\xF0\x9F\x99\x8F"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+1F64F
|
||||
|
||||
// Outside range
|
||||
{std::string("\xF0\x9F\x97\xBF"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+1F5FF (before range)
|
||||
{std::string("\xF0\x9F\x99\x90"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+1F650 (after range)
|
||||
{std::string("\xF0\x9F\x9A\x80"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+1F680 (outside range)
|
||||
|
||||
// Incomplete sequences
|
||||
{std::string("\xF0\x9F\x98"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Incomplete emoji
|
||||
{std::string("\xF0\x9F"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT}, // Very incomplete
|
||||
};
|
||||
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.sequence({p.chars(R"([\U0001F600-\U0001F64F])"), p.end()});
|
||||
});
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
// Assert result type matches
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
|
||||
// Assert matched text if success or need_more_input
|
||||
if (result.success() || result.need_more_input()) {
|
||||
std::string matched = tc.input.substr(result.start, result.end - result.start);
|
||||
t.assert_equal(tc.expected_text, matched);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("mixed unicode ranges", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// Match CJK
|
||||
{std::string("\xE4\xB8\x80"), std::string("\xE4\xB8\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4E00
|
||||
{std::string("\xE4\xBD\xA0"), std::string("\xE4\xBD\xA0"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4F60
|
||||
|
||||
// Match emoticons
|
||||
{std::string("\xF0\x9F\x98\x80"), std::string("\xF0\x9F\x98\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+1F600
|
||||
|
||||
// Match ASCII digits
|
||||
{"5", "5", COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// Don't match outside any range
|
||||
{"a", "", COMMON_PEG_PARSE_RESULT_FAIL},
|
||||
{std::string("\xF0\x9F\x9A\x80"), "", COMMON_PEG_PARSE_RESULT_FAIL}, // U+1F680
|
||||
|
||||
// Incomplete
|
||||
{std::string("\xE4\xB8"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
{std::string("\xF0\x9F\x98"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
};
|
||||
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.sequence({p.chars(R"([\u4E00-\u9FFF\U0001F600-\U0001F64F0-9])"), p.end()});
|
||||
});
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
// Assert result type matches
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
|
||||
// Assert matched text if success or need_more_input
|
||||
if (result.success() || result.need_more_input()) {
|
||||
std::string matched = tc.input.substr(result.start, result.end - result.start);
|
||||
t.assert_equal(tc.expected_text, matched);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
t.test("until parser", [](testing &t) {
|
||||
t.test("ASCII delimiter with Unicode content", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// CJK characters before delimiter
|
||||
{std::string("\xE4\xBD\xA0\xE5\xA5\xBD</tag>"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// Emoji before delimiter
|
||||
{std::string("\xF0\x9F\x98\x80</tag>"), std::string("\xF0\x9F\x98\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// Mixed content
|
||||
{std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!</tag>"), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
};
|
||||
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.until("</tag>");
|
||||
});
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
common_peg_parse_context ctx(tc.input);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
|
||||
if (result.success()) {
|
||||
std::string matched = tc.input.substr(result.start, result.end - result.start);
|
||||
t.assert_equal(tc.expected_text, matched);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("incomplete UTF-8 at end", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// Incomplete emoji at end, no delimiter
|
||||
{std::string("content\xF0\x9F\x98"), std::string("content"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
|
||||
// Incomplete CJK at end, no delimiter
|
||||
{std::string("hello\xE4\xB8"), std::string("hello"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
|
||||
// Complete content, no delimiter (should consume all valid UTF-8)
|
||||
{std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
};
|
||||
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.until("</tag>");
|
||||
});
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
|
||||
if (result.success() || result.need_more_input()) {
|
||||
std::string matched = tc.input.substr(result.start, result.end - result.start);
|
||||
t.assert_equal(tc.expected_text, matched);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("malformed UTF-8", [](testing &t) {
|
||||
struct passthrough_case {
|
||||
std::string input;
|
||||
std::string expected_text;
|
||||
std::string expected_sanitized;
|
||||
};
|
||||
|
||||
std::vector<passthrough_case> test_cases {
|
||||
// Invalid UTF-8 bytes
|
||||
{std::string("Hello\xFF\xFE</tag>"), std::string("Hello\xFF\xFE"), "Hello\xEF\xBF\xBD\xEF\xBF\xBD"},
|
||||
|
||||
// Continuation byte without lead byte
|
||||
{std::string("Hello\x80World</tag>"), std::string("Hello\x80World"), "Hello\xEF\xBF\xBDWorld"},
|
||||
|
||||
// Invalid continuation byte, the lead byte is dropped and '(' survives
|
||||
{std::string("\xC3\x28</tag>"), std::string("\xC3\x28"), "\xEF\xBF\xBD("},
|
||||
|
||||
// Two good bytes of a 3-byte sequence then a bad third byte, the prefix is replaced once and the third byte is kept
|
||||
{std::string("\xE4\xB8" "A</tag>"), std::string("\xE4\xB8" "A"), "\xEF\xBF\xBD" "A"},
|
||||
{std::string("\xE4\xB8</tag>"), std::string("\xE4\xB8"), "\xEF\xBF\xBD"},
|
||||
|
||||
// Truncated sequence in a complete input, the leftover prefix is replaced once
|
||||
{std::string("Hello\xE4\xB8"), std::string("Hello\xE4\xB8"), "Hello\xEF\xBF\xBD"},
|
||||
|
||||
// Valid multi-byte content around the bad byte is left alone
|
||||
{std::string("\xE4\xBD\xA0\xFF\xE5\xA5\xBD</tag>"), std::string("\xE4\xBD\xA0\xFF\xE5\xA5\xBD"), "\xE4\xBD\xA0\xEF\xBF\xBD\xE5\xA5\xBD"},
|
||||
};
|
||||
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.tag("body", p.until("</tag>")) + p.optional(p.literal("</tag>"));
|
||||
});
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
common_peg_parse_context ctx(tc.input);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
assert_result_equal(t, COMMON_PEG_PARSE_RESULT_SUCCESS, result.type);
|
||||
const auto & node = ctx.ast.get(result.nodes[0]);
|
||||
t.assert_equal("raw text", tc.expected_text, std::string(node.text));
|
||||
t.assert_equal("sanitized text", tc.expected_sanitized, node.sanitized_text());
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("malformed UTF-8 rescanned by backtracking", [](testing &t) {
|
||||
// The failed alternative and the lookahead scan the same bad byte, it must only be recorded once
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return (p.until("<a>") + p.literal("<a>")) | (p.peek(p.until("<b>")) + p.until("<b>") + p.literal("<b>"));
|
||||
});
|
||||
|
||||
std::string input("x\xFFy<b>");
|
||||
common_peg_parse_context ctx(input);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
assert_result_equal(t, COMMON_PEG_PARSE_RESULT_SUCCESS, result.type);
|
||||
t.assert_equal("invalid count", 1u, result.invalid_utf8.size());
|
||||
t.assert_equal("invalid offset", 1u, result.invalid_utf8[0].pos);
|
||||
});
|
||||
});
|
||||
|
||||
t.test("json_string parser", [](testing &t) {
|
||||
t.test("valid UTF-8 characters", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// ASCII only
|
||||
{"Hello World\"", "Hello World", COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// 2-byte UTF-8 (accented characters)
|
||||
{std::string("Caf\xC3\xA9\""), std::string("Caf\xC3\xA9"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// 3-byte UTF-8 (CJK)
|
||||
{std::string("\xE4\xBD\xA0\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// 4-byte UTF-8 (emoji)
|
||||
{std::string("\xF0\x9F\x98\x80\""), std::string("\xF0\x9F\x98\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// Mixed content
|
||||
{std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!\""), std::string("Hello \xE4\xB8\x96\xE7\x95\x8C!"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.sequence({p.string_content('"'), p.literal("\"")});
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx(tc.input);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
|
||||
if (result.success()) {
|
||||
std::string matched = tc.input.substr(result.start, result.end - result.start - 1); // -1 to exclude closing quote
|
||||
t.assert_equal(tc.expected_text, matched);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("incomplete UTF-8", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// Incomplete 2-byte sequence
|
||||
{std::string("Caf\xC3"), std::string("Caf"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
|
||||
// Incomplete 3-byte sequence
|
||||
{std::string("Hello\xE4\xB8"), std::string("Hello"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
|
||||
// Incomplete 4-byte sequence
|
||||
{std::string("Text\xF0\x9F\x98"), std::string("Text"), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
|
||||
// Incomplete at very start
|
||||
{std::string("\xE4\xBD"), std::string(""), COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.string_content('"');
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
|
||||
if (result.need_more_input()) {
|
||||
std::string matched = tc.input.substr(result.start, result.end - result.start);
|
||||
t.assert_equal(tc.expected_text, matched);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("malformed UTF-8", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// Invalid UTF-8 bytes
|
||||
{std::string("Hello\xFF\xFE"), "", COMMON_PEG_PARSE_RESULT_FAIL},
|
||||
|
||||
// Continuation byte without lead byte
|
||||
{std::string("Hello\x80World"), "", COMMON_PEG_PARSE_RESULT_FAIL},
|
||||
|
||||
// Invalid continuation byte
|
||||
{std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL},
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.string_content('"');
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx(tc.input);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
t.test("escape sequences with UTF-8", [](testing &t) {
|
||||
std::vector<test_case> test_cases {
|
||||
// Unicode escape sequence
|
||||
{"Hello\\u0041\"", "Hello\\u0041", COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// Mix of UTF-8 and escape sequences
|
||||
{std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\n\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
|
||||
// Escaped quote in UTF-8 string
|
||||
{std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD\""), std::string("\xE4\xBD\xA0\\\"\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS},
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < test_cases.size(); i++) {
|
||||
const auto & tc = test_cases[i];
|
||||
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
|
||||
|
||||
t.test(test_name, [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
|
||||
return p.sequence({p.string_content('"'), p.literal("\"")});
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx(tc.input);
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
assert_result_equal(t, tc.expected_result, result.type);
|
||||
|
||||
if (result.success()) {
|
||||
std::string matched = tc.input.substr(result.start, result.end - result.start - 1); // -1 to exclude closing quote
|
||||
t.assert_equal(tc.expected_text, matched);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
#pragma once
|
||||
|
||||
// Common includes for all test files
|
||||
#include "json.h"
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../testing.h"
|
||||
#include "peg-parser.h"
|
||||
#include "chat-peg-parser.h"
|
||||
#include "simple-tokenize.h"
|
||||
|
||||
struct bench_tool_call {
|
||||
std::string id;
|
||||
std::string name;
|
||||
common_json args;
|
||||
};
|
||||
|
||||
// Test function declarations
|
||||
void test_basic(testing &t);
|
||||
void test_json_parser(testing &t);
|
||||
void test_gbnf_generation(testing &t);
|
||||
void test_unicode(testing &t);
|
||||
void test_python_dict_parser(testing &t);
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,418 @@
|
||||
#include "arg.h"
|
||||
#include "common.h"
|
||||
#include "download.h"
|
||||
#include "llama.h"
|
||||
#include "speculative.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
#include <unordered_set>
|
||||
|
||||
#undef NDEBUG
|
||||
#include <cassert>
|
||||
|
||||
static void test(void) {
|
||||
common_params params;
|
||||
|
||||
auto assert_output_limits = [](int32_t n_batch, int32_t n_parallel, int32_t n_draft,
|
||||
int32_t total, int32_t per_seq) {
|
||||
const auto limits = common_speculative_get_output_limits(n_batch, n_parallel, n_draft);
|
||||
assert(limits.total == total);
|
||||
assert(limits.per_seq == per_seq);
|
||||
};
|
||||
|
||||
assert_output_limits(16, 2, 3, 8, 4);
|
||||
assert_output_limits(16, 2, -1, 2, 1);
|
||||
assert_output_limits( 6, 2, 3, 6, 4);
|
||||
assert_output_limits( 2, 1, 3, 2, 2);
|
||||
assert_output_limits(
|
||||
std::numeric_limits<int32_t>::max(),
|
||||
std::numeric_limits<int32_t>::max(),
|
||||
std::numeric_limits<int32_t>::max(),
|
||||
std::numeric_limits<int32_t>::max(),
|
||||
std::numeric_limits<int32_t>::max());
|
||||
|
||||
{
|
||||
common_params_speculative spec;
|
||||
spec.synth_len = 3.4;
|
||||
|
||||
auto assert_invalid = [](const common_params_speculative & value, int32_t n_max) {
|
||||
try {
|
||||
common_speculative_synth_rates_resolve(&value, n_max);
|
||||
assert(false);
|
||||
} catch (const std::invalid_argument &) {
|
||||
}
|
||||
};
|
||||
|
||||
const auto rates = common_speculative_synth_rates_resolve(&spec, 4);
|
||||
assert(rates.size() == 4);
|
||||
assert(std::abs(rates[0] - 0.80581) < 1e-5);
|
||||
assert(std::abs(rates[1] - 0.64933) < 1e-5);
|
||||
assert(std::abs(rates[2] - 0.52323) < 1e-5);
|
||||
assert(std::abs(rates[3] - 0.42163) < 1e-5);
|
||||
assert(std::abs(1.0 + rates[0] + rates[1] + rates[2] + rates[3] - 3.4) < 1e-8);
|
||||
|
||||
spec.synth_len = 1.0;
|
||||
assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({0.0, 0.0, 0.0, 0.0}));
|
||||
|
||||
spec.synth_len = 5.0;
|
||||
assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({1.0, 1.0, 1.0, 1.0}));
|
||||
|
||||
spec.synth_len = 5.1;
|
||||
assert_invalid(spec, 4);
|
||||
|
||||
spec.synth_len = std::numeric_limits<double>::quiet_NaN();
|
||||
assert_invalid(spec, 4);
|
||||
|
||||
spec.synth_len = 0.0;
|
||||
assert_invalid(spec, 4);
|
||||
|
||||
spec.synth_len = -1.0;
|
||||
spec.synth_rates = {0.8, 0.6, 0.4};
|
||||
assert_invalid(spec, 4);
|
||||
|
||||
spec.synth_rates = {0.8, 0.6, 0.4, 0.2};
|
||||
assert(common_speculative_synth_rates_resolve(&spec, 4) == spec.synth_rates);
|
||||
|
||||
spec.synth_rates = {0.8, 0.9, 0.4, 0.2};
|
||||
assert_invalid(spec, 4);
|
||||
|
||||
spec.synth_rates = {0.8, std::numeric_limits<double>::quiet_NaN(), 0.4, 0.2};
|
||||
assert_invalid(spec, 4);
|
||||
|
||||
spec.synth_rates = {0.8, 0.6, 0.4, -0.2};
|
||||
assert_invalid(spec, 4);
|
||||
|
||||
spec.synth_rates = {0.8, 0.6, 0.4, 0.2};
|
||||
spec.synth_len = 3.0;
|
||||
assert_invalid(spec, 4);
|
||||
}
|
||||
|
||||
{
|
||||
common_params base;
|
||||
base.n_parallel = 4;
|
||||
base.n_outputs_max_per_seq = 8;
|
||||
base.n_ubatch = 512;
|
||||
|
||||
{
|
||||
const auto draft = common_base_params_to_speculative(base);
|
||||
assert(draft.n_outputs_max == 4);
|
||||
assert(draft.n_outputs_max_per_seq == 1);
|
||||
assert(draft.n_ubatch == 512); // inherits the target by default
|
||||
}
|
||||
|
||||
base.speculative.draft.n_ubatch = 8;
|
||||
{
|
||||
const auto draft = common_base_params_to_speculative(base);
|
||||
assert(draft.n_ubatch == 8);
|
||||
}
|
||||
}
|
||||
|
||||
printf("test-arg-parser: make sure there is no duplicated arguments in any examples\n\n");
|
||||
for (int ex = 0; ex < LLAMA_EXAMPLE_COUNT; ex++) {
|
||||
try {
|
||||
auto ctx_arg = common_params_parser_init(params, (enum llama_example)ex);
|
||||
common_params_add_preset_options(ctx_arg.options);
|
||||
std::unordered_set<std::string> seen_args;
|
||||
std::unordered_set<std::string> seen_env_vars;
|
||||
for (const auto & opt : ctx_arg.options) {
|
||||
// check for args duplications
|
||||
for (const auto & arg : opt.get_args()) {
|
||||
if (seen_args.find(arg) == seen_args.end()) {
|
||||
seen_args.insert(arg);
|
||||
} else {
|
||||
fprintf(stderr, "test-arg-parser: found different handlers for the same argument: %s", arg.c_str());
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
// check for env var duplications
|
||||
for (const auto & env : opt.get_env()) {
|
||||
if (seen_env_vars.find(env) == seen_env_vars.end()) {
|
||||
seen_env_vars.insert(env);
|
||||
} else {
|
||||
fprintf(stderr, "test-arg-parser: found different handlers for the same env var: %s", env.c_str());
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// exclude spec args from this check
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/22397
|
||||
const bool skip = opt.is_spec;
|
||||
|
||||
// ensure shorter argument precedes longer argument
|
||||
if (!skip && opt.args.size() > 1) {
|
||||
const std::string first(opt.args.front());
|
||||
const std::string last(opt.args.back());
|
||||
|
||||
if (first.length() > last.length()) {
|
||||
fprintf(stderr, "test-arg-parser: shorter argument should come before longer one: %s, %s\n",
|
||||
first.c_str(), last.c_str());
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
// same check for negated arguments
|
||||
if (opt.args_neg.size() > 1) {
|
||||
const std::string first(opt.args_neg.front());
|
||||
const std::string last(opt.args_neg.back());
|
||||
|
||||
if (first.length() > last.length()) {
|
||||
fprintf(stderr, "test-arg-parser: shorter negated argument should come before longer one: %s, %s\n",
|
||||
first.c_str(), last.c_str());
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (std::exception & e) {
|
||||
printf("%s\n", e.what());
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
auto list_str_to_char = [](std::vector<std::string> & argv) -> std::vector<char *> {
|
||||
std::vector<char *> res;
|
||||
for (auto & arg : argv) {
|
||||
res.push_back(const_cast<char *>(arg.data()));
|
||||
}
|
||||
return res;
|
||||
};
|
||||
|
||||
std::vector<std::string> argv;
|
||||
|
||||
printf("test-arg-parser: test invalid usage\n\n");
|
||||
|
||||
// missing value
|
||||
argv = {"binary_name", "-m"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
// wrong value (int)
|
||||
argv = {"binary_name", "-ngl", "hello"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
// wrong value (enum)
|
||||
argv = {"binary_name", "-sm", "hello"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
{
|
||||
common_params penalty_params;
|
||||
assert(penalty_params.sampling.penalty_last_n == 64);
|
||||
assert(penalty_params.sampling.dry_penalty_last_n == 64);
|
||||
|
||||
argv = {"binary_name", "--repeat-last-n", "-1"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
argv = {"binary_name", "--dry-penalty-last-n", "-1"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
argv = {"binary_name", "--repeat-penalty", "0"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
argv = {"binary_name", "--repeat-penalty", "-1"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
argv = {"binary_name", "--repeat-penalty", "nan"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
argv = {"binary_name", "--repeat-penalty", "inf"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
argv = {"binary_name", "--repeat-penalty", "-inf"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
const char * penalty_options[] = {"--frequency-penalty", "--presence-penalty"};
|
||||
const char * nonfinite_values[] = {"nan", "inf", "-inf"};
|
||||
for (const char * option : penalty_options) {
|
||||
for (const char * value : nonfinite_values) {
|
||||
argv = {"binary_name", option, value};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// non-existence arg in specific example (--draft cannot be used outside llama-speculative)
|
||||
argv = {"binary_name", "--draft", "123"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_EMBEDDING));
|
||||
|
||||
argv = {"binary_name", "-lm", "hello"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
printf("test-arg-parser: test valid usage\n\n");
|
||||
|
||||
argv = {"binary_name", "-m", "model_file.gguf"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.model.path == "model_file.gguf");
|
||||
|
||||
argv = {"binary_name", "-t", "1234"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.cpuparams.n_threads == 1234);
|
||||
|
||||
argv = {"binary_name", "--verbose"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.verbosity > 1);
|
||||
|
||||
argv = {"binary_name", "-m", "abc.gguf", "--predict", "6789", "--batch-size", "9090"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.model.path == "abc.gguf");
|
||||
assert(params.n_predict == 6789);
|
||||
assert(params.n_batch == 9090);
|
||||
|
||||
// --draft cannot be used outside llama-speculative
|
||||
argv = {"binary_name", "--spec-draft-n-max", "123"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE));
|
||||
assert(params.speculative.draft.n_max == 123);
|
||||
|
||||
{
|
||||
common_params synth_params;
|
||||
argv = {"binary_name", "--spec-synth-len", "3.4"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
|
||||
assert(synth_params.speculative.synth_len == 3.4);
|
||||
}
|
||||
|
||||
{
|
||||
common_params synth_params;
|
||||
argv = {"binary_name", "--spec-synth-rates", "0.8,0.6,0.2"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
|
||||
assert(synth_params.speculative.synth_rates == std::vector<double>({0.8, 0.6, 0.2}));
|
||||
}
|
||||
|
||||
{
|
||||
common_params synth_params;
|
||||
argv = {"binary_name", "--spec-synth-len", "3.4x"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
|
||||
}
|
||||
|
||||
argv = {"binary_name", "-lm", "none"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_NONE);
|
||||
|
||||
argv = {"binary_name", "-lm", "mmap"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);
|
||||
|
||||
argv = {"binary_name", "-lm", "mlock"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);
|
||||
|
||||
argv = {"binary_name", "-lm", "mmap+mlock"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);
|
||||
|
||||
argv = {"binary_name", "-lm", "dio"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);
|
||||
|
||||
// multi-value args (CSV)
|
||||
argv = {"binary_name", "--lora", "file1.gguf,\"file2,2.gguf\",\"file3\"\"3\"\".gguf\",file4\".gguf"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.lora_adapters.size() == 4);
|
||||
assert(params.lora_adapters[0].path == "file1.gguf");
|
||||
assert(params.lora_adapters[1].path == "file2,2.gguf");
|
||||
assert(params.lora_adapters[2].path == "file3\"3\".gguf");
|
||||
assert(params.lora_adapters[3].path == "file4\".gguf");
|
||||
|
||||
// skip this part on windows, because setenv is not supported
|
||||
#ifdef _WIN32
|
||||
printf("test-arg-parser: skip on windows build\n");
|
||||
#else
|
||||
printf("test-arg-parser: test environment variables (valid + invalid usages)\n\n");
|
||||
|
||||
setenv("LLAMA_ARG_THREADS", "blah", true);
|
||||
argv = {"binary_name"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
setenv("LLAMA_ARG_MODEL", "blah.gguf", true);
|
||||
setenv("LLAMA_ARG_THREADS", "1010", true);
|
||||
argv = {"binary_name"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.model.path == "blah.gguf");
|
||||
assert(params.cpuparams.n_threads == 1010);
|
||||
|
||||
setenv("LLAMA_ARG_LOAD_MODE", "blah", true);
|
||||
argv = {"binary_name"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
setenv("LLAMA_ARG_LOAD_MODE", "mmap", true);
|
||||
argv = {"binary_name"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);
|
||||
|
||||
setenv("LLAMA_ARG_LOAD_MODE", "mlock", true);
|
||||
argv = {"binary_name"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);
|
||||
|
||||
setenv("LLAMA_ARG_LOAD_MODE", "mmap+mlock", true);
|
||||
argv = {"binary_name"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);
|
||||
|
||||
setenv("LLAMA_ARG_LOAD_MODE", "dio", true);
|
||||
argv = {"binary_name"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);
|
||||
|
||||
printf("test-arg-parser: test negated environment variables\n\n");
|
||||
|
||||
setenv("LLAMA_ARG_LOAD_MODE", "none", true);
|
||||
setenv("LLAMA_ARG_NO_PERF", "1", true); // legacy format
|
||||
argv = {"binary_name"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.load_mode == LLAMA_LOAD_MODE_NONE);
|
||||
assert(params.no_perf == true);
|
||||
|
||||
printf("test-arg-parser: test environment variables being overwritten\n\n");
|
||||
|
||||
setenv("LLAMA_ARG_MODEL", "blah.gguf", true);
|
||||
setenv("LLAMA_ARG_THREADS", "1010", true);
|
||||
argv = {"binary_name", "-m", "overwritten.gguf"};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
assert(params.model.path == "overwritten.gguf");
|
||||
assert(params.cpuparams.n_threads == 1010);
|
||||
#endif // _WIN32
|
||||
|
||||
printf("test-arg-parser: test download functions\n\n");
|
||||
const char * GOOD_URL = "http://ggml.ai/";
|
||||
const char * BAD_URL = "http://ggml.ai/404";
|
||||
|
||||
{
|
||||
printf("test-arg-parser: test good URL\n\n");
|
||||
auto res = common_remote_get_content(GOOD_URL, {});
|
||||
assert(res.first == 200);
|
||||
assert(res.second.size() > 0);
|
||||
std::string str(res.second.data(), res.second.size());
|
||||
assert(str.find("llama.cpp") != std::string::npos);
|
||||
}
|
||||
|
||||
{
|
||||
printf("test-arg-parser: test bad URL\n\n");
|
||||
auto res = common_remote_get_content(BAD_URL, {});
|
||||
assert(res.first == 404);
|
||||
}
|
||||
|
||||
{
|
||||
printf("test-arg-parser: test max size error\n");
|
||||
common_remote_params params;
|
||||
params.max_size = 1;
|
||||
try {
|
||||
common_remote_get_content(GOOD_URL, params);
|
||||
assert(false && "it should throw an error");
|
||||
} catch (std::exception & e) {
|
||||
printf(" expected error: %s\n\n", e.what());
|
||||
}
|
||||
}
|
||||
|
||||
printf("test-arg-parser: all tests OK\n\n");
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
try {
|
||||
test();
|
||||
} catch (std::exception & e) {
|
||||
fprintf(stderr, "test-arg-parser: exception: %s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
// ref: https://github.com/ggml-org/llama.cpp/issues/4952#issuecomment-1892864763
|
||||
|
||||
#include <thread>
|
||||
|
||||
#include "llama.h"
|
||||
#include "common.h"
|
||||
|
||||
// This creates a new context inside a pthread and then tries to exit cleanly.
|
||||
int main(int argc, char ** argv) {
|
||||
auto * model_path = common_get_model_or_exit(argc, argv);
|
||||
|
||||
std::thread([&model_path]() {
|
||||
llama_backend_init();
|
||||
auto * model = llama_model_load_from_file(model_path, llama_model_default_params());
|
||||
auto * ctx = llama_init_from_model(model, llama_context_default_params());
|
||||
llama_free(ctx);
|
||||
llama_model_free(model);
|
||||
llama_backend_free();
|
||||
}).join();
|
||||
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,236 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cassert>
|
||||
#include <vector>
|
||||
#include <thread>
|
||||
|
||||
#define MAX_NARGS 2
|
||||
|
||||
static void test_barrier(int n_threads, int n_rounds) {
|
||||
struct ggml_init_params params = {
|
||||
/* .mem_size = */ 1024*1024*1024,
|
||||
/* .mem_buffer = */ NULL,
|
||||
/* .no_alloc = */ false,
|
||||
};
|
||||
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
// Create graph
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
|
||||
// Lots of small, parallel ops where barriers in between will dominate
|
||||
struct ggml_tensor * out = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 64);
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 64, 128);
|
||||
out = ggml_mul_mat(ctx, a, out);
|
||||
|
||||
struct ggml_tensor * d = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 128, 64);
|
||||
out = ggml_mul_mat(ctx, d, out);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, out);
|
||||
int n_nodes = ggml_graph_n_nodes(gf);
|
||||
|
||||
// Create threadpool
|
||||
struct ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);
|
||||
struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp);
|
||||
if (!threadpool) {
|
||||
fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// The test runs with constant number of threads
|
||||
struct ggml_cplan cplan = ggml_graph_plan(gf, n_threads, threadpool);
|
||||
|
||||
std::vector<uint8_t> work_data(cplan.work_size);
|
||||
cplan.work_data = work_data.data();
|
||||
|
||||
std::cerr << "graph-compute with"
|
||||
<< "\n n_threads: " << n_threads
|
||||
<< "\n n_nodes: " << n_nodes
|
||||
<< "\n n_rounds: " << n_rounds
|
||||
<< "\n";
|
||||
// ggml_graph_print(gf);
|
||||
|
||||
// Warmup
|
||||
ggml_graph_compute(gf, &cplan);
|
||||
|
||||
auto t0 = std::chrono::high_resolution_clock::now();
|
||||
|
||||
for (int i=0; i < n_rounds; i++) {
|
||||
ggml_graph_compute(gf, &cplan);
|
||||
}
|
||||
|
||||
auto t1 = std::chrono::high_resolution_clock::now();
|
||||
|
||||
auto usec = std::chrono::duration_cast<std::chrono::microseconds>(t1-t0).count();
|
||||
auto nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(t1-t0).count();
|
||||
std::cerr << "graph-compute took " << usec << " usec "
|
||||
<< "\n " << (float) usec / n_rounds << " usec per-iter"
|
||||
<< "\n " << (float) nsec / (n_rounds * n_nodes) << " nsec per-node"
|
||||
<< "\n";
|
||||
|
||||
ggml_threadpool_free(threadpool);
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
static void test_active(int n_threads, int n_rounds) {
|
||||
struct ggml_init_params params = {
|
||||
/* .mem_size = */ 1024*1024*1024,
|
||||
/* .mem_buffer = */ NULL,
|
||||
/* .no_alloc = */ false,
|
||||
};
|
||||
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
// Create graph
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
|
||||
// Small graph with, parallel ops with barriers
|
||||
struct ggml_tensor * out = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 64);
|
||||
for (int i = 0; i < 2; i++) {
|
||||
struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 64, 128);
|
||||
out = ggml_mul_mat(ctx, a, out);
|
||||
|
||||
struct ggml_tensor * d = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 128, 64);
|
||||
out = ggml_mul_mat(ctx, d, out);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, out);
|
||||
int n_nodes = ggml_graph_n_nodes(gf);
|
||||
|
||||
// Create threadpool
|
||||
struct ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);
|
||||
struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp);
|
||||
if (!threadpool) {
|
||||
fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
std::cerr << "graph-compute with"
|
||||
<< "\n n_threads: " << n_threads
|
||||
<< "\n n_nodes: " << n_nodes
|
||||
<< "\n n_rounds: " << n_rounds
|
||||
<< "\n";
|
||||
// ggml_graph_print(gf);
|
||||
|
||||
// In this test we keep changing the number of threads every 4th iteration
|
||||
// to test for race conditions in that path
|
||||
|
||||
for (int i=0; i < n_rounds; i++) {
|
||||
struct ggml_cplan cplan = ggml_graph_plan(gf, (i % 4) == 0 ? 1 : n_threads, threadpool);
|
||||
|
||||
std::vector<uint8_t> work_data(cplan.work_size);
|
||||
cplan.work_data = work_data.data();
|
||||
|
||||
ggml_graph_compute(gf, &cplan);
|
||||
}
|
||||
|
||||
ggml_threadpool_free(threadpool);
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
static void test_multi_graph(int n_threads, int n_rounds) {
|
||||
struct ggml_init_params params = {
|
||||
/* .mem_size = */ 1024*1024*1024,
|
||||
/* .mem_buffer = */ NULL,
|
||||
/* .no_alloc = */ false,
|
||||
};
|
||||
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
// Create graphs
|
||||
struct ggml_cgraph * gf0 = ggml_new_graph(ctx);
|
||||
{
|
||||
// Small graph with parallel ops with barriers
|
||||
struct ggml_tensor * out = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 64);
|
||||
for (int i = 0; i < 2; i++) {
|
||||
struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 64, 128);
|
||||
out = ggml_mul_mat(ctx, a, out);
|
||||
|
||||
struct ggml_tensor * d = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 128, 64);
|
||||
out = ggml_mul_mat(ctx, d, out);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf0, out);
|
||||
}
|
||||
|
||||
struct ggml_cgraph * gf1 = ggml_new_graph(ctx);
|
||||
{
|
||||
// Small graph with parallel ops with barriers
|
||||
// Use larger tensors to make sure work_data size is larger than gf0
|
||||
struct ggml_tensor * out = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 256);
|
||||
for (int i = 0; i < 4; i++) {
|
||||
struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 256, 128);
|
||||
out = ggml_mul_mat(ctx, a, out);
|
||||
|
||||
struct ggml_tensor * d = ggml_new_tensor_2d(ctx, GGML_TYPE_Q4_0, 128, 256);
|
||||
out = ggml_mul_mat(ctx, d, out);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf1, out);
|
||||
}
|
||||
|
||||
|
||||
// Create threadpool
|
||||
struct ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);
|
||||
struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp);
|
||||
if (!threadpool) {
|
||||
fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
std::cerr << "graph-compute with"
|
||||
<< "\n gf0 n_nodes: " << ggml_graph_n_nodes(gf0)
|
||||
<< "\n gf1 n_nodes: " << ggml_graph_n_nodes(gf1)
|
||||
<< "\n n_threads: " << n_threads
|
||||
<< "\n n_rounds: " << n_rounds
|
||||
<< "\n";
|
||||
|
||||
// In this test we keep changing the number of threads every 4th iteration
|
||||
// and we compute two graphs back to back to test graph frequent graph switching
|
||||
|
||||
for (int i=0; i < n_rounds; i++) {
|
||||
struct ggml_cplan cplan0 = ggml_graph_plan(gf0, (i % 4) == 0 ? 1 : n_threads, threadpool);
|
||||
std::vector<uint8_t> work_data0(cplan0.work_size);
|
||||
cplan0.work_data = work_data0.data();
|
||||
|
||||
struct ggml_cplan cplan1 = ggml_graph_plan(gf1, (i % 4) == 0 ? 1 : n_threads, threadpool);
|
||||
std::vector<uint8_t> work_data1(cplan1.work_size);
|
||||
cplan1.work_data = work_data1.data();
|
||||
|
||||
ggml_graph_compute(gf0, &cplan0);
|
||||
ggml_graph_compute(gf1, &cplan1);
|
||||
}
|
||||
|
||||
ggml_threadpool_free(threadpool);
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
|
||||
int n_threads = std::max(1, std::min(4, (int) std::thread::hardware_concurrency()));
|
||||
int n_rounds = 100;
|
||||
|
||||
if (argc > 1) {
|
||||
n_threads = std::atoi(argv[1]);
|
||||
}
|
||||
|
||||
if (argc > 2) {
|
||||
n_rounds = std::atoi(argv[2]);
|
||||
}
|
||||
|
||||
test_barrier(n_threads, n_rounds);
|
||||
|
||||
test_active(n_threads, n_rounds * 100);
|
||||
|
||||
test_multi_graph(n_threads, n_rounds * 10);
|
||||
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,3 @@
|
||||
#include "llama.h"
|
||||
|
||||
int main(void) {}
|
||||
@@ -0,0 +1,615 @@
|
||||
#include "chat-auto-parser.h"
|
||||
#include "chat-auto-parser-helpers.h"
|
||||
#include "chat.h"
|
||||
#include "log.h"
|
||||
#include "jinja/caps.h"
|
||||
#include "jinja/runtime.h"
|
||||
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <algorithm>
|
||||
|
||||
#include "json.h"
|
||||
|
||||
using json = common_json;
|
||||
|
||||
// ANSI color codes - using 256-color palette for brighter colors (all bold)
|
||||
#define ANSI_RESET "\033[0m"
|
||||
#define ANSI_PURPLE "\033[1m\x1b[38;5;126m" // Bold bright purple for main headers
|
||||
#define ANSI_CYAN "\033[1m\x1b[38;5;81m" // Bold bright cyan for section headers
|
||||
#define ANSI_BLUE "\033[1m\x1b[38;5;12m" // Bold bright blue for labels
|
||||
#define ANSI_ORANGE "\033[1m\x1b[38;5;209m" // Bold orange for right differences
|
||||
#define ANSI_GREEN "\033[1m\x1b[38;5;83m" // Bold bright green for left differences
|
||||
#define ANSI_GRAY "\033[1m\x1b[38;5;240m" // Bold gray (used for "no variables" message)
|
||||
#define ANSI_BOLD "\033[1m" // Standalone bold
|
||||
#define ANSI_PREFIX "\033[1m\x1b[38;5;176m" // Bold color for common prefix
|
||||
#define ANSI_SUFFIX "\033[1m\x1b[38;5;61m" // Bold color for common suffix
|
||||
|
||||
// All template paths extracted from tests/test-chat.cpp
|
||||
static const std::vector<std::string> ALL_TEMPLATE_PATHS = {
|
||||
"models/templates/Apertus-8B-Instruct.jinja",
|
||||
"models/templates/Apriel-1.6-15b-Thinker-fixed.jinja",
|
||||
"models/templates/ByteDance-Seed-OSS.jinja",
|
||||
"models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja",
|
||||
"models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja",
|
||||
"models/templates/GLM-4.6.jinja",
|
||||
"models/templates/GLM-4.7-Flash.jinja",
|
||||
"models/templates/Kimi-K2-Instruct.jinja",
|
||||
"models/templates/Kimi-K2-Thinking.jinja",
|
||||
"models/templates/MiMo-VL.jinja",
|
||||
"models/templates/MiniMax-M2.jinja",
|
||||
"models/templates/Mistral-Small-3.2-24B-Instruct-2506.jinja",
|
||||
"models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja",
|
||||
"models/templates/NVIDIA-Nemotron-Nano-v2.jinja",
|
||||
"models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja",
|
||||
"models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja",
|
||||
"models/templates/Qwen-QwQ-32B.jinja",
|
||||
"models/templates/Qwen-Qwen2.5-7B-Instruct.jinja",
|
||||
"models/templates/Qwen3-Coder.jinja",
|
||||
"models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja",
|
||||
"models/templates/deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja",
|
||||
"models/templates/deepseek-ai-DeepSeek-V3.1.jinja",
|
||||
"models/templates/fireworks-ai-llama-3-firefunction-v2.jinja",
|
||||
"models/templates/google-gemma-2-2b-it.jinja",
|
||||
"models/templates/ibm-granite-granite-3.3-2B-Instruct.jinja",
|
||||
"models/templates/llama-cpp-deepseek-r1.jinja",
|
||||
"models/templates/meetkai-functionary-medium-v3.1.jinja",
|
||||
"models/templates/meetkai-functionary-medium-v3.2.jinja",
|
||||
"models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja",
|
||||
"models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja",
|
||||
"models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja",
|
||||
"models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja",
|
||||
"models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja",
|
||||
"models/templates/moonshotai-Kimi-K2.jinja",
|
||||
"models/templates/openai-gpt-oss-120b.jinja",
|
||||
"models/templates/unsloth-Apriel-1.5.jinja",
|
||||
"models/templates/unsloth-mistral-Devstral-Small-2507.jinja",
|
||||
};
|
||||
|
||||
struct analysis_options {
|
||||
std::vector<std::string> template_paths;
|
||||
bool analyze_all = false;
|
||||
};
|
||||
|
||||
static std::string read_file(const std::string & path) {
|
||||
std::ifstream fin(path, std::ios::binary);
|
||||
if (!fin.is_open()) {
|
||||
throw std::runtime_error("Could not open file: " + path);
|
||||
}
|
||||
std::ostringstream buf;
|
||||
buf << fin.rdbuf();
|
||||
return buf.str();
|
||||
}
|
||||
|
||||
static void print_usage(const char * program_name) {
|
||||
LOG_ERR("Debug the auto-parser's differential analysis: render a template with/without tools, reasoning, etc. and show the diffs.\n");
|
||||
LOG_ERR("\nUsage: %s [options]\n", program_name);
|
||||
LOG_ERR("\nOptions:\n");
|
||||
LOG_ERR(" --template <name> Analyze specific template from test suite (e.g., 'deepseek' or 'DeepSeek-V3.1')\n");
|
||||
LOG_ERR(" --template-file <path> Analyze custom template file\n");
|
||||
LOG_ERR(" --all Analyze all templates from test suite (default when no arguments are given)\n");
|
||||
LOG_ERR("\nExamples:\n");
|
||||
LOG_ERR(" %s --all\n", program_name);
|
||||
LOG_ERR(" %s --template deepseek\n", program_name);
|
||||
LOG_ERR(" %s --template-file my-template.jinja\n", program_name);
|
||||
}
|
||||
|
||||
static bool parse_options(int argc, char ** argv, analysis_options & opts) {
|
||||
if (argc < 2) {
|
||||
// default mode: analyze all templates from the test suite
|
||||
opts.analyze_all = true;
|
||||
}
|
||||
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
std::string arg = argv[i];
|
||||
|
||||
if (arg == "-h" || arg == "--help") {
|
||||
print_usage(argv[0]);
|
||||
return false;
|
||||
} else if (arg == "--all") {
|
||||
opts.analyze_all = true;
|
||||
} else if (arg == "--template") {
|
||||
if (i + 1 >= argc) {
|
||||
LOG_ERR("--template requires an argument\n");
|
||||
return false;
|
||||
}
|
||||
std::string pattern = argv[++i];
|
||||
std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower);
|
||||
|
||||
// Find matching templates
|
||||
bool found = false;
|
||||
for (const auto & path : ALL_TEMPLATE_PATHS) {
|
||||
std::string path_lower = path;
|
||||
std::transform(path_lower.begin(), path_lower.end(), path_lower.begin(), ::tolower);
|
||||
if (path_lower.find(pattern) != std::string::npos) {
|
||||
opts.template_paths.push_back(path);
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
LOG_ERR("No templates found matching: %s\n", pattern.c_str());
|
||||
return false;
|
||||
}
|
||||
} else if (arg == "--template-file") {
|
||||
if (i + 1 >= argc) {
|
||||
LOG_ERR("--template-file requires an argument\n");
|
||||
return false;
|
||||
}
|
||||
opts.template_paths.push_back(argv[++i]);
|
||||
} else {
|
||||
LOG_ERR("Unknown option: %s\n", arg.c_str());
|
||||
print_usage(argv[0]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.analyze_all) {
|
||||
opts.template_paths = ALL_TEMPLATE_PATHS;
|
||||
}
|
||||
|
||||
if (opts.template_paths.empty()) {
|
||||
LOG_ERR("No templates specified\n");
|
||||
print_usage(argv[0]);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static json build_tools_definition() {
|
||||
json parameters_schema = json::object();
|
||||
parameters_schema["type"] = "object";
|
||||
parameters_schema["properties"] = json::object();
|
||||
parameters_schema["properties"]["param1"] = json::object({
|
||||
{ "type", "string" },
|
||||
{ "description", "First parameter" }
|
||||
});
|
||||
parameters_schema["properties"]["param2"] = json::object({
|
||||
{ "type", "string" },
|
||||
{ "description", "Second parameter" }
|
||||
});
|
||||
parameters_schema["required"] = json::array({ "param1", "param2" });
|
||||
|
||||
return json::array({
|
||||
json{ { "type", "function" },
|
||||
{ "function", json{ { "name", "test_function_name" },
|
||||
{ "description", "A test function for debugging" },
|
||||
{ "parameters", parameters_schema } } } }
|
||||
});
|
||||
}
|
||||
|
||||
// Helper to create a tool call with arguments as JSON object
|
||||
static json build_tool_call(const std::string & name, const json & args_object, const std::string & id = "call_001") {
|
||||
return json{
|
||||
{"id", id},
|
||||
{"type", "function"},
|
||||
{"function", json{
|
||||
{"name", name},
|
||||
{"arguments", args_object} // Pass as JSON object, not serialized string
|
||||
}}
|
||||
};
|
||||
}
|
||||
|
||||
// Helper functions to create repeating message definitions
|
||||
static json make_user_msg() {
|
||||
return json{
|
||||
{"role", "user"},
|
||||
{"content", "Hello, please help me."}
|
||||
};
|
||||
}
|
||||
|
||||
static json make_user_msg2() {
|
||||
return json{
|
||||
{"role", "user"},
|
||||
{"content", "Thank you."}
|
||||
};
|
||||
}
|
||||
|
||||
static json make_user_msg2_continue() {
|
||||
return json{
|
||||
{"role", "user"},
|
||||
{"content", "Continue."}
|
||||
};
|
||||
}
|
||||
|
||||
static json make_assistant_no_tool() {
|
||||
return json{
|
||||
{"role", "assistant"},
|
||||
{"content", "Let me help you."}
|
||||
};
|
||||
}
|
||||
|
||||
static json make_assistant_one_tool() {
|
||||
return json{
|
||||
{"role", "assistant"},
|
||||
{"content", nullptr},
|
||||
{"tool_calls", json::array({
|
||||
build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))
|
||||
})}
|
||||
};
|
||||
}
|
||||
|
||||
static json make_assistant_two_tools() {
|
||||
return json{
|
||||
{"role", "assistant"},
|
||||
{"content", nullptr},
|
||||
{"tool_calls", json::array({
|
||||
build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}})),
|
||||
build_tool_call("test_function_name", json::object({{"param1", "value3"}, {"param2", "value4"}}), "call_002")
|
||||
})}
|
||||
};
|
||||
}
|
||||
|
||||
static json make_assistant_no_reasoning() {
|
||||
return json{
|
||||
{"role", "assistant"},
|
||||
{"content", "I can help you with that."}
|
||||
};
|
||||
}
|
||||
|
||||
static json make_assistant_with_reasoning() {
|
||||
return json{
|
||||
{"role", "assistant"},
|
||||
{"content", "I can help you with that."},
|
||||
{"reasoning_content", "The user is asking for help. I should respond positively."}
|
||||
};
|
||||
}
|
||||
|
||||
static json make_assistant_one_tool_with_reasoning() {
|
||||
return json{
|
||||
{"role", "assistant"},
|
||||
{"content", nullptr},
|
||||
{"tool_calls", json::array({
|
||||
build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}}))
|
||||
})},
|
||||
{"reasoning_content", "I need to call the tool first."}
|
||||
};
|
||||
}
|
||||
|
||||
static void print_diff_split(const std::string & title, const diff_split & diff) {
|
||||
LOG_ERR("\n%s=== %s ===%s\n", ANSI_CYAN, title.c_str(), ANSI_RESET);
|
||||
LOG_ERR("%sCommon Prefix:%s '%s'\n", ANSI_PREFIX, ANSI_RESET, diff.prefix.c_str());
|
||||
LOG_ERR("%sCommon Suffix:%s '%s'\n", ANSI_SUFFIX, ANSI_RESET, diff.suffix.c_str());
|
||||
LOG_ERR("%sLeft (difference):%s '%s'\n", ANSI_GREEN, ANSI_RESET, diff.left.c_str());
|
||||
LOG_ERR("%sRight (difference):%s '%s'\n", ANSI_ORANGE, ANSI_RESET, diff.right.c_str());
|
||||
}
|
||||
|
||||
static void check_reasoning_variables(const common_chat_template & tmpl) {
|
||||
LOG_ERR("\n%s=== Checking Reasoning Variables ===%s\n", ANSI_CYAN, ANSI_RESET);
|
||||
|
||||
try {
|
||||
// Create a list of candidate reasoning/thinking variable names to probe
|
||||
std::vector<std::string> candidate_vars = {
|
||||
"enable_reasoning",
|
||||
"use_reasoning",
|
||||
"reasoning_enabled",
|
||||
"has_reasoning",
|
||||
"reasoning_mode",
|
||||
"reasoning_format",
|
||||
"reasoning_active",
|
||||
"with_reasoning",
|
||||
"use_thinking",
|
||||
"thinking_enabled",
|
||||
"has_thinking",
|
||||
"thinking_mode",
|
||||
"thinking_format",
|
||||
"thinking_active",
|
||||
"with_thinking",
|
||||
"enable_reason",
|
||||
"reason_enabled",
|
||||
"enable_think",
|
||||
"think_enabled",
|
||||
};
|
||||
|
||||
jinja::context ctx;
|
||||
ctx.is_get_stats = true;
|
||||
|
||||
json messages = json::array({
|
||||
json{
|
||||
{"role", "user"},
|
||||
{"content", "Test message"}
|
||||
},
|
||||
json{
|
||||
{"role", "assistant"},
|
||||
{"content", "Response"},
|
||||
{"reasoning_content", "Some reasoning"}
|
||||
}
|
||||
});
|
||||
|
||||
// Set up base context
|
||||
jinja::global_from_json(ctx, json{
|
||||
{"messages", messages},
|
||||
{"tools", json::array()},
|
||||
{"bos_token", ""},
|
||||
{"eos_token", ""},
|
||||
{"add_generation_prompt", false},
|
||||
{"enable_thinking", true} // Already passed, so we'll exclude this from results
|
||||
}, true);
|
||||
|
||||
// Add candidate variables as undefined to probe which ones are accessed
|
||||
for (const auto & var_name : candidate_vars) {
|
||||
ctx.set_val(var_name, jinja::mk_val<jinja::value_undefined_t>(var_name));
|
||||
}
|
||||
|
||||
try {
|
||||
jinja::runtime runtime(ctx);
|
||||
runtime.execute(tmpl.prog);
|
||||
} catch (const std::exception & e) {
|
||||
// Execution may fail, that's okay - we just want to see what variables were accessed
|
||||
}
|
||||
|
||||
// Check which candidate variables were accessed (stats.used = true)
|
||||
std::vector<std::string> accessed_vars;
|
||||
for (const auto & var_name : candidate_vars) {
|
||||
auto val = ctx.get_val(var_name);
|
||||
if (!val->is_undefined()) {
|
||||
// Variable was overwritten, skip it
|
||||
continue;
|
||||
}
|
||||
if (val->stats.used) {
|
||||
accessed_vars.push_back(var_name);
|
||||
}
|
||||
}
|
||||
|
||||
if (accessed_vars.empty()) {
|
||||
LOG_ERR("%sNo reasoning/thinking-related variables were queried by the template%s\n", ANSI_GRAY, ANSI_RESET);
|
||||
} else {
|
||||
LOG_ERR("Template queries the following reasoning/thinking-related variables:\n");
|
||||
for (const auto & var : accessed_vars) {
|
||||
LOG_ERR(" %s- %s%s\n", ANSI_ORANGE, var.c_str(), ANSI_RESET);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("Error checking reasoning variables: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
|
||||
static void analyze_template(const std::string & template_path) {
|
||||
LOG_ERR("\n");
|
||||
LOG_ERR("%s", ANSI_PURPLE);
|
||||
LOG_ERR("================================================================================\n");
|
||||
LOG_ERR(" ANALYZING TEMPLATE: %s\n", template_path.c_str());
|
||||
LOG_ERR("================================================================================\n");
|
||||
LOG_ERR("%s", ANSI_RESET);
|
||||
|
||||
std::string template_source;
|
||||
try {
|
||||
template_source = read_file(template_path);
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("Error reading template: %s\n", e.what());
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
common_chat_template chat_template(template_source, "", "");
|
||||
json tools = build_tools_definition();
|
||||
|
||||
// ===== CAPABILITIES ANALYSIS =====
|
||||
LOG_ERR("\n%s=== Template Capabilities (from jinja::caps) ===%s\n", ANSI_CYAN, ANSI_RESET);
|
||||
auto caps = chat_template.original_caps();
|
||||
LOG_ERR("%ssupports_tools:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tools ? "true" : "false");
|
||||
LOG_ERR("%ssupports_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tool_calls ? "true" : "false");
|
||||
LOG_ERR("%ssupports_system_role:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_system_role ? "true" : "false");
|
||||
LOG_ERR("%ssupports_parallel_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_parallel_tool_calls ? "true" : "false");
|
||||
LOG_ERR("%ssupports_typed_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_typed_content ? "true" : "false");
|
||||
LOG_ERR("%ssupports_string_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_string_content ? "true" : "false");
|
||||
|
||||
// ===== DIFFERENTIAL ANALYSIS =====
|
||||
|
||||
// Test 1: With and without tools (single user message)
|
||||
{
|
||||
json user_msg = make_user_msg();
|
||||
|
||||
autoparser::generation_params params_no_tools;
|
||||
params_no_tools.messages = json::array({ user_msg });
|
||||
params_no_tools.add_generation_prompt = false;
|
||||
params_no_tools.tools = json::array();
|
||||
|
||||
autoparser::generation_params params_with_tools = params_no_tools;
|
||||
params_with_tools.tools = tools;
|
||||
|
||||
std::string output_no_tools = common_chat_template_direct_apply(chat_template, params_no_tools).str();
|
||||
std::string output_with_tools = common_chat_template_direct_apply(chat_template, params_with_tools).str();
|
||||
|
||||
auto diff = calculate_diff_split(output_no_tools, output_with_tools);
|
||||
print_diff_split("Diff: With vs Without Tools (single user message)", diff);
|
||||
}
|
||||
|
||||
// Test 2: With and without add_generation_prompt (single user message)
|
||||
{
|
||||
json user_msg = make_user_msg();
|
||||
|
||||
autoparser::generation_params params_no_prompt;
|
||||
params_no_prompt.messages = json::array({ user_msg });
|
||||
params_no_prompt.add_generation_prompt = false;
|
||||
params_no_prompt.tools = json::array();
|
||||
|
||||
autoparser::generation_params params_with_prompt = params_no_prompt;
|
||||
params_with_prompt.add_generation_prompt = true;
|
||||
|
||||
std::string output_no_prompt = common_chat_template_direct_apply(chat_template, params_no_prompt).str();
|
||||
std::string output_with_prompt = common_chat_template_direct_apply(chat_template, params_with_prompt).str();
|
||||
|
||||
auto diff = calculate_diff_split(output_no_prompt, output_with_prompt);
|
||||
print_diff_split("Diff: With vs Without add_generation_prompt (single user message)", diff);
|
||||
}
|
||||
|
||||
// Test 3: Assistant with reasoning_content (user, assistant)
|
||||
{
|
||||
json user_msg = make_user_msg();
|
||||
|
||||
autoparser::generation_params params_no_reasoning;
|
||||
params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning() });
|
||||
params_no_reasoning.add_generation_prompt = false;
|
||||
params_no_reasoning.enable_thinking = true;
|
||||
|
||||
autoparser::generation_params params_with_reasoning = params_no_reasoning;
|
||||
params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning() });
|
||||
|
||||
std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning).str();
|
||||
std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning).str();
|
||||
|
||||
auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);
|
||||
print_diff_split("Diff: With vs Without reasoning_content (user, assistant)", diff);
|
||||
}
|
||||
|
||||
// Test 4: Assistant with reasoning_content (user, assistant, user)
|
||||
{
|
||||
json user_msg = make_user_msg();
|
||||
json user_msg2 = make_user_msg2();
|
||||
|
||||
autoparser::generation_params params_no_reasoning;
|
||||
params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning(), user_msg2 });
|
||||
params_no_reasoning.add_generation_prompt = false;
|
||||
params_no_reasoning.enable_thinking = true;
|
||||
|
||||
autoparser::generation_params params_with_reasoning = params_no_reasoning;
|
||||
params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning(), user_msg2 });
|
||||
|
||||
std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning).str();
|
||||
std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning).str();
|
||||
|
||||
auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);
|
||||
print_diff_split("Diff: With vs Without reasoning_content (user, assistant, user)", diff);
|
||||
}
|
||||
|
||||
// Test 5: Tool call in last assistant message (user, assistant)
|
||||
{
|
||||
json user_msg = make_user_msg();
|
||||
|
||||
autoparser::generation_params params_no_tool;
|
||||
params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool() });
|
||||
params_no_tool.add_generation_prompt = false;
|
||||
params_no_tool.tools = tools;
|
||||
|
||||
autoparser::generation_params params_with_tool = params_no_tool;
|
||||
params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool() });
|
||||
|
||||
std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool).str();
|
||||
std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool).str();
|
||||
|
||||
auto diff = calculate_diff_split(output_no_tool, output_with_tool);
|
||||
print_diff_split("Diff: With vs Without tool call (user, assistant)", diff);
|
||||
}
|
||||
|
||||
// Test 6: Tool call in last assistant message (user, assistant, user)
|
||||
{
|
||||
json user_msg = make_user_msg();
|
||||
json user_msg2 = make_user_msg2_continue();
|
||||
|
||||
autoparser::generation_params params_no_tool;
|
||||
params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool(), user_msg2 });
|
||||
params_no_tool.add_generation_prompt = false;
|
||||
params_no_tool.tools = tools;
|
||||
|
||||
autoparser::generation_params params_with_tool = params_no_tool;
|
||||
params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 });
|
||||
|
||||
std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool).str();
|
||||
std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool).str();
|
||||
|
||||
auto diff = calculate_diff_split(output_no_tool, output_with_tool);
|
||||
print_diff_split("Diff: With vs Without tool call (user, assistant, user)", diff);
|
||||
}
|
||||
|
||||
// Test 7: One vs two tool calls (user, assistant)
|
||||
{
|
||||
json user_msg = make_user_msg();
|
||||
|
||||
autoparser::generation_params params_one_tool;
|
||||
params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool() });
|
||||
params_one_tool.add_generation_prompt = false;
|
||||
params_one_tool.tools = tools;
|
||||
|
||||
autoparser::generation_params params_two_tools = params_one_tool;
|
||||
params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools() });
|
||||
|
||||
std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool).str();
|
||||
std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools).str();
|
||||
|
||||
auto diff = calculate_diff_split(output_one_tool, output_two_tools);
|
||||
print_diff_split("Diff: One vs Two tool calls (user, assistant)", diff);
|
||||
}
|
||||
|
||||
// Test 8: One vs two tool calls (user, assistant, user)
|
||||
{
|
||||
json user_msg = make_user_msg();
|
||||
json user_msg2 = make_user_msg2_continue();
|
||||
|
||||
autoparser::generation_params params_one_tool;
|
||||
params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 });
|
||||
params_one_tool.add_generation_prompt = false;
|
||||
params_one_tool.tools = tools;
|
||||
|
||||
autoparser::generation_params params_two_tools = params_one_tool;
|
||||
params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools(), user_msg2 });
|
||||
|
||||
std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool).str();
|
||||
std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools).str();
|
||||
|
||||
auto diff = calculate_diff_split(output_one_tool, output_two_tools);
|
||||
print_diff_split("Diff: One vs Two tool calls (user, assistant, user)", diff);
|
||||
}
|
||||
|
||||
// Test 9: Tool call with vs without reasoning_content (user, assistant)
|
||||
{
|
||||
json user_msg = make_user_msg();
|
||||
|
||||
autoparser::generation_params params_no_reasoning;
|
||||
params_no_reasoning.messages = json::array({ user_msg, make_assistant_one_tool() });
|
||||
params_no_reasoning.add_generation_prompt = false;
|
||||
params_no_reasoning.tools = tools;
|
||||
params_no_reasoning.enable_thinking = true;
|
||||
|
||||
autoparser::generation_params params_with_reasoning = params_no_reasoning;
|
||||
params_with_reasoning.messages = json::array({ user_msg, make_assistant_one_tool_with_reasoning() });
|
||||
|
||||
std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning).str();
|
||||
std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning).str();
|
||||
|
||||
auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning);
|
||||
print_diff_split("Diff: Tool call with vs without reasoning_content (user, assistant)", diff);
|
||||
}
|
||||
|
||||
// Check reasoning variables
|
||||
check_reasoning_variables(chat_template);
|
||||
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("Analysis failed: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
// Set log level to capture all output
|
||||
common_log_set_verbosity_thold(99);
|
||||
|
||||
analysis_options opts;
|
||||
if (!parse_options(argc, argv, opts)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
LOG_ERR("\n");
|
||||
LOG_ERR("%s", ANSI_PURPLE);
|
||||
LOG_ERR("================================================================================\n");
|
||||
LOG_ERR(" TEMPLATE ANALYSIS TOOL\n");
|
||||
LOG_ERR("================================================================================\n");
|
||||
LOG_ERR("%s", ANSI_RESET);
|
||||
LOG_ERR("Analyzing %s%zu%s template(s)\n", ANSI_CYAN, opts.template_paths.size(), ANSI_RESET);
|
||||
|
||||
for (const auto & path : opts.template_paths) {
|
||||
analyze_template(path);
|
||||
}
|
||||
|
||||
LOG_ERR("\n");
|
||||
LOG_ERR("%s", ANSI_GREEN);
|
||||
LOG_ERR("================================================================================\n");
|
||||
LOG_ERR(" ANALYSIS COMPLETE\n");
|
||||
LOG_ERR("================================================================================\n");
|
||||
LOG_ERR("%s", ANSI_RESET);
|
||||
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,738 @@
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
#include <regex>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <filesystem>
|
||||
|
||||
#include "json.h"
|
||||
|
||||
#undef NDEBUG
|
||||
#include <cassert>
|
||||
|
||||
#include "llama.h"
|
||||
#include "common.h"
|
||||
#include "chat.h"
|
||||
#include "jinja/runtime.h"
|
||||
#include "jinja/parser.h"
|
||||
#include "jinja/lexer.h"
|
||||
#include "jinja/caps.h"
|
||||
|
||||
using json = common_json;
|
||||
|
||||
static int main_automated_tests(void);
|
||||
|
||||
static void run_multiple(const std::string& dir_path, bool stop_on_first_failure, const json& input, bool use_common = false);
|
||||
static void run_single(const std::string& contents, json input, bool use_common = false, bool dump_prog = false, const std::string & output_path = "");
|
||||
|
||||
static std::string HELP = R"(
|
||||
Test the Jinja engine by rendering chat templates and comparing the output against expected results.
|
||||
|
||||
Usage: test-chat-template [OPTIONS] PATH_TO_TEMPLATE
|
||||
Options:
|
||||
-h, --help Show this help message and exit.
|
||||
--with-tools Add a tool and a tool call to the default JSON input
|
||||
--json <path> Path to the JSON input file.
|
||||
--stop-on-first-fail Stop testing on the first failure (default: false).
|
||||
--no-common Use direct Jinja engine instead of common chat templates (default: use common).
|
||||
--dump-prog Dump the parsed program for debugging (only for single template runs).
|
||||
--output <path> Path to output results (only for single template runs).
|
||||
If PATH_TO_TEMPLATE is a file, runs that single template.
|
||||
If PATH_TO_TEMPLATE is a directory, runs all .jinja files in that directory.
|
||||
If PATH_TO_TEMPLATE is omitted, runs automated tests (default CI mode).
|
||||
)";
|
||||
|
||||
static std::string DEFAULT_JSON = R"({
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, how are you?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "I am fine, thank you!"
|
||||
}
|
||||
],
|
||||
"bos_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"add_generation_prompt": true
|
||||
})";
|
||||
|
||||
static std::string DEFAULT_JSON_WITH_TOOLS = R"({
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, how are you?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "I am fine, thank you!"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Call a tool!"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call00001",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "test",
|
||||
"arguments": { "arg": "hello" }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "test",
|
||||
"description": "Test",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"arg": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["arg"]
|
||||
}
|
||||
}
|
||||
],
|
||||
"bos_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"add_generation_prompt": true
|
||||
})";
|
||||
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
std::vector<std::string> args(argv, argv + argc);
|
||||
|
||||
std::string tmpl_path;
|
||||
std::string json_path;
|
||||
std::string output_path;
|
||||
std::string & json_to_use = DEFAULT_JSON;
|
||||
bool stop_on_first_fail = false;
|
||||
bool use_common = true;
|
||||
bool dump_prog = false;
|
||||
|
||||
for (size_t i = 1; i < args.size(); i++) {
|
||||
if (args[i] == "--help" || args[i] == "-h") {
|
||||
std::cout << HELP << "\n";
|
||||
return 0;
|
||||
}
|
||||
if (args[i] == "--json" && i + 1 < args.size()) {
|
||||
json_path = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] == "--with-tools") {
|
||||
json_to_use = DEFAULT_JSON_WITH_TOOLS;
|
||||
} else if (args[i] == "--stop-on-first-fail") {
|
||||
stop_on_first_fail = true;
|
||||
} else if (args[i] == "--output" && i + 1 < args.size()) {
|
||||
output_path = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] == "--no-common") {
|
||||
use_common = false;
|
||||
} else if (args[i] == "--dump-prog") {
|
||||
dump_prog = true;
|
||||
} else if (tmpl_path.empty()) {
|
||||
tmpl_path = args[i];
|
||||
} else {
|
||||
std::cerr << "Unknown argument: " << args[i] << "\n";
|
||||
std::cout << HELP << "\n";
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (tmpl_path.empty()) {
|
||||
return main_automated_tests();
|
||||
}
|
||||
|
||||
json input_json;
|
||||
if (!json_path.empty()) {
|
||||
std::ifstream json_file(json_path);
|
||||
if (!json_file) {
|
||||
std::cerr << "Error: Could not open JSON file: " << json_path << "\n";
|
||||
return 1;
|
||||
}
|
||||
std::string content = std::string(
|
||||
std::istreambuf_iterator<char>(json_file),
|
||||
std::istreambuf_iterator<char>());
|
||||
input_json = json::parse(content);
|
||||
} else {
|
||||
input_json = json::parse(json_to_use);
|
||||
}
|
||||
|
||||
std::filesystem::path p(tmpl_path);
|
||||
if (std::filesystem::is_directory(p)) {
|
||||
run_multiple(tmpl_path, stop_on_first_fail, input_json, use_common);
|
||||
} else if (std::filesystem::is_regular_file(p)) {
|
||||
std::ifstream infile(tmpl_path);
|
||||
std::string contents = std::string(
|
||||
std::istreambuf_iterator<char>(infile),
|
||||
std::istreambuf_iterator<char>());
|
||||
run_single(contents, input_json, use_common, dump_prog, output_path);
|
||||
} else {
|
||||
std::cerr << "Error: PATH_TO_TEMPLATE is not a valid file or directory: " << tmpl_path << "\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void run_multiple(const std::string& dir_path, bool stop_on_first_fail, const json& input, bool use_common) {
|
||||
std::vector<std::string> failed_tests;
|
||||
|
||||
// list all files in models/templates/ and run each
|
||||
size_t test_count = 0;
|
||||
|
||||
for (const auto & entry : std::filesystem::directory_iterator(dir_path)) {
|
||||
// only process .jinja files
|
||||
if (entry.path().extension() == ".jinja" && entry.is_regular_file()) {
|
||||
test_count++;
|
||||
std::cout << "\n\n=== RUNNING TEMPLATE FILE: " << entry.path().string() << " ===\n";
|
||||
std::ifstream infile(entry.path());
|
||||
std::string contents((std::istreambuf_iterator<char>(infile)), std::istreambuf_iterator<char>());
|
||||
try {
|
||||
run_single(contents, input, use_common);
|
||||
} catch (const std::exception & e) {
|
||||
std::cout << "Exception: " << e.what() << "\n";
|
||||
std::cout << "=== ERROR WITH TEMPLATE FILE: " << entry.path().string() << " ===\n";
|
||||
failed_tests.push_back(entry.path().string());
|
||||
if (stop_on_first_fail) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "\n\n=== TEST SUMMARY ===\n";
|
||||
std::cout << "Total tests run: " << test_count << "\n";
|
||||
std::cout << "Total failed tests: " << failed_tests.size() << "\n";
|
||||
for (const auto & test : failed_tests) {
|
||||
std::cout << "FAILED TEST: " << test << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static std::string normalize_newlines(const std::string & s) {
|
||||
#ifdef _WIN32
|
||||
static const std::regex nl_regex("\r\n");
|
||||
return std::regex_replace(s, nl_regex, "\n");
|
||||
#else
|
||||
return s;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
static std::string format_using_common(
|
||||
const std::string & template_str,
|
||||
const std::string & bos_token,
|
||||
const std::string & eos_token,
|
||||
std::vector<common_chat_msg> & messages,
|
||||
std::vector<common_chat_tool> tools = {}) {
|
||||
auto tmpls = common_chat_templates_init(/* model= */ nullptr, template_str, bos_token, eos_token);
|
||||
common_chat_templates_inputs inputs;
|
||||
inputs.use_jinja = true;
|
||||
inputs.messages = messages;
|
||||
inputs.tools = std::move(tools);
|
||||
inputs.add_generation_prompt = true;
|
||||
auto output = common_chat_templates_apply(tmpls.get(), inputs).prompt.str();
|
||||
output = normalize_newlines(output);
|
||||
return output;
|
||||
}
|
||||
|
||||
|
||||
// skip libcommon, use direct jinja engine
|
||||
static jinja::value_string format_using_direct_engine(
|
||||
const std::string & template_str,
|
||||
json & input) {
|
||||
// lexing
|
||||
jinja::lexer lexer;
|
||||
auto lexer_res = lexer.tokenize(template_str);
|
||||
|
||||
// compile to AST
|
||||
jinja::program ast = jinja::parse_from_tokens(lexer_res);
|
||||
|
||||
// check caps for workarounds
|
||||
jinja::caps_get(ast);
|
||||
|
||||
std::cout << "\n=== RUN ===\n";
|
||||
jinja::context ctx(template_str);
|
||||
|
||||
jinja::global_from_json(ctx, input, true);
|
||||
|
||||
jinja::runtime runtime(ctx);
|
||||
const jinja::value results = runtime.execute(ast);
|
||||
auto parts = jinja::runtime::gather_string_parts(results);
|
||||
|
||||
std::cout << "\n=== RESULTS ===\n";
|
||||
for (const auto & part : parts->as_string().parts) {
|
||||
std::cout << (part.is_input ? "DATA" : "TMPL") << ": " << part.val << "\n";
|
||||
}
|
||||
|
||||
return parts;
|
||||
}
|
||||
|
||||
|
||||
void run_single(const std::string& contents, json input, bool use_common, bool dump_prog, const std::string & output_path) {
|
||||
jinja::enable_debug(true);
|
||||
|
||||
jinja::value_string output_parts;
|
||||
|
||||
if (dump_prog) {
|
||||
jinja::lexer lexer;
|
||||
auto lexer_res = lexer.tokenize(contents);
|
||||
jinja::program ast = jinja::parse_from_tokens(lexer_res);
|
||||
std::string prog_dump = jinja::runtime::debug_dump_program(ast, contents);
|
||||
std::cout << "\n=== DUMPED PROGRAM ===\n";
|
||||
std::cout << prog_dump << "\n";
|
||||
return;
|
||||
}
|
||||
|
||||
if (use_common) {
|
||||
std::string bos_token = "<s>";
|
||||
std::string eos_token = "</s>";
|
||||
if (input.contains("bos_token")) {
|
||||
bos_token = input["bos_token"].get<std::string>();
|
||||
}
|
||||
if (input.contains("eos_token")) {
|
||||
eos_token = input["eos_token"].get<std::string>();
|
||||
}
|
||||
common_json msgs_json = input["messages"];
|
||||
common_json tools_json = input["tools"];
|
||||
auto messages = common_chat_msgs_parse_oaicompat(msgs_json);
|
||||
auto tools = common_chat_tools_parse_oaicompat(tools_json);
|
||||
auto output = format_using_common(contents, bos_token, eos_token, messages, tools);
|
||||
std::cout << "\n=== OUTPUT ===\n";
|
||||
std::cout << output << "\n";
|
||||
output_parts = jinja::mk_val<jinja::value_string>(output);
|
||||
|
||||
} else {
|
||||
output_parts = format_using_direct_engine(contents, input);
|
||||
std::cout << "\n=== OUTPUT ===\n";
|
||||
std::cout << output_parts->as_string().str() << "\n";
|
||||
}
|
||||
|
||||
if (!output_path.empty()) {
|
||||
std::ofstream outfile(output_path);
|
||||
if (!outfile) {
|
||||
throw std::runtime_error("Could not open output file: " + output_path);
|
||||
}
|
||||
outfile << output_parts->as_string().str();
|
||||
outfile.close();
|
||||
std::cout << "\n=== OUTPUT WRITTEN TO " << output_path << " ===\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
//
|
||||
// Automated tests for chat templates
|
||||
//
|
||||
|
||||
#define U8C(x) (const char*)(u8##x)
|
||||
|
||||
static common_chat_msg simple_msg(const std::string & role, const std::string & content) {
|
||||
common_chat_msg msg;
|
||||
msg.role = role;
|
||||
msg.content = content;
|
||||
return msg;
|
||||
}
|
||||
|
||||
int main_automated_tests(void) {
|
||||
// jinja::enable_debug(true);
|
||||
|
||||
std::vector<llama_chat_message> conversation {
|
||||
{"system", "You are a helpful assistant"},
|
||||
{"user", "Hello"},
|
||||
{"assistant", "Hi there"},
|
||||
{"user", "Who are you"},
|
||||
{"assistant", " I am an assistant "},
|
||||
{"user", "Another question"},
|
||||
};
|
||||
|
||||
// std::string wrong = /* .template_str= */ u8"[gMASK]<sop>{% for item in messages %}{% if item['tools'] is defined %}<|system|>\n你是一个名为 ChatGLM 的人工智能助手。你是基于智谱AI训练的语言模型 GLM-4 模型开发的，你的任务是针对用户的问题和要求提供适当的答复和支持。\n\n# 可用工具{% set tools = item['tools'] %}{% for tool in tools %}{% if tool['type'] == 'function' %}\n\n## {{ tool['function']['name'] }}\n\n{{ tool['function'] | tojson(indent=4) }}\n......{% endif %}{% endfor %}{% endif %}{% if item['content'] %}<|{{ item['role'] }}|>{{ item['metadata'] }}\n{{ item['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}";
|
||||
struct TestCase {
|
||||
std::string name;
|
||||
std::string template_str;
|
||||
std::string expected_output;
|
||||
std::string expected_output_jinja;
|
||||
std::string bos_token = "";
|
||||
std::string eos_token = "";
|
||||
bool supported_with_jinja = true;
|
||||
std::vector<llama_chat_message> extra_conversation = {};
|
||||
};
|
||||
std::vector<TestCase> test_cases {
|
||||
{
|
||||
/* .name= */ "teknium/OpenHermes-2.5-Mistral-7B",
|
||||
/* .template_str= */ "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}",
|
||||
/* .expected_output= */ "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nWho are you<|im_end|>\n<|im_start|>assistant\n I am an assistant <|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant\n",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "mistralai/Mistral-7B-Instruct-v0.2 (NOTE: Old pre-v1 without a system prompt)",
|
||||
/* .template_str= */ "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
|
||||
/* .expected_output= */ "[INST] You are a helpful assistant\nHello [/INST]Hi there</s>[INST] Who are you [/INST] I am an assistant </s>[INST] Another question [/INST]",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "</s>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "TheBloke/FusionNet_34Bx2_MoE-AWQ",
|
||||
/* .template_str= */ "{%- for idx in range(0, messages|length) -%}\n{%- if messages[idx]['role'] == 'user' -%}\n{%- if idx > 1 -%}\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\n{%- else -%}\n{{- messages[idx]['content'] + ' [/INST]' -}}\n{%- endif -%}\n{% elif messages[idx]['role'] == 'system' %}\n{{- '[INST] <<SYS>>\\n' + messages[idx]['content'] + '\\n<</SYS>>\\n\\n' -}}\n{%- elif messages[idx]['role'] == 'assistant' -%}\n{{- ' ' + messages[idx]['content'] + ' ' + eos_token -}}\n{% endif %}\n{% endfor %}",
|
||||
/* .expected_output= */ "[INST] <<SYS>>\nYou are a helpful assistant\n<</SYS>>\n\nHello [/INST]Hi there</s><s>[INST] Who are you [/INST] I am an assistant </s><s>[INST] Another question [/INST]",
|
||||
/* .expected_output_jinja= */ "[INST] <<SYS>>\nYou are a helpful assistant\n<</SYS>>\n\nHello [/INST] Hi there </s><s>[INST] Who are you [/INST] I am an assistant </s><s>[INST] Another question [/INST]",
|
||||
/* .bos_token= */ "<s>",
|
||||
/* .eos_token= */ "</s>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "bofenghuang/vigogne-2-70b-chat",
|
||||
/* .template_str= */ "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif true == true and not '<<SYS>>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'Vous êtes Vigogne, un assistant IA créé par Zaion Lab. Vous suivez extrêmement bien les instructions. Aidez autant que vous le pouvez.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<<SYS>>\\n' + content.strip() + '\\n<</SYS>>\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
|
||||
/* .expected_output= */ "[INST] <<SYS>>\nYou are a helpful assistant\n<</SYS>>\n\nHello [/INST]Hi there</s>[INST] Who are you [/INST]I am an assistant</s>[INST] Another question [/INST]",
|
||||
/* .expected_output_jinja= */ "[INST] <<SYS>>\nYou are a helpful assistant\n<</SYS>>\n\nHello [/INST] Hi there </s>[INST] Who are you [/INST] I am an assistant </s>[INST] Another question [/INST]",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "</s>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "mlabonne/AlphaMonarch-7B",
|
||||
/* .template_str= */ "{% for message in messages %}{{bos_token + message['role'] + '\\n' + message['content'] + eos_token + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ bos_token + 'assistant\\n' }}{% endif %}",
|
||||
/* .expected_output= */ "system\nYou are a helpful assistant</s>\n<s>user\nHello</s>\n<s>assistant\nHi there</s>\n<s>user\nWho are you</s>\n<s>assistant\n I am an assistant </s>\n<s>user\nAnother question</s>\n<s>assistant\n",
|
||||
/* .expected_output_jinja= */ "<s>system\nYou are a helpful assistant</s>\n<s>user\nHello</s>\n<s>assistant\nHi there</s>\n<s>user\nWho are you</s>\n<s>assistant\n I am an assistant </s>\n<s>user\nAnother question</s>\n<s>assistant\n",
|
||||
/* .bos_token= */ "<s>",
|
||||
/* .eos_token= */ "</s>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "google/gemma-7b-it",
|
||||
/* .template_str= */ "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\\n' + message['content'] | trim + '<end_of_turn>\\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\\n'}}{% endif %}",
|
||||
/* .expected_output= */ "<start_of_turn>user\nYou are a helpful assistant\n\nHello<end_of_turn>\n<start_of_turn>model\nHi there<end_of_turn>\n<start_of_turn>user\nWho are you<end_of_turn>\n<start_of_turn>model\nI am an assistant<end_of_turn>\n<start_of_turn>user\nAnother question<end_of_turn>\n<start_of_turn>model\n",
|
||||
/* .expected_output_jinja= */ "<start_of_turn>user\nYou are a helpful assistant\nHello<end_of_turn>\n<start_of_turn>model\nHi there<end_of_turn>\n<start_of_turn>user\nWho are you<end_of_turn>\n<start_of_turn>model\nI am an assistant<end_of_turn>\n<start_of_turn>user\nAnother question<end_of_turn>\n<start_of_turn>model\n",
|
||||
},
|
||||
{
|
||||
/* .name= */ "OrionStarAI/Orion-14B-Chat",
|
||||
/* .template_str= */ "{% for message in messages %}{% if loop.first %}{{ bos_token }}{% endif %}{% if message['role'] == 'user' %}{{ 'Human: ' + message['content'] + '\\n\\nAssistant: ' + eos_token }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token }}{% endif %}{% endfor %}",
|
||||
/* .expected_output= */ "Human: You are a helpful assistant\n\nHello\n\nAssistant: </s>Hi there</s>Human: Who are you\n\nAssistant: </s> I am an assistant </s>Human: Another question\n\nAssistant: </s>",
|
||||
/* .expected_output_jinja= */ "Human: You are a helpful assistant\nHello\n\nAssistant: </s>Hi there</s>Human: Who are you\n\nAssistant: </s> I am an assistant </s>Human: Another question\n\nAssistant: </s>",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "</s>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "openchat/openchat-3.5-0106",
|
||||
// The included chat_template differs from the author's suggestions here: https://huggingface.co/openchat/openchat_3.5/discussions/5#65448109b4a3f3a2f486fd9d
|
||||
// So we match against the included template but implement the suggested version.
|
||||
/* .template_str= */ "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}",
|
||||
/* .expected_output= */ "You are a helpful assistant<|end_of_turn|>GPT4 Correct User: Hello<|end_of_turn|>GPT4 Correct Assistant: Hi there<|end_of_turn|>GPT4 Correct User: Who are you<|end_of_turn|>GPT4 Correct Assistant: I am an assistant <|end_of_turn|>GPT4 Correct User: Another question<|end_of_turn|>GPT4 Correct Assistant:",
|
||||
/* .expected_output_jinja= */ "GPT4 Correct System: You are a helpful assistant<|end_of_turn|>GPT4 Correct User: Hello<|end_of_turn|>GPT4 Correct Assistant: Hi there<|end_of_turn|>GPT4 Correct User: Who are you<|end_of_turn|>GPT4 Correct Assistant: I am an assistant <|end_of_turn|>GPT4 Correct User: Another question<|end_of_turn|>GPT4 Correct Assistant:",
|
||||
},
|
||||
{
|
||||
/* .name= */ "deepseek-ai/deepseek-coder-33b-instruct",
|
||||
/* .template_str= */ "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
|
||||
/* .expected_output= */ "You are a helpful assistant### Instruction:\nHello\n### Response:\nHi there\n<|EOT|>\n### Instruction:\nWho are you\n### Response:\n I am an assistant \n<|EOT|>\n### Instruction:\nAnother question\n### Response:\n",
|
||||
/* .expected_output_jinja= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "eachadea/vicuna-13b-1.1",
|
||||
// No template included in tokenizer_config.json, so this template likely needs to be manually set.
|
||||
/* .template_str= */ "{%- for message in messages %}{%- if message['role'] == 'system' -%}{{- '' + message['content'] + '\n\n' -}}{%- else -%}{%- if message['role'] == 'user' -%}{{-'USER: ' + message['content'] + '\n'-}}{%- else -%}{{-'ASSISTANT: ' + message['content'] + '</s>\n' -}}{%- endif -%}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{-'ASSISTANT:'-}}{%- endif -%}",
|
||||
/* .expected_output= */ "You are a helpful assistant\n\nUSER: Hello\nASSISTANT: Hi there</s>\nUSER: Who are you\nASSISTANT: I am an assistant </s>\nUSER: Another question\nASSISTANT:",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "Orca-Vicuna",
|
||||
// No template included in tokenizer_config.json, so this template likely needs to be manually set.
|
||||
/* .template_str= */ "{%- for message in messages %}{%- if message['role'] == 'system' -%}{{-'SYSTEM: ' + message['content'] + '\n' -}}{%- else -%}{%- if message['role'] == 'user' -%}{{-'USER: ' + message['content'] + '\n'-}}{%- else -%}{{-'ASSISTANT: ' + message['content'] + '</s>\n' -}}{%- endif -%}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{-'ASSISTANT:'-}}{%- endif -%}",
|
||||
/* .expected_output= */ "SYSTEM: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there</s>\nUSER: Who are you\nASSISTANT: I am an assistant </s>\nUSER: Another question\nASSISTANT:",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "CohereForAI/c4ai-command-r-plus",
|
||||
/* .template_str= */ "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif false == true %}{% set loop_messages = messages %}{% set system_message = 'You are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by providing thorough responses. You are trained by Cohere.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% if system_message != false %}{{ '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + system_message + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% elif message['role'] == 'assistant' %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}{% endif %}",
|
||||
/* .expected_output= */ "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>You are a helpful assistant<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>Hi there<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Who are you<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>I am an assistant<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Another question<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
|
||||
/* .expected_output_jinja= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "Llama-3",
|
||||
/* .template_str= */ "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}",
|
||||
/* .expected_output= */ "<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nHi there<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWho are you<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nI am an assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nAnother question<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
|
||||
/* .expected_output_jinja= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "Phi-3-mini",
|
||||
/* .template_str= */ "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
|
||||
/* .expected_output= */ "<|system|>\nYou are a helpful assistant<|end|>\n<|user|>\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n",
|
||||
/* .expected_output_jinja= */ "<|user|>\nYou are a helpful assistant\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n",
|
||||
},
|
||||
{
|
||||
/* .name= */ "Phi-3-small",
|
||||
/* .template_str= */ "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
|
||||
/* .expected_output= */ "<|system|>\nYou are a helpful assistant<|end|>\n<|user|>\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n",
|
||||
/* .expected_output_jinja= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "Phi-3-medium",
|
||||
/* .template_str= */ "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
|
||||
/* .expected_output= */ "<|system|>\nYou are a helpful assistant<|end|>\n<|user|>\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n",
|
||||
/* .expected_output_jinja= */ "<|user|>\nYou are a helpful assistant\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n",
|
||||
},
|
||||
{
|
||||
/* .name= */ "Phi-3-vision",
|
||||
/* .template_str= */ "{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{- '<|assistant|>\n' -}}{% endif %}",
|
||||
/* .expected_output= */ "<|system|>\nYou are a helpful assistant<|end|>\n<|user|>\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "ChatGLM3",
|
||||
/* .template_str= */ "{% for message in messages %}{% if loop.first %}[gMASK]sop<|{{ message['role'] }}|>\n {{ message['content'] }}{% else %}<|{{ message['role'] }}|>\n {{ message['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}",
|
||||
/* .expected_output= */ "[gMASK]sop<|system|>\n You are a helpful assistant<|user|>\n Hello<|assistant|>\n Hi there<|user|>\n Who are you<|assistant|>\n I am an assistant <|user|>\n Another question<|assistant|>",
|
||||
/* .expected_output_jinja= */ "[gMASK]sop<|system|>\n You are a helpful assistant<|user|>\n Hello<|assistant|>\n Hi there<|user|>\n Who are you<|assistant|>\n I am an assistant <|user|>\n Another question<|assistant|>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "ChatGLM4",
|
||||
/* .template_str= */ U8C("[gMASK]<sop>{% for item in messages %}{% if item['tools'] is defined %}<|system|>\n你是一个名为 ChatGLM 的人工智能助手。你是基于智谱AI训练的语言模型 GLM-4 模型开发的，你的任务是针对用户的问题和要求提供适当的答复和支持。\n\n# 可用工具{% set tools = item['tools'] %}{% for tool in tools %}{% if tool['type'] == 'function' %}\n\n## {{ tool['function']['name'] }}\n\n{{ tool['function'] | tojson(indent=4) }}\n......{% endif %}{% endfor %}{% endif %}{% if item['content'] %}<|{{ item['role'] }}|>{{ item['metadata'] }}\n{{ item['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>\n{% endif %}"),
|
||||
/* .expected_output= */ "[gMASK]<sop><|system|>\nYou are a helpful assistant<|user|>\nHello<|assistant|>\nHi there<|user|>\nWho are you<|assistant|>\n I am an assistant <|user|>\nAnother question<|assistant|>\n",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "GLMEdge",
|
||||
/* .template_str= */ "{% for item in messages %}{% if item['role'] == 'system' %}<|system|>\n{{ item['content'] }}{% elif item['role'] == 'user' %}<|user|>\n{{ item['content'] }}{% elif item['role'] == 'assistant' %}<|assistant|>\n{{ item['content'] }}{% endif %}{% endfor %}<|assistant|>",
|
||||
/* .expected_output= */ "<|system|>\nYou are a helpful assistant<|user|>\nHello<|assistant|>\nHi there<|user|>\nWho are you<|assistant|>\n I am an assistant <|user|>\nAnother question<|assistant|>",
|
||||
/* .expected_output_jinja= */ "<|system|>\nYou are a helpful assistant<|user|>\nHello<|assistant|>\nHi there<|user|>\nWho are you<|assistant|>\n I am an assistant <|user|>\nAnother question<|assistant|>",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "MiniCPM-3B-OpenHermes-2.5-v2-GGUF",
|
||||
/* .template_str= */ U8C("{% for message in messages %}{% if message['role'] == 'user' %}{{'<用户>' + message['content'].strip() + '<AI>'}}{% else %}{{message['content'].strip()}}{% endif %}{% endfor %}"),
|
||||
/* .expected_output= */ U8C("You are a helpful assistant<用户>Hello<AI>Hi there<用户>Who are you<AI>I am an assistant<用户>Another question<AI>"),
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "DeepSeek-V2",
|
||||
/* .template_str= */ "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ 'User: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ message['content'] + '\n\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}",
|
||||
/* .expected_output= */ U8C("You are a helpful assistant\n\nUser: Hello\n\nAssistant: Hi there<｜end▁of▁sentence｜>User: Who are you\n\nAssistant: I am an assistant <｜end▁of▁sentence｜>User: Another question\n\nAssistant:"),
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "<｜end▁of▁sentence｜>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "ibm-granite/granite-3.0-8b-instruct",
|
||||
/* .template_str= */ "{%- if tools %}\n {{- '<|start_of_role|>available_tools<|end_of_role|>\n' }}\n {%- for tool in tools %}\n {{- tool | tojson(indent=4) }}\n {%- if not loop.last %}\n {{- '\n\n' }}\n {%- endif %}\n {%- endfor %}\n {{- '<|end_of_text|>\n' }}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n {{- '<|start_of_role|>system<|end_of_role|>' + message['content'] + '<|end_of_text|>\n' }}\n {%- elif message['role'] == 'user' %}\n {{- '<|start_of_role|>user<|end_of_role|>' + message['content'] + '<|end_of_text|>\n' }}\n {%- elif message['role'] == 'assistant' %}\n {{- '<|start_of_role|>assistant<|end_of_role|>' + message['content'] + '<|end_of_text|>\n' }}\n {%- elif message['role'] == 'assistant_tool_call' %}\n {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'] + '<|end_of_text|>\n' }}\n {%- elif message['role'] == 'tool_response' %}\n {{- '<|start_of_role|>tool_response<|end_of_role|>' + message['content'] + '<|end_of_text|>\n' }}\n {%- endif %}\n {%- if loop.last and add_generation_prompt %}\n {{- '<|start_of_role|>assistant<|end_of_role|>' }}\n {%- endif %}\n{%- endfor %}",
|
||||
/* .expected_output= */ "<|start_of_role|>system<|end_of_role|>You are a helpful assistant<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Hello<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>Hi there<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Who are you<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|> I am an assistant <|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Another question<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>",
|
||||
/* .expected_output_jinja= */ "<|start_of_role|>system<|end_of_role|>You are a helpful assistant<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Hello<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>Hi there<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Who are you<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|> I am an assistant <|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Another question<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "mistralai/Mistral-7B-Instruct-v0.2 (mistralai 'v1' template with a system prompt)",
|
||||
/* .template_str= */ "{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set loop_messages = messages %}\n{%- endif %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n {{- raise_exception('After the optional system message, conversation roles must alternate user/assistant/user/assistant/...') }}\n {%- endif %}\n {%- if message['role'] == 'user' %}\n {%- if loop.first and system_message is defined %}\n {{- ' [INST] ' + system_message + '\\n\\n' + message['content'] + ' [/INST]' }}\n {%- else %}\n {{- ' [INST] ' + message['content'] + ' [/INST]' }}\n {%- endif %}\n {%- elif message['role'] == 'assistant' %}\n {{- ' ' + message['content'] + eos_token}}\n {%- else %}\n {{- raise_exception('Only user and assistant roles are supported, with the exception of an initial optional system message!') }}\n {%- endif %}\n{%- endfor %}\n",
|
||||
/* .expected_output= */ " [INST] You are a helpful assistant\n\nHello [/INST] Hi there</s> [INST] Who are you [/INST] I am an assistant </s> [INST] Another question [/INST]",
|
||||
/* .expected_output_jinja= */ " [INST] You are a helpful assistant\n\nHello [/INST] Hi there</s> [INST] Who are you [/INST] I am an assistant </s> [INST] Another question [/INST]",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "</s>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "Mistral-Large-Instruct-2407 (mistralai 'v3' template; modified to have system prompt at start)",
|
||||
/* .template_str= */ "{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}\n{%- set ns = namespace() %}\n{%- set ns.index = 0 %}\n{%- for message in loop_messages %}\n {%- if not (message.role == \"tool\" or message.role == \"tool_results\" or (message.tool_calls is defined and message.tool_calls is not none)) %}\n {%- if (message[\"role\"] == \"user\") != (ns.index % 2 == 0) %}\n {{- raise_exception(\"After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif %}\n {%- set ns.index = ns.index + 1 %}\n {%- endif %}\n{%- endfor %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"user\" %}\n {%- if tools is not none and (message == user_messages[-1]) %}\n {{- \"[AVAILABLE_TOOLS] [\" }}\n {%- for tool in tools %}\n {%- set tool = tool.function %}\n {{- '{\"type\": \"function\", \"function\": {' }}\n {%- for key, val in tool.items() if key != \"return\" %}\n {%- if val is string %}\n {{- '\"' + key + '\": \"' + val + '\"' }}\n {%- else %}\n {{- '\"' + key + '\": ' + val|tojson }}\n {%- endif %}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \"}}\" }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" }}\n {%- endif %}\n {%- endfor %}\n {{- \"[/AVAILABLE_TOOLS]\" }}\n {%- endif %}\n {%- if loop.last and system_message is defined %}\n {{- \"[INST] \" + system_message + \"\\n\\n\" + message[\"content\"] + \"[/INST]\" }}\n {%- else %}\n {{- \"[INST] \" + message[\"content\"] + \"[/INST]\" }}\n {%- endif %}\n {%- elif message.tool_calls is defined and message.tool_calls is not none %}\n {{- \"[TOOL_CALLS] [\" }}\n {%- for tool_call in message.tool_calls %}\n {%- set out = tool_call.function|tojson %}\n {{- out[:-1] }}\n {%- if not tool_call.id is defined or tool_call.id|length != 9 %}\n {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n {%- endif %}\n {{- ', \"id\": \"' + tool_call.id + '\"}' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" + eos_token }}\n {%- endif %}\n {%- endfor %}\n {%- elif message[\"role\"] == \"assistant\" %}\n {{- \" \" + message[\"content\"]|trim + eos_token}}\n {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n {%- if message.content is defined and message.content.content is defined %}\n {%- set content = message.content.content %}\n {%- else %}\n {%- set content = message.content %}\n {%- endif %}\n {{- '[TOOL_RESULTS] {\"content\": ' + content|string + \", \" }}\n {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}\n {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n {%- endif %}\n {{- '\"call_id\": \"' + message.tool_call_id + '\"}[/TOOL_RESULTS]' }}\n {%- else %}\n {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}\n",
|
||||
/* .expected_output= */ "[INST] You are a helpful assistant\n\nHello[/INST] Hi there</s>[INST] Who are you[/INST] I am an assistant</s>[INST] Another question[/INST]",
|
||||
/* .expected_output_jinja= */ "[INST] Hello[/INST] Hi there</s>[INST] Who are you[/INST] I am an assistant</s>[INST] You are a helpful assistant\n\nAnother question[/INST]",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "</s>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "Mistral-Nemo-Instruct-2407 (mistralai 'v3-tekken' template; modified to have system prompt at start)",
|
||||
/* .template_str= */ "{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}\n{%- set ns = namespace() %}\n{%- set ns.index = 0 %}\n{%- for message in loop_messages %}\n {%- if not (message.role == \"tool\" or message.role == \"tool_results\" or (message.tool_calls is defined and message.tool_calls is not none)) %}\n {%- if (message[\"role\"] == \"user\") != (ns.index % 2 == 0) %}\n {{- raise_exception(\"After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif %}\n {%- set ns.index = ns.index + 1 %}\n {%- endif %}\n{%- endfor %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"user\" %}\n {%- if tools is not none and (message == user_messages[-1]) %}\n {{- \"[AVAILABLE_TOOLS][\" }}\n {%- for tool in tools %}\n {%- set tool = tool.function %}\n {{- '{\"type\": \"function\", \"function\": {' }}\n {%- for key, val in tool.items() if key != \"return\" %}\n {%- if val is string %}\n {{- '\"' + key + '\": \"' + val + '\"' }}\n {%- else %}\n {{- '\"' + key + '\": ' + val|tojson }}\n {%- endif %}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \"}}\" }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" }}\n {%- endif %}\n {%- endfor %}\n {{- \"[/AVAILABLE_TOOLS]\" }}\n {%- endif %}\n {%- if loop.last and system_message is defined %}\n {{- \"[INST]\" + system_message + \"\\n\\n\" + message[\"content\"] + \"[/INST]\" }}\n {%- else %}\n {{- \"[INST]\" + message[\"content\"] + \"[/INST]\" }}\n {%- endif %}\n {%- elif (message.tool_calls is defined and message.tool_calls is not none) %}\n {{- \"[TOOL_CALLS][\" }}\n {%- for tool_call in message.tool_calls %}\n {%- set out = tool_call.function|tojson %}\n {{- out[:-1] }}\n {%- if not tool_call.id is defined or tool_call.id|length != 9 %}\n {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n {%- endif %}\n {{- ', \"id\": \"' + tool_call.id + '\"}' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" + eos_token }}\n {%- endif %}\n {%- endfor %}\n {%- elif message[\"role\"] == \"assistant\" %}\n {{- message[\"content\"] + eos_token}}\n {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n {%- if message.content is defined and message.content.content is defined %}\n {%- set content = message.content.content %}\n {%- else %}\n {%- set content = message.content %}\n {%- endif %}\n {{- '[TOOL_RESULTS]{\"content\": ' + content|string + \", \" }}\n {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}\n {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n {%- endif %}\n {{- '\"call_id\": \"' + message.tool_call_id + '\"}[/TOOL_RESULTS]' }}\n {%- else %}\n {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}\n",
|
||||
/* .expected_output= */ "[INST]You are a helpful assistant\n\nHello[/INST]Hi there</s>[INST]Who are you[/INST] I am an assistant </s>[INST]Another question[/INST]",
|
||||
/* .expected_output_jinja= */ "[INST]Hello[/INST]Hi there</s>[INST]Who are you[/INST] I am an assistant </s>[INST]You are a helpful assistant\n\nAnother question[/INST]",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "</s>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "mistralai/Mistral-Large-Instruct-2411 (mistralai 'v7' template)",
|
||||
/* .template_str= */ "{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + '[/INST]' }}{% elif message['role'] == 'system' %}{{ '[SYSTEM_PROMPT] ' + message['content'] + '[/SYSTEM_PROMPT]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + message['content'] + eos_token }}{% else %}{{ raise_exception('Only user, system and assistant roles are supported!') }}{% endif %}{% endfor %}",
|
||||
/* .expected_output= */ "[SYSTEM_PROMPT] You are a helpful assistant[/SYSTEM_PROMPT][INST] Hello[/INST] Hi there</s>[INST] Who are you[/INST] I am an assistant </s>[INST] Another question[/INST]",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "</s>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "ai-sage/GigaChat-20B-A3B-instruct",
|
||||
/* .template_str= */ "{% if messages[0]['role'] == 'system' -%}\n {%- set loop_messages = messages[1:] -%}\n {%- set system_message = bos_token + messages[0]['content'] + additional_special_tokens[1] -%}\n{%- else -%}\n {%- set loop_messages = messages -%}\n {%- set system_message = bos_token + '' -%}\n{%- endif -%}\n{%- for message in loop_messages %}\n {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}\n {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n {% endif %}\n \n {%- if loop.index0 == 0 -%}\n {{ system_message -}}\n {%- endif -%}\n {%- if message['role'] == 'user' -%}\n {{ message['role'] + additional_special_tokens[0] + message['content'] + additional_special_tokens[1] -}}\n {{ 'available functions' + additional_special_tokens[0] + additional_special_tokens[2] + additional_special_tokens[3] + additional_special_tokens[1] -}}\n {%- endif -%}\n {%- if message['role'] == 'assistant' -%}\n {{ message['role'] + additional_special_tokens[0] + message['content'] + additional_special_tokens[1] -}}\n {%- endif -%}\n {%- if loop.last and add_generation_prompt -%}\n {{ 'assistant' + additional_special_tokens[0] -}}\n {%- endif -%}\n{%- endfor %}",
|
||||
/* .expected_output= */ "<s>You are a helpful assistant<|message_sep|>user<|role_sep|>Hello<|message_sep|>available functions<|role_sep|>[]<|message_sep|>assistant<|role_sep|>Hi there<|message_sep|>user<|role_sep|>Who are you<|message_sep|>available functions<|role_sep|>[]<|message_sep|>assistant<|role_sep|> I am an assistant <|message_sep|>user<|role_sep|>Another question<|message_sep|>available functions<|role_sep|>[]<|message_sep|>assistant<|role_sep|>",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
/* .supported_with_jinja= */ false, // Requires additional_special_tokens as extra context
|
||||
},
|
||||
{
|
||||
/* .name= */ "Infinigence/Megrez-3B-Instruct",
|
||||
/* .template_str= */ U8C("{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|role_start|>system<|role_end|>你是Megrez-3B-Instruct，将针对用户的问题给出详细的、积极的回答。<|turn_end|>' }}{% endif %}{{ '<|role_start|>' + message['role'] + '<|role_end|>' + message['content'] + '<|turn_end|>' }}{% endfor %}{% if add_generation_prompt %}{{ '<|role_start|>assistant<|role_end|>' }}{% endif %}"),
|
||||
/* .expected_output= */ "<|role_start|>system<|role_end|>You are a helpful assistant<|turn_end|><|role_start|>user<|role_end|>Hello<|turn_end|><|role_start|>assistant<|role_end|>Hi there<|turn_end|><|role_start|>user<|role_end|>Who are you<|turn_end|><|role_start|>assistant<|role_end|> I am an assistant <|turn_end|><|role_start|>user<|role_end|>Another question<|turn_end|><|role_start|>assistant<|role_end|>",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "phi-4",
|
||||
/* .template_str= */ "{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|im_start|>system<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'user') %}{{'<|im_start|>user<|im_sep|>' + message['content'] + '<|im_end|><|im_start|>assistant<|im_sep|>'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|im_end|>'}}{% endif %}{% endfor %}",
|
||||
/* .expected_output= */ "<|im_start|>system<|im_sep|>You are a helpful assistant<|im_end|><|im_start|>user<|im_sep|>Hello<|im_end|><|im_start|>assistant<|im_sep|>Hi there<|im_end|><|im_start|>user<|im_sep|>Who are you<|im_end|><|im_start|>assistant<|im_sep|> I am an assistant <|im_end|><|im_start|>user<|im_sep|>Another question<|im_end|><|im_start|>assistant<|im_sep|>",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "yandex/YandexGPT-5-Lite-8B-instruct",
|
||||
/* .template_str= */ "<s>{%- set names = {'assistant': ' Ассистент:', 'user': ' Пользователь:'} %}\n{%- set tools_prefix = 'Тебе доступны следующие функции:' %}\n{%- macro __render_tool(tool) %}\n {%- set name = tool.function.name %}\n {%- set description = tool.function.description|default('') %}\n {%- set parameters = tool.function.parameters|tojson %}\n {{- '\\n' }}function {{ '{' }}'name':'{{ name }}',\n {%- if tool.function.description %}'description':'{{ description }}',{% endif %}\n'parameters':{{ parameters }}\n {{- '}' }}\n{%- endmacro %}\n{%- macro __render_tools(tools) %}\n {{- tools_prefix }}\n {%- for tool in tools %}\n {{- __render_tool(tool) }}\n {%- endfor %}\n {{- '\\n\\n' }}\n{%- endmacro %}\n{%- macro __render_tool_message(message) %}\n {{- '\\n\\nРезультат вызова' }} {{ message.name }}: {{ message.content }} {{ '\\n\\n' }}\n{%- endmacro %}\n{%- if tools -%}\n {{- __render_tools(tools) }}\n{%- endif -%}\n{%- macro __render_user_message(message) %}\n{{ names.user }} {{ message.content + '\\n\\n' }}\n{%- endmacro %}\n{%- macro __render_assistant_message(message) %}\n {{- names.assistant }}\n {%- set call = message['function_call'] %}\n {%- if call %}\n {{- '\\n[TOOL_CALL_START]' }}{{ call.name }}{{ '\\n' }}{{ call.arguments|tojson }}\n {%- else %}\n {{- ' ' + message.content + '\\n\\n' }}\n {%- endif %}\n{%- endmacro %}\n{%- if not add_generation_prompt is defined %}\n{%- set add_generation_prompt = false %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'user' %}\n {{- __render_user_message(message) }}\n {%- endif %}\n {%- if message.role == 'assistant' and not loop.last %}\n {{- __render_assistant_message(message) }}\n {%- endif %}\n {%- if message.role == 'tool' %}\n {{- __render_tool_message(message) }}\n {%- endif %}\n {%- if loop.last %}\n {{- ' Ассистент:[SEP]' }}\n {%- endif %}\n{%- endfor %}\n",
|
||||
/* .expected_output= */ " Пользователь: Hello\n\n Ассистент: Hi there\n\n Пользователь: Who are you\n\n Ассистент: I am an assistant \n\n Пользователь: Another question\n\n Ассистент:[SEP]",
|
||||
/* .expected_output_jinja= */ "<s> Пользователь: You are a helpful assistant\nHello\n\n Ассистент: Hi there\n\n Пользователь: Who are you\n\n Ассистент: I am an assistant \n\n Пользователь: Another question\n\n Ассистент:[SEP]",
|
||||
/* .bos_token= */ "<s>",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "inclusionAI/Ling-lite",
|
||||
/* .template_str */ "{% for message in messages %}{% set role = message['role'] | lower %}{% if role == 'user' %}{% set role = 'HUMAN' %}{% endif %}{% set role = role | upper %}{{ '<role>' + role + '</role>' + message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ '<role>ASSISTANT</role>' }}{% endif %}",
|
||||
/* .expected_output= */ "<role>SYSTEM</role>You are a helpful assistant<role>HUMAN</role>Hello<role>ASSISTANT</role>Hi there<role>HUMAN</role>Who are you<role>ASSISTANT</role> I am an assistant <role>HUMAN</role>Another question<role>ASSISTANT</role>",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
},
|
||||
{
|
||||
/* .name= */ "ByteDance-Seed/Seed-OSS-36B-Instruct",
|
||||
/* .template_str */ "{# <seed:bos> #}{%- for message in messages %}{%- if message.role in [\"user\", \"system\"] %}{{ bos_token + message.role + \"\\n\" + message.content + eos_token }}{%- elif message.role == \"assistant\" %}{{ bos_token + message.role }}{%- if message.content is defined and message.content is string and message.content|trim|length > 0 %}{{ \"\\n\" + message.content|trim + eos_token }}{%- endif %}{%- else %}{{ bos_token + message.role + \"\\n\" + message.content + eos_token }}{%- endif %}{%- endfor %}{%- if add_generation_prompt %}{{ bos_token + \"assistant\\n\" }}{%- endif %}",
|
||||
/* .expected_output= */ "<seed:bos>system\nYou are a helpful assistant<seed:eos><seed:bos>user\nHello<seed:eos><seed:bos>assistant\nHi there<seed:eos><seed:bos>user\nWho are you<seed:eos><seed:bos>assistant\nI am an assistant<seed:eos><seed:bos>user\nAnother question<seed:eos><seed:bos>assistant\n",
|
||||
/* .expected_output_jinja= */ "<seed:bos>system\nYou are a helpful assistant<seed:eos><seed:bos>user\nHello<seed:eos><seed:bos>assistant\nHi there<seed:eos><seed:bos>user\nWho are you<seed:eos><seed:bos>assistant\nI am an assistant<seed:eos><seed:bos>user\nAnother question<seed:eos><seed:bos>assistant\n",
|
||||
/* .bos_token= */ "<seed:bos>",
|
||||
/* .eos_token= */ "<seed:eos>",
|
||||
},
|
||||
{
|
||||
/* .name= */ "ibm-granite/granite-3.x (tool call)",
|
||||
/* .template_str= */ "{%- for message in messages %}\n {%- if message['role'] == 'assistant_tool_call' %}\n {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'] + '<|end_of_text|>\\n' }}\n {%- else %}\n {{- '<|start_of_role|>' + message['role'] + '<|end_of_role|>' + message['content'] + '<|end_of_text|>\\n' }}\n {%- endif %}\n {%- if loop.last and add_generation_prompt %}\n {{- '<|start_of_role|>assistant<|end_of_role|>' }}\n {%- endif %}\n{%- endfor %}",
|
||||
/* .expected_output= */ "<|start_of_role|>system<|end_of_role|>You are a helpful assistant<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Hello<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>Hi there<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Who are you<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|> I am an assistant <|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Another question<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>What is the weather?<|end_of_text|>\n<|start_of_role|>assistant_tool_call<|end_of_role|><|tool_call|>[{\"name\": \"get_weather\", \"arguments\": {\"location\": \"NYC\"}}]<|end_of_text|>\n<|start_of_role|>tool_response<|end_of_role|>{\"temperature\": 72}<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>",
|
||||
/* .expected_output_jinja= */ "<|start_of_role|>system<|end_of_role|>You are a helpful assistant<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Hello<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>Hi there<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Who are you<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|> I am an assistant <|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Another question<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>What is the weather?<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|><|tool_call|>[{\"name\": \"get_weather\", \"arguments\": {\"location\": \"NYC\"}}]<|end_of_text|>\n<|start_of_role|>tool_response<|end_of_role|>{\"temperature\": 72}<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
/* .supported_with_jinja= */ true,
|
||||
/* .extra_conversation= */ {{"user", "What is the weather?"}, {"assistant_tool_call", "[{\"name\": \"get_weather\", \"arguments\": {\"location\": \"NYC\"}}]"}, {"tool_response", "{\"temperature\": 72}"}},
|
||||
},
|
||||
{
|
||||
/* .name= */ "ibm-granite/granite-4.0 (tool call)",
|
||||
/* .template_str= */ "{%- for message in messages %}\n {%- if message['role'] == 'assistant_tool_call' %}\n {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'] + '<|end_of_text|>\\n' }}\n {%- else %}\n {{- '<|start_of_role|>' + message['role'] + '<|end_of_role|>' + message['content'] + '<|end_of_text|>\\n' }}\n {%- endif %}\n {%- if loop.last and add_generation_prompt %}\n {{- '<|start_of_role|>assistant<|end_of_role|>' }}\n {%- endif %}\n{%- endfor %}\n{# <tool_call> <tools> g4_default_system_message #}",
|
||||
/* .expected_output= */ "<|start_of_role|>system<|end_of_role|>You are a helpful assistant<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Hello<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>Hi there<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Who are you<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|> I am an assistant <|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Another question<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>What is the weather?<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|><|tool_call|><tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"NYC\"}}\n</tool_call><|end_of_text|>\n<|start_of_role|>tool_response<|end_of_role|>{\"temperature\": 72}<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
/* .supported_with_jinja= */ true,
|
||||
/* .extra_conversation= */ {{"user", "What is the weather?"}, {"assistant_tool_call", "<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"NYC\"}}\n</tool_call>"}, {"tool_response", "{\"temperature\": 72}"}},
|
||||
},
|
||||
{
|
||||
/* .name= */ "ibm-granite/granite-4.1 (tool call)",
|
||||
/* .template_str= */ "{%- for message in messages %}\n {%- if message['role'] == 'assistant_tool_call' %}\n {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'] + '<|end_of_text|>\\n' }}\n {%- else %}\n {{- '<|start_of_role|>' + message['role'] + '<|end_of_role|>' + message['content'] + '<|end_of_text|>\\n' }}\n {%- endif %}\n {%- if loop.last and add_generation_prompt %}\n {{- '<|start_of_role|>assistant<|end_of_role|>' }}\n {%- endif %}\n{%- endfor %}\n{# <tool_call> <tools> #}",
|
||||
/* .expected_output= */ "<|start_of_role|>system<|end_of_role|>You are a helpful assistant<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Hello<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>Hi there<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Who are you<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|> I am an assistant <|end_of_text|>\n<|start_of_role|>user<|end_of_role|>Another question<|end_of_text|>\n<|start_of_role|>user<|end_of_role|>What is the weather?<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|><|tool_call|><tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"NYC\"}}\n</tool_call><|end_of_text|>\n<|start_of_role|>tool_response<|end_of_role|>{\"temperature\": 72}<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>",
|
||||
/* .expected_output_jinja= */ "",
|
||||
/* .bos_token= */ "",
|
||||
/* .eos_token= */ "",
|
||||
/* .supported_with_jinja= */ true,
|
||||
/* .extra_conversation= */ {{"user", "What is the weather?"}, {"assistant_tool_call", "<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"NYC\"}}\n</tool_call>"}, {"tool_response", "{\"temperature\": 72}"}},
|
||||
}
|
||||
};
|
||||
std::vector<char> formatted_chat(1024);
|
||||
int32_t res;
|
||||
|
||||
// list all supported templates
|
||||
std::vector<const char *> supported_tmpl;
|
||||
res = llama_chat_builtin_templates(nullptr, 0);
|
||||
assert(res > 0);
|
||||
supported_tmpl.resize(res);
|
||||
res = llama_chat_builtin_templates(supported_tmpl.data(), supported_tmpl.size());
|
||||
std::cout << "Built-in chat templates:\n";
|
||||
for (const auto *tmpl : supported_tmpl) {
|
||||
std::cout << " " << tmpl << "\n";
|
||||
}
|
||||
|
||||
// test invalid chat template
|
||||
res = llama_chat_apply_template("INVALID TEMPLATE", conversation.data(), conversation.size(), true, formatted_chat.data(), formatted_chat.size());
|
||||
assert(res < 0);
|
||||
const auto add_generation_prompt = true;
|
||||
|
||||
for (const auto & test_case : test_cases) {
|
||||
std::cout << "\n\n=== " << test_case.name << " ===\n\n";
|
||||
auto conv = conversation;
|
||||
conv.insert(conv.end(), test_case.extra_conversation.begin(), test_case.extra_conversation.end());
|
||||
formatted_chat.resize(2048);
|
||||
res = llama_chat_apply_template(
|
||||
test_case.template_str.c_str(),
|
||||
conv.data(),
|
||||
conv.size(),
|
||||
add_generation_prompt,
|
||||
formatted_chat.data(),
|
||||
formatted_chat.size()
|
||||
);
|
||||
formatted_chat.resize(res);
|
||||
std::string output(formatted_chat.data(), formatted_chat.size());
|
||||
if (output != test_case.expected_output) {
|
||||
std::cout << "Expected:\n" << test_case.expected_output << "\n";
|
||||
std::cout << "-------------------------\n";
|
||||
std::cout << "Actual:\n" << output << "\n";
|
||||
std::cout.flush();
|
||||
assert(output == test_case.expected_output);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<common_chat_msg> messages;
|
||||
messages.reserve(conversation.size());
|
||||
for (const auto & msg : conversation) {
|
||||
messages.push_back(simple_msg(msg.role, msg.content));
|
||||
}
|
||||
for (const auto & test_case : test_cases) {
|
||||
if (!test_case.supported_with_jinja) {
|
||||
continue;
|
||||
}
|
||||
std::cout << "\n\n=== " << test_case.name << " (jinja) ===\n\n";
|
||||
try {
|
||||
auto msgs = messages;
|
||||
for (const auto & msg : test_case.extra_conversation) {
|
||||
msgs.push_back(simple_msg(msg.role, msg.content));
|
||||
}
|
||||
auto output = format_using_common(
|
||||
test_case.template_str,
|
||||
test_case.bos_token,
|
||||
test_case.eos_token,
|
||||
msgs);
|
||||
auto expected_output = normalize_newlines(test_case.expected_output_jinja.empty() ? test_case.expected_output : test_case.expected_output_jinja);
|
||||
if (output != expected_output) {
|
||||
std::cout << "Template:```\n" << test_case.template_str << "\n```";
|
||||
std::cout << "-------------------------\n";
|
||||
std::cout << "Expected:```\n" << expected_output << "\n```";
|
||||
std::cout << "-------------------------\n";
|
||||
std::cout << "Actual:```\n" << output << "\n```";
|
||||
std::cout.flush();
|
||||
assert(output == expected_output);
|
||||
}
|
||||
} catch (const std::exception & e) {
|
||||
std::cerr << "ERROR: " << e.what() << "\n";
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "\nOK: All tests passed successfully.\n";
|
||||
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,159 @@
|
||||
// test-col2im-1d.cpp: validate GGML_OP_COL2IM_1D against ggml_conv_transpose_1d.
|
||||
//
|
||||
// A ConvTranspose1d factorizes as a GEMM followed by an overlap-add:
|
||||
// conv_transpose_1d(w, x) equals col2im_1d(mul_mat(w_perm, x_t), s0, OC, p0)
|
||||
// with w_perm the [IC, K*OC] permutation of the [K, OC, IC] kernel and x_t the
|
||||
// [IC, T_in] transpose of the [T_in, IC] input. The test derives both alternative
|
||||
// layouts from one logical weight and one logical input with graph ops only
|
||||
// (permute + cont + reshape), runs the two paths on the CPU backend, and compares
|
||||
// them in F32. The F16 and BF16 kernels are exercised by casting the column
|
||||
// matrix before the scatter. Cropping (p0 > 0) is checked against the shifted
|
||||
// slice of the uncropped reference, which conv_transpose_1d cannot express.
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
// One geometry: kernel size, output channels, input length, stride, crop
|
||||
struct col2im_case {
|
||||
int64_t K;
|
||||
int64_t OC;
|
||||
int64_t T_in;
|
||||
int s0;
|
||||
int p0;
|
||||
};
|
||||
|
||||
// Mirrors the eval grid of test-backend-ops
|
||||
static const col2im_case CASES[] = {
|
||||
{ 16, 32, 197, 8, 0 }, // kernel = 2*stride, DAC upsampling shape
|
||||
{ 4, 3, 7, 2, 0 },
|
||||
{ 1, 5, 13, 1, 0 }, // stride 1, no overlap
|
||||
{ 6, 4, 11, 3, 1 }, // with cropping
|
||||
{ 2, 3, 9, 3, 0 }, // kernel < stride, gap positions are zeroed
|
||||
{ 5, 4, 11, 2, 0 }, // kernel not a multiple of stride, alternating overlap
|
||||
{ 8, 4, 13, 4, 2 }, // padding = stride/2, DAC causal cropping
|
||||
{ 4, 3, 1, 2, 0 }, // single column, pure kernel unfold
|
||||
{ 16, 1, 197, 8, 0 }, // OC = 1, mono output stage
|
||||
{ 1, 5, 13, 3, 0 }, // K = 1 with stride > 1, sparse scatter
|
||||
{ 8, 2, 3, 2, 5 }, // cropping eats most of the signal, T_out = 2
|
||||
};
|
||||
|
||||
// Input channels of the GEMM, shared by every case
|
||||
static const int64_t IC = 7;
|
||||
|
||||
// Deterministic LCG mapped to [-1, 1]
|
||||
static uint64_t g_rng = 0x12345678ULL;
|
||||
static float frand(void) {
|
||||
g_rng = g_rng * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||
return (float)((g_rng >> 33) & 0xffffff) / (float)0x800000 - 1.0f;
|
||||
}
|
||||
|
||||
// Read a F32/F16/BF16 tensor back as a flat F32 vector
|
||||
static std::vector<float> tensor_to_f32(const struct ggml_tensor * t) {
|
||||
const int64_t n = ggml_nelements(t);
|
||||
std::vector<float> out(n);
|
||||
if (t->type == GGML_TYPE_F32) {
|
||||
memcpy(out.data(), t->data, n * sizeof(float));
|
||||
} else if (t->type == GGML_TYPE_F16) {
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
out[i] = ggml_fp16_to_fp32(((const ggml_fp16_t *) t->data)[i]);
|
||||
}
|
||||
} else {
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
out[i] = ggml_bf16_to_fp32(((const ggml_bf16_t *) t->data)[i]);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// NMSE of the cropped output against the p0 shifted slice of the full reference
|
||||
static double nmse_cropped(const float * y, const float * ref, int64_t T_out, int64_t T_ref, int64_t OC, int p0) {
|
||||
double num = 0.0;
|
||||
double den = 0.0;
|
||||
for (int64_t oc = 0; oc < OC; oc++) {
|
||||
for (int64_t t = 0; t < T_out; t++) {
|
||||
const double a = y [t + oc * T_out];
|
||||
const double b = ref[t + p0 + oc * T_ref];
|
||||
num += (a - b) * (a - b);
|
||||
den += b * b;
|
||||
}
|
||||
}
|
||||
return num / (den + 1e-30);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
int fails = 0;
|
||||
|
||||
for (const col2im_case & c : CASES) {
|
||||
const int64_t T_ref = (c.T_in - 1) * c.s0 + c.K;
|
||||
const int64_t T_out = T_ref - 2 * c.p0;
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/* .mem_size = */ (size_t) 64 << 20,
|
||||
/* .mem_base = */ NULL,
|
||||
/* .no_alloc = */ false,
|
||||
};
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
// One logical weight and one logical input feed both paths
|
||||
struct ggml_tensor * w = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, c.K, c.OC, IC);
|
||||
struct ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, c.T_in, IC);
|
||||
for (int64_t i = 0; i < ggml_nelements(w); i++) {
|
||||
((float *) w->data)[i] = frand();
|
||||
}
|
||||
for (int64_t i = 0; i < ggml_nelements(x); i++) {
|
||||
((float *) x->data)[i] = frand();
|
||||
}
|
||||
|
||||
// Reference path: the native op, uncropped
|
||||
struct ggml_tensor * y_ref = ggml_conv_transpose_1d(ctx, w, x, c.s0, 0, 1);
|
||||
|
||||
// Decomposed path: [K, OC, IC] -> [IC, K, OC] -> [IC, K*OC], k fastest inside each oc block
|
||||
struct ggml_tensor * w_perm = ggml_cont(ctx, ggml_permute(ctx, w, 1, 2, 0, 3));
|
||||
w_perm = ggml_reshape_2d(ctx, w_perm, IC, c.K * c.OC);
|
||||
struct ggml_tensor * x_t = ggml_cont(ctx, ggml_transpose(ctx, x));
|
||||
struct ggml_tensor * col = ggml_mul_mat(ctx, w_perm, x_t);
|
||||
struct ggml_tensor * y32 = ggml_col2im_1d(ctx, col, c.s0, (int) c.OC, c.p0);
|
||||
|
||||
// Half precision kernels: the same columns cast before the scatter
|
||||
struct ggml_tensor * y16 = ggml_col2im_1d(ctx, ggml_cast(ctx, col, GGML_TYPE_F16), c.s0, (int) c.OC, c.p0);
|
||||
struct ggml_tensor * ybf = ggml_col2im_1d(ctx, ggml_cast(ctx, col, GGML_TYPE_BF16), c.s0, (int) c.OC, c.p0);
|
||||
|
||||
GGML_ASSERT(y_ref->ne[0] == T_ref && y_ref->ne[1] == c.OC);
|
||||
GGML_ASSERT(y32->ne[0] == T_out && y32->ne[1] == c.OC);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(gf, y_ref);
|
||||
ggml_build_forward_expand(gf, y32);
|
||||
ggml_build_forward_expand(gf, y16);
|
||||
ggml_build_forward_expand(gf, ybf);
|
||||
ggml_graph_compute_with_ctx(ctx, gf, 4);
|
||||
|
||||
const std::vector<float> f32 = tensor_to_f32(y32);
|
||||
const std::vector<float> f16 = tensor_to_f32(y16);
|
||||
const std::vector<float> fbf = tensor_to_f32(ybf);
|
||||
const float * ref = (const float *) y_ref->data;
|
||||
|
||||
const double e32 = nmse_cropped(f32.data(), ref, T_out, T_ref, c.OC, c.p0);
|
||||
const double e16 = nmse_cropped(f16.data(), ref, T_out, T_ref, c.OC, c.p0);
|
||||
const double ebf = nmse_cropped(fbf.data(), ref, T_out, T_ref, c.OC, c.p0);
|
||||
|
||||
// Same thresholds as test-backend-ops: 1e-7 full precision, 5e-4 half
|
||||
const bool ok = e32 <= 1e-7 && e16 <= 5e-4 && ebf <= 5e-4;
|
||||
if (!ok) {
|
||||
fails++;
|
||||
}
|
||||
printf("col2im_1d K=%2d OC=%2d T_in=%3d s0=%d p0=%d: nmse f32=%.2e f16=%.2e bf16=%.2e %s\n",
|
||||
(int) c.K, (int) c.OC, (int) c.T_in, c.s0, c.p0, e32, e16, ebf, ok ? "OK" : "FAIL");
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
printf(fails == 0 ? "all col2im_1d checks passed\n" : "%d col2im_1d checks FAILED\n", fails);
|
||||
return fails == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// test-conv-1d.cpp: validate ggml_conv_1d against its own single batch result.
|
||||
//
|
||||
// conv_1d folds the batch into the rows of the GEMM, so the product is laid out
|
||||
// as [OL, N, OC] and reaches the documented [OL, OC, N] only after a transpose
|
||||
// of the two slowest dimensions. The test runs one batched convolution and N
|
||||
// single batch convolutions sharing the same weight and the same input slices,
|
||||
// on the CPU backend, and compares every channel of every batch in F32. A wrong
|
||||
// layout keeps the shape intact and scrambles the values, so the comparison is
|
||||
// on the contents.
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <vector>
|
||||
|
||||
// One geometry: kernel size, input and output channels, input length, batch,
|
||||
// stride, padding, dilation
|
||||
struct conv_1d_case {
|
||||
int64_t K;
|
||||
int64_t IC;
|
||||
int64_t OC;
|
||||
int64_t L;
|
||||
int64_t N;
|
||||
int s0;
|
||||
int p0;
|
||||
int d0;
|
||||
};
|
||||
|
||||
static const conv_1d_case CASES[] = {
|
||||
{ 3, 4, 5, 17, 1, 1, 1, 1 }, // single batch, the shape every caller already exercises
|
||||
{ 3, 4, 5, 17, 2, 1, 1, 1 }, // smallest batched case
|
||||
{ 7, 8, 16, 64, 3, 2, 3, 1 }, // strided with padding, vocoder front end shape
|
||||
{ 1, 3, 3, 11, 4, 1, 0, 1 }, // K = 1, pointwise mix
|
||||
{ 5, 2, 7, 31, 5, 3, 2, 2 }, // dilation > 1, kernel not a multiple of stride
|
||||
{ 4, 6, 1, 23, 2, 1, 0, 1 }, // OC = 1, mono output stage
|
||||
{ 2, 1, 4, 9, 8, 2, 1, 1 }, // IC = 1, batch larger than every other dimension
|
||||
};
|
||||
|
||||
// Deterministic LCG mapped to [-1, 1]
|
||||
static uint64_t g_rng = 0x12345678ULL;
|
||||
static float frand(void) {
|
||||
g_rng = g_rng * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||
return (float)((g_rng >> 33) & 0xffffff) / (float)0x800000 - 1.0f;
|
||||
}
|
||||
|
||||
// NMSE of one batch of the packed output against its single batch reference
|
||||
static double nmse(const float * y, const float * ref, int64_t n) {
|
||||
double num = 0.0;
|
||||
double den = 0.0;
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
const double a = y[i];
|
||||
const double b = ref[i];
|
||||
num += (a - b) * (a - b);
|
||||
den += b * b;
|
||||
}
|
||||
return num / (den + 1e-30);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
int fails = 0;
|
||||
|
||||
for (const conv_1d_case & c : CASES) {
|
||||
const int64_t OL = (c.L + 2 * c.p0 - c.d0 * (c.K - 1) - 1) / c.s0 + 1;
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/* .mem_size = */ (size_t) 64 << 20,
|
||||
/* .mem_base = */ NULL,
|
||||
/* .no_alloc = */ false,
|
||||
};
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
// One logical weight and one logical input feed both paths
|
||||
struct ggml_tensor * w = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, c.K, c.IC, c.OC);
|
||||
struct ggml_tensor * x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, c.L, c.IC, c.N);
|
||||
for (int64_t i = 0; i < ggml_nelements(w); i++) {
|
||||
((float *) w->data)[i] = frand();
|
||||
}
|
||||
for (int64_t i = 0; i < ggml_nelements(x); i++) {
|
||||
((float *) x->data)[i] = frand();
|
||||
}
|
||||
|
||||
struct ggml_tensor * y = ggml_conv_1d(ctx, w, x, c.s0, c.p0, c.d0);
|
||||
|
||||
GGML_ASSERT(y->ne[0] == OL && y->ne[1] == c.OC && y->ne[2] == c.N);
|
||||
GGML_ASSERT(ggml_is_contiguous(y));
|
||||
|
||||
// Reference path: the same convolution on one batch at a time
|
||||
std::vector<struct ggml_tensor *> ref(c.N);
|
||||
for (int64_t n = 0; n < c.N; n++) {
|
||||
struct ggml_tensor * slice = ggml_cont(ctx,
|
||||
ggml_view_2d(ctx, x, c.L, c.IC, x->nb[1], n * x->nb[2]));
|
||||
ref[n] = ggml_conv_1d(ctx, w, slice, c.s0, c.p0, c.d0);
|
||||
}
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(gf, y);
|
||||
for (int64_t n = 0; n < c.N; n++) {
|
||||
ggml_build_forward_expand(gf, ref[n]);
|
||||
}
|
||||
ggml_graph_compute_with_ctx(ctx, gf, 4);
|
||||
|
||||
// Worst batch decides the case
|
||||
double worst = 0.0;
|
||||
for (int64_t n = 0; n < c.N; n++) {
|
||||
const float * yn = (const float *) y->data + n * OL * c.OC;
|
||||
const double e = nmse(yn, (const float *) ref[n]->data, OL * c.OC);
|
||||
if (e > worst) {
|
||||
worst = e;
|
||||
}
|
||||
}
|
||||
|
||||
// Both paths share the F16 im2col and the same dot products, so only
|
||||
// the row count of the GEMM differs
|
||||
const bool ok = worst <= 1e-7;
|
||||
if (!ok) {
|
||||
fails++;
|
||||
}
|
||||
printf("conv_1d K=%d IC=%2d OC=%2d L=%3d N=%d s0=%d p0=%d d0=%d: nmse=%.2e %s\n",
|
||||
(int) c.K, (int) c.IC, (int) c.OC, (int) c.L, (int) c.N, c.s0, c.p0, c.d0,
|
||||
worst, ok ? "OK" : "FAIL");
|
||||
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
printf(fails == 0 ? "all conv_1d checks passed\n" : "%d conv_1d checks FAILED\n", fails);
|
||||
return fails == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// These tests may take a long time!
|
||||
// They are to prove that conversion from double to float of various functions in ggml.c doesn't affect the result.
|
||||
// This is done by checking all finite (non-NaN, non-infinite) floats.
|
||||
|
||||
#undef NDEBUG
|
||||
#include <cassert>
|
||||
#if !defined(__riscv) && !defined(__s390__) && !defined(__ARM_NEON)
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wdouble-promotion"
|
||||
|
||||
// ggml.c::quantize_row_q4_0_ref
|
||||
inline static uint8_t round_orig(float v0) { return ((int8_t) (round(v0))) + 8; }
|
||||
|
||||
// ggml.c::ggml_silu_f32
|
||||
inline static float silu_orig(float x) {
|
||||
return x/(1.0 + exp(-x));
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
// ggml.c::quantize_row_q4_0_ref
|
||||
inline static uint8_t round_float(float v0) { return (int8_t)roundf(v0) + 8; }
|
||||
|
||||
// ggml.c::ggml_silu_f32
|
||||
inline static float silu_float(float x) {
|
||||
return x/(1.0f + expf(-x));
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
uint32_t x = UINT32_MAX;
|
||||
do {
|
||||
float f;
|
||||
memcpy(&f, &x, sizeof(x));
|
||||
assert(!std::isfinite(f) || (round_orig(f) == round_float(f)));
|
||||
} while (x--);
|
||||
|
||||
#ifdef __F16C__
|
||||
// GELU and SILU implementations are used with a FP16 lookup table.
|
||||
// The original and float-only results are not equal for all inputs after converting to FP16.
|
||||
// GELU is an approximation anyway (tanh), not tested here.
|
||||
// For SILU, verify that the results are at least the closest floating point numbers, if the FP16 values don't match.
|
||||
for (x = 0; x <= UINT16_MAX; x++) {
|
||||
float f = _cvtsh_ss(x);
|
||||
const float so = silu_orig(f);
|
||||
const float sf = silu_float(f);
|
||||
assert( (_cvtss_sh(so, 0) == _cvtss_sh(sf, 0))
|
||||
|| (nextafterf(so, sf) == sf)
|
||||
|| (nextafterf(sf, so) == so));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
#include "arg.h"
|
||||
#include "common.h"
|
||||
#include "log.h"
|
||||
#include "llama-cpp.h"
|
||||
#include "../src/llama-ext.h"
|
||||
#include "ggml.h"
|
||||
#include "gguf-model-data.h"
|
||||
#include "gguf.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "download.h"
|
||||
|
||||
#include <array>
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <random>
|
||||
|
||||
// Noop because weights are not needed
|
||||
static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
|
||||
GGML_UNUSED(tensor);
|
||||
GGML_UNUSED(userdata);
|
||||
}
|
||||
|
||||
struct input_tensor {
|
||||
ggml_type type;
|
||||
std::array<int64_t, 4> ne;
|
||||
std::array<size_t, 4> nb;
|
||||
|
||||
input_tensor(ggml_type type, int64_t * ne, size_t * nb): type(type) {
|
||||
memcpy(this->ne.data(), ne, 4 * sizeof(int64_t));
|
||||
memcpy(this->nb.data(), nb, 4 * sizeof(size_t));
|
||||
}
|
||||
|
||||
bool operator<(const input_tensor &b) const {
|
||||
return std::tie(type, ne, nb) <
|
||||
std::tie(b.type, b.ne, b.nb);
|
||||
}
|
||||
|
||||
void serialize(std::ostream& out) const {
|
||||
out << type << ' ';
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
out << ne[i] << ' ';
|
||||
}
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
out << nb[i] << ' ';
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
struct test_object {
|
||||
ggml_op op;
|
||||
ggml_type type;
|
||||
std::array<int64_t, 4> ne;
|
||||
std::vector<int32_t> op_params;
|
||||
std::vector<input_tensor> sources;
|
||||
std::string name;
|
||||
|
||||
void serialize(std::ostream& out) const {
|
||||
out << op << ' ' << type << ' ';
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
out << ne[i] << ' ';
|
||||
}
|
||||
|
||||
out << op_params.size() << ' ';
|
||||
for (size_t i = 0; i < op_params.size(); i++) {
|
||||
out << op_params[i] << ' ';
|
||||
}
|
||||
|
||||
out << sources.size() << ' ';
|
||||
for (size_t s = 0; s < sources.size(); s++) {
|
||||
sources[s].serialize(out);
|
||||
}
|
||||
|
||||
if (!name.empty()) {
|
||||
out << name;
|
||||
} else {
|
||||
out << '-';
|
||||
}
|
||||
|
||||
out << '\n';
|
||||
}
|
||||
|
||||
bool operator<(const test_object &b) const {
|
||||
return std::tie(op, type, ne, op_params, sources) <
|
||||
std::tie(b.op, b.type, b.ne, b.op_params, b.sources);
|
||||
}
|
||||
};
|
||||
|
||||
static void extract_graph_ops(ggml_cgraph * cgraph, const char * label, std::set<test_object> & tests) {
|
||||
int n_nodes = ggml_graph_n_nodes(cgraph);
|
||||
int n_skipped = 0;
|
||||
int n_before = (int) tests.size();
|
||||
for (int i = 0; i < n_nodes; i++) {
|
||||
ggml_tensor * node = ggml_graph_node(cgraph, i);
|
||||
|
||||
if (node->op == GGML_OP_NONE || node->op == GGML_OP_VIEW || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) {
|
||||
n_skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
test_object test;
|
||||
|
||||
test.op = node->op;
|
||||
test.type = node->type;
|
||||
memcpy(&test.ne, node->ne, 4 * sizeof(int64_t));
|
||||
|
||||
test.op_params.resize(GGML_MAX_OP_PARAMS / sizeof(int32_t));
|
||||
memcpy(test.op_params.data(), node->op_params, GGML_MAX_OP_PARAMS);
|
||||
|
||||
for (size_t s = 0; s < GGML_MAX_SRC; s++) {
|
||||
if (node->src[s] == nullptr) {
|
||||
break;
|
||||
}
|
||||
|
||||
test.sources.emplace_back(node->src[s]->type, node->src[s]->ne, node->src[s]->nb);
|
||||
}
|
||||
|
||||
test.name = node->name;
|
||||
tests.insert(test);
|
||||
}
|
||||
|
||||
int n_new = (int) tests.size() - n_before;
|
||||
LOG_INF("%s: %d unique ops, %d total nodes, %d skipped (view ops)\n",
|
||||
label, n_new, n_nodes, n_skipped);
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
common_params params;
|
||||
params.out_file = "tests.txt";
|
||||
|
||||
common_init();
|
||||
|
||||
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Load CPU-only
|
||||
ggml_backend_dev_t cpu_device = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
|
||||
params.devices = { cpu_device, nullptr };
|
||||
params.fit_params = false;
|
||||
params.n_gpu_layers = 0;
|
||||
|
||||
params.warmup = false;
|
||||
|
||||
llama_context * ctx;
|
||||
common_init_result_ptr init_result;
|
||||
llama_context_ptr ctx2;
|
||||
llama_model_ptr model;
|
||||
|
||||
if (params.model.hf_repo.empty()) {
|
||||
init_result = common_init_from_params(params);
|
||||
|
||||
ctx = init_result->context();
|
||||
if (!ctx) {
|
||||
LOG_ERR("failed to initialize params\n");
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
#ifdef LLAMA_HF_FETCH
|
||||
auto [hf_repo, hf_quant] = common_download_split_repo_tag(params.model.hf_repo);
|
||||
if (hf_quant.empty() || hf_quant == "latest") {
|
||||
hf_quant = "Q4_K_M";
|
||||
}
|
||||
|
||||
gguf_context_ptr gguf_ctx = gguf_fetch_gguf_ctx(hf_repo, hf_quant);
|
||||
if (!gguf_ctx) {
|
||||
LOG_ERR("failed to fetch GGUF metadata from %s\n", hf_repo.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
llama_model_params model_params = llama_model_default_params();
|
||||
model_params.devices = params.devices.data();
|
||||
model_params.no_alloc = true;
|
||||
|
||||
model.reset(llama_model_init_from_user(gguf_ctx.get(), set_tensor_data, nullptr, model_params));
|
||||
|
||||
if (!model) {
|
||||
LOG_ERR("failed to create llama_model from %s\n", hf_repo.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
llama_context_params ctx_params = llama_context_default_params();
|
||||
ctx2.reset(llama_init_from_model(model.get(), ctx_params));
|
||||
ctx = ctx2.get();
|
||||
|
||||
if (!ctx) {
|
||||
LOG_ERR("failed to create llama_context\n");
|
||||
return 1;
|
||||
}
|
||||
#else
|
||||
LOG_ERR("test-export-graph-ops compiled without HF fetch support\n");
|
||||
return 1;
|
||||
#endif
|
||||
}
|
||||
|
||||
const uint32_t n_seqs = llama_n_seq_max(ctx);
|
||||
const uint32_t n_tokens = std::min(llama_n_ctx(ctx), llama_n_ubatch(ctx));
|
||||
|
||||
std::set<test_object> tests;
|
||||
|
||||
auto * gf_pp = llama_graph_reserve(ctx, n_tokens, n_seqs, n_tokens);
|
||||
if (!gf_pp) {
|
||||
LOG_ERR("failed to reserve prompt processing graph\n");
|
||||
return 1;
|
||||
}
|
||||
extract_graph_ops(gf_pp, "pp", tests);
|
||||
|
||||
auto * gf_tg = llama_graph_reserve(ctx, n_seqs, n_seqs, n_seqs);
|
||||
if (!gf_tg) {
|
||||
LOG_ERR("failed to reserve token generation graph\n");
|
||||
return 1;
|
||||
}
|
||||
extract_graph_ops(gf_tg, "tg", tests);
|
||||
|
||||
LOG_INF("%d unique ops total\n", (int) tests.size());
|
||||
|
||||
std::ofstream f(params.out_file);
|
||||
|
||||
if (!f.is_open()) {
|
||||
LOG_ERR("unable to open output file: %s\n", params.out_file.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
for (const auto& test : tests) {
|
||||
test.serialize(f);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,589 @@
|
||||
// test-fusion: verify the backend fusion logic against a per-device baseline.
|
||||
//
|
||||
// for every dummy model generated by test-llama-archs, the tool runs the model on a single
|
||||
// device with fusion enabled and disabled, and reports:
|
||||
// - the per-fusion-type counters for each mode (prefill / decode, merged into "any" when the
|
||||
// per-graph counts match)
|
||||
// - the NMSE between the fused and unfused logits
|
||||
// - the NMSE between the device and a CPU reference
|
||||
//
|
||||
// the per-fusion-type counters are compared against a per-device baseline file (CSV) so a
|
||||
// fusion pattern that silently stops matching (or fires when it should not) is caught as a
|
||||
// regression.
|
||||
//
|
||||
// usage:
|
||||
// test-fusion --models DIR --device MTL0 --record baseline.csv # generate a baseline
|
||||
// test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it
|
||||
// test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model
|
||||
|
||||
#include "common.h"
|
||||
#include "log.h"
|
||||
#include "llama-cpp.h"
|
||||
|
||||
#include "ggml.h"
|
||||
#include "gguf.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <random>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// generic fusion debugging API, resolved through the ad-hoc get_proc_address mechanism
|
||||
// (not part of the official ggml backend interface yet). a backend that adopts fusion debugging
|
||||
// exports these exact names.
|
||||
typedef void * ggml_backend_fusion_t;
|
||||
|
||||
typedef ggml_backend_fusion_t ( * fusion_get_t) (ggml_backend_dev_t);
|
||||
typedef void ( * fusion_stats_init_t) (ggml_backend_fusion_t);
|
||||
typedef void ( * fusion_stats_reset_t) (ggml_backend_fusion_t);
|
||||
typedef int ( * fusion_stats_get_t) (ggml_backend_fusion_t, const char **, uint64_t *, int);
|
||||
typedef void ( * fusion_set_enabled_t) (ggml_backend_fusion_t, bool);
|
||||
|
||||
static bool silent_model_load_progress(float, void *) {
|
||||
return true;
|
||||
}
|
||||
|
||||
struct gguf_context_ptr {
|
||||
gguf_context * ctx;
|
||||
gguf_context_ptr(gguf_context * c) : ctx(c) {}
|
||||
~gguf_context_ptr() { if (ctx) { gguf_free(ctx); } }
|
||||
gguf_context * get() const { return ctx; }
|
||||
gguf_context_ptr(const gguf_context_ptr &) = delete;
|
||||
gguf_context_ptr & operator=(const gguf_context_ptr &) = delete;
|
||||
};
|
||||
|
||||
// NMSE between two vectors (same as tests/test-llama-archs.cpp)
|
||||
static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
|
||||
GGML_ASSERT(a.size() == b.size());
|
||||
double mse_a_b = 0.0;
|
||||
double mse_a_0 = 0.0;
|
||||
|
||||
for (size_t i = 0; i < a.size(); i++) {
|
||||
const float a_i = a[i];
|
||||
const float b_i = b[i];
|
||||
|
||||
mse_a_b += (a_i - b_i) * (a_i - b_i);
|
||||
mse_a_0 += a_i * a_i;
|
||||
}
|
||||
|
||||
return mse_a_b / mse_a_0;
|
||||
}
|
||||
|
||||
// deterministic token sequence
|
||||
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) {
|
||||
std::mt19937 gen(seed);
|
||||
std::uniform_int_distribution<> dis(0, n_vocab - 1);
|
||||
std::vector<llama_token> ret;
|
||||
ret.reserve(n_tokens);
|
||||
for (uint32_t i = 0; i < n_tokens; i++) {
|
||||
ret.push_back(dis(gen));
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
// trim leading/trailing whitespace (used when parsing padded CSV columns)
|
||||
static std::string trim(const std::string & s) {
|
||||
const size_t b = s.find_first_not_of(" \t\r\n");
|
||||
if (b == std::string::npos) {
|
||||
return "";
|
||||
}
|
||||
const size_t e = s.find_last_not_of(" \t\r\n");
|
||||
return s.substr(b, e - b + 1);
|
||||
}
|
||||
|
||||
static std::string get_arch(const std::string & path) {
|
||||
gguf_init_params params = { /*no_alloc=*/true, /*ctx=*/nullptr };
|
||||
gguf_context_ptr ctx(gguf_init_from_file(path.c_str(), params));
|
||||
if (!ctx.get()) {
|
||||
throw std::runtime_error("failed to read gguf: " + path);
|
||||
}
|
||||
const int idx = gguf_find_key(ctx.get(), "general.architecture");
|
||||
if (idx < 0) {
|
||||
return "unknown";
|
||||
}
|
||||
const char * val = gguf_get_val_str(ctx.get(), idx);
|
||||
return val ? val : "unknown";
|
||||
}
|
||||
|
||||
static llama_model_ptr load_model(const std::string & path, ggml_backend_dev_t dev) {
|
||||
llama_model_params model_params = llama_model_default_params();
|
||||
model_params.progress_callback = silent_model_load_progress;
|
||||
std::vector<ggml_backend_dev_t> devs = { dev, nullptr };
|
||||
model_params.devices = devs.data();
|
||||
model_params.split_mode = LLAMA_SPLIT_MODE_LAYER;
|
||||
|
||||
llama_model_ptr model(llama_model_load_from_file(path.c_str(), model_params));
|
||||
if (!model) {
|
||||
throw std::runtime_error("failed to load model: " + path);
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
// a fresh context (fresh state) from an already-loaded model
|
||||
static llama_context_ptr create_ctx(llama_model * model, int n_ubatch) {
|
||||
llama_context_params ctx_params = llama_context_default_params();
|
||||
ctx_params.n_ctx = 0;
|
||||
ctx_params.n_threads = 4;
|
||||
ctx_params.n_threads_batch = 4;
|
||||
ctx_params.n_ubatch = n_ubatch;
|
||||
ctx_params.n_batch = n_ubatch;
|
||||
|
||||
llama_context_ptr lctx(llama_init_from_model(model, ctx_params));
|
||||
if (!lctx) {
|
||||
throw std::runtime_error("failed to init context");
|
||||
}
|
||||
return lctx;
|
||||
}
|
||||
|
||||
// decode all tokens in one batch; returns the logits of every token
|
||||
static std::vector<float> decode_prefill(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
|
||||
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
|
||||
common_batch batch(lctx);
|
||||
for (size_t i = 0; i < tokens.size(); i++) {
|
||||
batch.add(tokens[i], i, 0, true);
|
||||
}
|
||||
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
throw std::runtime_error("prefill decode failed");
|
||||
}
|
||||
|
||||
std::vector<float> ret;
|
||||
ret.reserve(tokens.size() * n_vocab);
|
||||
for (size_t i = 0; i < tokens.size(); i++) {
|
||||
const float * logits_ith = llama_get_logits_ith(lctx, i);
|
||||
for (uint32_t j = 0; j < n_vocab; j++) {
|
||||
ret.push_back(logits_ith[j]);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
// decode one token at a time; returns the logits of the last token of each step
|
||||
static std::vector<float> decode_gen(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
|
||||
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
|
||||
common_batch batch(lctx);
|
||||
std::vector<float> ret;
|
||||
for (size_t i = 0; i < tokens.size(); i++) {
|
||||
batch.clear();
|
||||
batch.add(tokens[i], i, 0, true);
|
||||
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
throw std::runtime_error("decode failed");
|
||||
}
|
||||
const float * logits = llama_get_logits_ith(lctx, 0);
|
||||
for (uint32_t j = 0; j < n_vocab; j++) {
|
||||
ret.push_back(logits[j]);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_fusion_t finfo,
|
||||
std::vector<const char *> & labels, std::vector<uint64_t> & counts) {
|
||||
const int n = api_stats_get(finfo, nullptr, nullptr, 0);
|
||||
labels.assign(n, nullptr);
|
||||
counts.assign(n, 0);
|
||||
api_stats_get(finfo, labels.data(), counts.data(), n);
|
||||
}
|
||||
|
||||
struct baseline_row {
|
||||
std::string arch;
|
||||
bool moe;
|
||||
std::string mode;
|
||||
std::string label;
|
||||
uint64_t count;
|
||||
};
|
||||
|
||||
// one row of the per-label report
|
||||
struct fusion_row {
|
||||
std::string arch;
|
||||
bool moe;
|
||||
std::string mode;
|
||||
std::string label;
|
||||
uint64_t count_fused;
|
||||
uint64_t count_unfused;
|
||||
uint64_t expected;
|
||||
double nmse_fus;
|
||||
double nmse_dev;
|
||||
bool ok_count; // counts match the baseline
|
||||
bool ok_nmse; // nmse within epsilon
|
||||
};
|
||||
|
||||
static void usage(const char * argv0) {
|
||||
printf("%s: verify fusion counts on a device against a per-device baseline\n\n", argv0);
|
||||
printf("usage: %s [options]\n\n", argv0);
|
||||
printf("options:\n");
|
||||
printf(" --models DIR run over all .gguf models in a directory\n");
|
||||
printf(" --model FILE run over a single model file (mutually exclusive with --models)\n");
|
||||
printf(" --device NAME device to run on (e.g. MTL0, CPU)\n");
|
||||
printf(" --record CSV write the golden baseline\n");
|
||||
printf(" --check CSV validate the counters against a baseline (default)\n");
|
||||
printf(" -h, --help show this message and exit\n");
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
std::string models_dir;
|
||||
std::string model_file;
|
||||
std::string device_name;
|
||||
std::string record_path;
|
||||
std::string check_path;
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
const std::string arg = argv[i];
|
||||
const auto next = [&](const char * name) -> std::string {
|
||||
if (i + 1 >= argc) {
|
||||
LOG_ERR("%s: %s requires an argument\n", __func__, name);
|
||||
exit(1);
|
||||
}
|
||||
return argv[++i];
|
||||
};
|
||||
if (arg == "-h" || arg == "--help") {
|
||||
usage(argv[0]);
|
||||
exit(0);
|
||||
}
|
||||
if (arg == "--models") { models_dir = next("--models"); }
|
||||
else if (arg == "--model") { model_file = next("--model"); }
|
||||
else if (arg == "--device"){ device_name = next("--device"); }
|
||||
else if (arg == "--record"){ record_path = next("--record"); }
|
||||
else if (arg == "--check") { check_path = next("--check"); }
|
||||
else {
|
||||
LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (device_name.empty()) {
|
||||
LOG_ERR("%s: --device NAME is required\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
if (models_dir.empty() && model_file.empty()) {
|
||||
LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
if (!models_dir.empty() && !model_file.empty()) {
|
||||
LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
if (!record_path.empty() && !check_path.empty()) {
|
||||
LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::vector<std::string> models;
|
||||
if (!model_file.empty()) {
|
||||
if (!std::filesystem::is_regular_file(model_file)) {
|
||||
LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str());
|
||||
return 1;
|
||||
}
|
||||
models.push_back(model_file);
|
||||
} else {
|
||||
if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {
|
||||
LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());
|
||||
return 1;
|
||||
}
|
||||
for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {
|
||||
if (entry.is_regular_file() && entry.path().extension() == ".gguf") {
|
||||
models.push_back(entry.path().string());
|
||||
}
|
||||
}
|
||||
std::sort(models.begin(), models.end());
|
||||
|
||||
if (models.empty()) {
|
||||
LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
common_init();
|
||||
llama_backend_init();
|
||||
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str());
|
||||
if (!dev) {
|
||||
LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n",
|
||||
__func__, device_name.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
// resolve the generic fusion debugging functions through the ad-hoc get_proc_address
|
||||
// mechanism; a backend that does not adopt fusion debugging exports none of them
|
||||
auto * reg = ggml_backend_dev_backend_reg(dev);
|
||||
|
||||
// output naming uses the backend base name (e.g. "MTL") rather than the specific device
|
||||
// name (e.g. "MTL0") the test was invoked with
|
||||
const std::string base_name = ggml_backend_reg_name(reg);
|
||||
|
||||
auto api_get = (fusion_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_get");
|
||||
auto api_stats_init = (fusion_stats_init_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_init");
|
||||
auto api_stats_reset = (fusion_stats_reset_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_reset");
|
||||
auto api_stats_get = (fusion_stats_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_get");
|
||||
auto api_set_enabled = (fusion_set_enabled_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_set_enabled");
|
||||
|
||||
if (!api_get || !api_stats_init || !api_set_enabled || !api_stats_reset || !api_stats_get) {
|
||||
LOG_ERR("%s: device '%s' does not export the generic fusion debugging API "
|
||||
"(ggml_backend_fusion_*) - cannot run the fusion regression test\n",
|
||||
__func__, device_name.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
ggml_backend_fusion_t finfo = api_get(dev);
|
||||
|
||||
// enable fusions stats
|
||||
api_stats_init(finfo);
|
||||
|
||||
const bool has_counts = true;
|
||||
|
||||
// load the baseline (if any): key arch|moe|mode|label -> expected row
|
||||
std::map<std::string, baseline_row> baseline;
|
||||
if (!check_path.empty()) {
|
||||
std::ifstream in(check_path);
|
||||
if (!in) {
|
||||
LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str());
|
||||
return 1;
|
||||
}
|
||||
std::string line;
|
||||
while (std::getline(in, line)) {
|
||||
if (line.empty() || line[0] == '#') {
|
||||
continue;
|
||||
}
|
||||
std::vector<std::string> cols;
|
||||
size_t pos = 0;
|
||||
while ((pos = line.find(',')) != std::string::npos) {
|
||||
cols.push_back(trim(line.substr(0, pos)));
|
||||
line.erase(0, pos + 1);
|
||||
}
|
||||
cols.push_back(trim(line));
|
||||
if (cols.size() != 5) {
|
||||
continue;
|
||||
}
|
||||
const std::string key = cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3];
|
||||
baseline[key] = { cols[0], cols[1] == "1", cols[2], cols[3], std::stoull(cols[4]) };
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<fusion_row> rows;
|
||||
|
||||
std::set<std::string> run_arches;
|
||||
std::set<std::string> produced;
|
||||
|
||||
LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str());
|
||||
|
||||
const size_t seed = 1;
|
||||
|
||||
for (const auto & model_path : models) {
|
||||
const std::string arch = get_arch(model_path);
|
||||
const bool moe = arch.find("moe") != std::string::npos;
|
||||
|
||||
llama_model_ptr model;
|
||||
llama_model_ptr model_cpu;
|
||||
uint32_t n_vocab = 0;
|
||||
try {
|
||||
model = load_model(model_path, dev);
|
||||
model_cpu = load_model(model_path, ggml_backend_dev_by_name("CPU"));
|
||||
n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get()));
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("%s: %s: %s\n", __func__, model_path.c_str(), e.what());
|
||||
continue;
|
||||
}
|
||||
|
||||
run_arches.insert(arch);
|
||||
|
||||
struct mode_cfg {
|
||||
std::string name;
|
||||
std::vector<float> (*decode)(llama_model *, llama_context *, const std::vector<llama_token> &);
|
||||
int n_tokens;
|
||||
int n_graphs; // graph runs per mode (prefill=1, decode=16)
|
||||
};
|
||||
const mode_cfg modes[] = {
|
||||
{ "prefill", decode_prefill, 32, 1 },
|
||||
{ "decode", decode_gen, 16, 16 },
|
||||
};
|
||||
|
||||
// per-label, per-mode data for this model; prefill and decode are merged into a single
|
||||
// "any" row when their per-graph counts match
|
||||
struct mode_data {
|
||||
bool present;
|
||||
uint64_t count_fused; // per graph
|
||||
uint64_t count_unfused; // per graph
|
||||
double nmse_fus;
|
||||
double nmse_dev;
|
||||
bool ok_nmse;
|
||||
};
|
||||
std::map<std::string, std::array<mode_data, 2>> mdata;
|
||||
|
||||
for (int mi = 0; mi < 2; mi++) {
|
||||
const mode_cfg & mode = modes[mi];
|
||||
const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed);
|
||||
|
||||
// CPU reference for this mode (fresh context, fresh state)
|
||||
std::vector<float> logits_cpu;
|
||||
try {
|
||||
llama_context_ptr ctx = create_ctx(model_cpu.get(), 32);
|
||||
logits_cpu = mode.decode(model_cpu.get(), ctx.get(), tokens);
|
||||
} catch (const std::exception & e) {
|
||||
LOG_WRN("%s: %s: cpu reference: %s\n", __func__, model_path.c_str(), e.what());
|
||||
}
|
||||
|
||||
// fused run on a fresh context (fresh state)
|
||||
std::vector<float> logits_fused;
|
||||
std::vector<const char *> labels;
|
||||
std::vector<uint64_t> counts_fused;
|
||||
{
|
||||
llama_context_ptr ctx = create_ctx(model.get(), 32);
|
||||
if (has_counts) {
|
||||
api_set_enabled(finfo, true);
|
||||
api_stats_reset(finfo);
|
||||
}
|
||||
logits_fused = mode.decode(model.get(), ctx.get(), tokens);
|
||||
if (has_counts) {
|
||||
read_counts(api_stats_get, finfo, labels, counts_fused);
|
||||
}
|
||||
}
|
||||
|
||||
// unfused run on another fresh context (fresh state)
|
||||
std::vector<float> logits_unfused;
|
||||
std::vector<uint64_t> counts_unfused;
|
||||
{
|
||||
llama_context_ptr ctx = create_ctx(model.get(), 32);
|
||||
if (has_counts) {
|
||||
api_set_enabled(finfo, false);
|
||||
api_stats_reset(finfo);
|
||||
}
|
||||
logits_unfused = mode.decode(model.get(), ctx.get(), tokens);
|
||||
if (has_counts) {
|
||||
read_counts(api_stats_get, finfo, labels, counts_unfused);
|
||||
}
|
||||
}
|
||||
|
||||
const double nmse_fus = nmse(logits_fused, logits_unfused);
|
||||
const double nmse_dev = logits_cpu.empty() ? 0.0 : nmse(logits_fused, logits_cpu);
|
||||
|
||||
if (has_counts) {
|
||||
for (int i = 0; i < (int) labels.size(); i++) {
|
||||
const uint64_t fused = counts_fused[i] / mode.n_graphs;
|
||||
const uint64_t unfused = counts_unfused[i] / mode.n_graphs;
|
||||
if (fused == 0 && unfused == 0) {
|
||||
continue;
|
||||
}
|
||||
auto & d = mdata[labels[i]][mi];
|
||||
d.present = true;
|
||||
d.count_fused = fused;
|
||||
d.count_unfused = unfused;
|
||||
d.nmse_fus = nmse_fus;
|
||||
d.nmse_dev = nmse_dev;
|
||||
d.ok_nmse = nmse_fus <= 1e-4;
|
||||
}
|
||||
} else {
|
||||
rows.push_back({ arch, moe, mode.name, "?", 0, 0, 0, nmse_fus, nmse_dev, true, nmse_fus <= 1e-4 });
|
||||
}
|
||||
}
|
||||
|
||||
// build the per-label rows, merging prefill and decode into "any" when the per-graph
|
||||
// counts match (they always do for the deterministic fusion table)
|
||||
if (has_counts) {
|
||||
for (auto & kv : mdata) {
|
||||
const std::string & label = kv.first;
|
||||
const auto & d = kv.second;
|
||||
const bool both = d[0].present && d[1].present;
|
||||
const bool match = both && d[0].count_fused == d[1].count_fused;
|
||||
|
||||
if (match) {
|
||||
// one "any" row; use the worst NMSE across the two modes
|
||||
const std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label;
|
||||
produced.insert(any_key);
|
||||
const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key).count : 0;
|
||||
const bool ok_count = check_path.empty() || d[0].count_fused == expected;
|
||||
const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse;
|
||||
const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus);
|
||||
const double nmse_dev = std::max(d[0].nmse_dev, d[1].nmse_dev);
|
||||
rows.push_back({ arch, moe, "any", label, d[0].count_fused, d[0].count_unfused,
|
||||
expected, nmse_fus, nmse_dev, ok_count, ok_nmse });
|
||||
} else {
|
||||
// counts differ - keep a separate row per mode
|
||||
for (int mi = 0; mi < 2; mi++) {
|
||||
if (!d[mi].present) {
|
||||
continue;
|
||||
}
|
||||
const mode_data & a = d[mi];
|
||||
const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" + modes[mi].name + "|" + label;
|
||||
produced.insert(mode_key);
|
||||
const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key).count : 0;
|
||||
const bool ok_count = check_path.empty() || a.count_fused == expected;
|
||||
rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused,
|
||||
expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str());
|
||||
}
|
||||
|
||||
if (!check_path.empty()) {
|
||||
for (const auto & kv : baseline) {
|
||||
if (produced.count(kv.first) || !run_arches.count(kv.second.arch)) {
|
||||
continue;
|
||||
}
|
||||
const baseline_row & b = kv.second;
|
||||
LOG_WRN("%s: %s: fusion %s x%llu is no longer matched\n", __func__, b.arch.c_str(),
|
||||
b.label.c_str(), (unsigned long long) b.count);
|
||||
rows.push_back({ b.arch, b.moe, b.mode, b.label, 0, 0, b.count, 0.0, 0.0, false, true });
|
||||
}
|
||||
}
|
||||
|
||||
// print the report
|
||||
{
|
||||
std::ofstream out(record_path);
|
||||
std::ostream & os = record_path.empty() ? std::cout : out;
|
||||
if (!record_path.empty()) {
|
||||
os << "# test-fusion baseline for device " << base_name << "\n";
|
||||
os << "# " << std::left
|
||||
<< std::setw(18) << "arch" << ','
|
||||
<< std::setw(4) << "moe" << ','
|
||||
<< std::setw(8) << "mode" << ','
|
||||
<< std::setw(28) << "label" << ','
|
||||
<< std::right << std::setw(7) << "count" << '\n';
|
||||
}
|
||||
|
||||
LOG_INF("%-20s %-4s %-8s %-22s %7s %7s %7s %10s %10s %s\n",
|
||||
"arch", "moe", "mode", "label", "fused", "unfused", "expected", "nmse_fus", "nmse_dev", "status");
|
||||
int n_ok = 0;
|
||||
int n_bad = 0;
|
||||
for (const auto & r : rows) {
|
||||
const bool ok = r.ok_count && r.ok_nmse;
|
||||
const char * status = ok ? "ok" : "FAIL";
|
||||
if (ok) { n_ok++; } else { n_bad++; }
|
||||
LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n",
|
||||
r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label.c_str(),
|
||||
(unsigned long long) r.count_fused, (unsigned long long) r.count_unfused,
|
||||
(unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status);
|
||||
if (!record_path.empty()) {
|
||||
os << std::left
|
||||
<< std::setw(20) << r.arch << ','
|
||||
<< std::setw(4) << (r.moe ? "1" : "0") << ','
|
||||
<< std::setw(8) << r.mode << ','
|
||||
<< std::setw(28) << r.label << ','
|
||||
<< std::right << std::setw(7) << r.count_fused << '\n';
|
||||
}
|
||||
}
|
||||
LOG_INF("summary: %d ok, %d failed\n", n_ok, n_bad);
|
||||
if (!record_path.empty()) {
|
||||
LOG_INF("%s: baseline written to '%s'\n", __func__, record_path.c_str());
|
||||
}
|
||||
|
||||
if (n_bad && !models_dir.empty() && !check_path.empty()) {
|
||||
LOG_WRN("%s: if the fusion counts are expected to change, run with --record to update the baseline:\n"
|
||||
"\n"
|
||||
"./bin/test-llama-archs -o %s\n"
|
||||
"%s --device %s --models %s --record %s\n",
|
||||
__func__, models_dir.c_str(), argv[0], device_name.c_str(), models_dir.c_str(), check_path.c_str());
|
||||
}
|
||||
|
||||
return n_bad;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
#include "../src/unicode.h"
|
||||
#include "../src/llama-grammar.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <sstream>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
static bool llama_grammar_validate(struct llama_grammar * grammar, const std::string & input_str, size_t & error_pos, std::string & error_msg) {
|
||||
const auto cpts = unicode_cpts_from_utf8(input_str);
|
||||
|
||||
auto & stacks_cur = llama_grammar_get_stacks(grammar);
|
||||
|
||||
size_t pos = 0;
|
||||
for (const auto & cpt : cpts) {
|
||||
llama_grammar_accept(grammar, cpt);
|
||||
|
||||
if (stacks_cur.empty()) {
|
||||
error_pos = pos;
|
||||
error_msg = "Unexpected character '" + unicode_cpt_to_utf8(cpt) + "'";
|
||||
return false;
|
||||
}
|
||||
++pos;
|
||||
}
|
||||
|
||||
for (const auto & stack : stacks_cur) {
|
||||
if (stack.empty()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
error_pos = pos;
|
||||
error_msg = "Unexpected end of input";
|
||||
return false;
|
||||
}
|
||||
|
||||
static void print_error_message(const std::string & input_str, size_t error_pos, const std::string & error_msg) {
|
||||
fprintf(stdout, "Input string is invalid according to the grammar.\n");
|
||||
fprintf(stdout, "Error: %s at position %zu\n", error_msg.c_str(), error_pos);
|
||||
fprintf(stdout, "\n");
|
||||
fprintf(stdout, "Input string:\n");
|
||||
fprintf(stdout, "%s", input_str.substr(0, error_pos).c_str());
|
||||
if (error_pos < input_str.size()) {
|
||||
fprintf(stdout, "\033[1;31m%c", input_str[error_pos]);
|
||||
if (error_pos+1 < input_str.size()) {
|
||||
fprintf(stdout, "\033[0;31m%s", input_str.substr(error_pos+1).c_str());
|
||||
}
|
||||
fprintf(stdout, "\033[0m\n");
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc != 3) {
|
||||
fprintf(stdout, "Usage: %s <grammar_filename> <input_filename>\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const std::string grammar_filename = argv[1];
|
||||
const std::string input_filename = argv[2];
|
||||
|
||||
// Read the GBNF grammar file
|
||||
FILE* grammar_file = fopen(grammar_filename.c_str(), "r");
|
||||
if (!grammar_file) {
|
||||
fprintf(stdout, "Failed to open grammar file: %s\n", grammar_filename.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::string grammar_str;
|
||||
{
|
||||
std::ifstream grammar_file(grammar_filename);
|
||||
GGML_ASSERT(grammar_file.is_open() && "Failed to open grammar file");
|
||||
std::stringstream buffer;
|
||||
buffer << grammar_file.rdbuf();
|
||||
grammar_str = buffer.str();
|
||||
}
|
||||
|
||||
llama_grammar * grammar = llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0);
|
||||
if (grammar == nullptr) {
|
||||
fprintf(stdout, "Failed to initialize llama_grammar\n");
|
||||
return 1;
|
||||
}
|
||||
// Read the input file
|
||||
std::string input_str;
|
||||
{
|
||||
std::ifstream input_file(input_filename);
|
||||
GGML_ASSERT(input_file.is_open() && "Failed to open input file");
|
||||
std::stringstream buffer;
|
||||
buffer << input_file.rdbuf();
|
||||
input_str = buffer.str();
|
||||
}
|
||||
|
||||
// Validate the input string against the grammar
|
||||
size_t error_pos;
|
||||
std::string error_msg;
|
||||
bool is_valid = llama_grammar_validate(grammar, input_str, error_pos, error_msg);
|
||||
|
||||
if (is_valid) {
|
||||
fprintf(stdout, "Input string is valid according to the grammar.\n");
|
||||
} else {
|
||||
print_error_message(input_str, error_pos, error_msg);
|
||||
}
|
||||
|
||||
// Clean up
|
||||
llama_grammar_free_impl(grammar);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// test-gelu-quick.cpp: validate the F32 gelu_quick and geglu_quick CPU kernels against the exact formula.
|
||||
//
|
||||
// test-backend-ops uses the CPU backend as its reference, so it cannot see an error in the CPU kernels
|
||||
// themselves. This test computes gelu_quick(x) = x / (1 + exp(-1.702 x)) in double precision and compares
|
||||
// every F32 output of the CPU backend with it: dense ranges, values outside the F16 range, values near
|
||||
// the float limits, and lengths that leave a tail after the vector loop.
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <cfloat>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
// Same coefficient as the CPU kernels
|
||||
static const double GELU_QUICK_COEF_REF = (double) 1.702f;
|
||||
|
||||
static double gelu_quick_ref(double x) {
|
||||
return x / (1.0 + std::exp(-GELU_QUICK_COEF_REF * x));
|
||||
}
|
||||
|
||||
// Deterministic LCG mapped to [-1, 1]
|
||||
static uint64_t g_rng = 0x12345678ULL;
|
||||
static float frand(void) {
|
||||
g_rng = g_rng * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||
return (float)((g_rng >> 33) & 0xffffff) / (float)0x800000 - 1.0f;
|
||||
}
|
||||
|
||||
static std::vector<float> make_inputs(void) {
|
||||
std::vector<float> x;
|
||||
for (int i = 0; i <= 200000; i++) {
|
||||
x.push_back(-10.0f + 20.0f * (float) i / 200000.0f);
|
||||
}
|
||||
for (int i = 0; i <= 2000; i++) {
|
||||
x.push_back(-150.0f + 300.0f * (float) i / 2000.0f);
|
||||
}
|
||||
// outside the F16 range, near the float limits, and subnormal; both signs
|
||||
const float edge[] = { 65504.0f, 65520.0f, 1e5f, 1e10f, 1e30f, FLT_MAX, 1e-30f, FLT_MIN, FLT_MIN / 16, 0.0f };
|
||||
for (float v : edge) {
|
||||
x.push_back(v);
|
||||
x.push_back(-v);
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
static bool matches(float y, double ref) {
|
||||
if (std::isnan(ref)) {
|
||||
return std::isnan(y);
|
||||
}
|
||||
if (std::isinf(ref)) {
|
||||
return (double) y == ref;
|
||||
}
|
||||
return std::fabs((double) y - ref) <= 1e-6 + 1e-6 * std::fabs(ref);
|
||||
}
|
||||
|
||||
// Runs gelu_quick and geglu_quick on x[0, n) and returns the number of outputs that differ from the reference
|
||||
static int check(const float * x, const float * g, int64_t n) {
|
||||
struct ggml_init_params params = {
|
||||
/* .mem_size = */ (size_t) 64 << 20,
|
||||
/* .mem_base = */ NULL,
|
||||
/* .no_alloc = */ false,
|
||||
};
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n);
|
||||
struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n);
|
||||
memcpy(a->data, x, n * sizeof(float));
|
||||
memcpy(b->data, g, n * sizeof(float));
|
||||
|
||||
struct ggml_tensor * y_gelu = ggml_gelu_quick(ctx, a);
|
||||
struct ggml_tensor * y_geglu = ggml_geglu_quick_split(ctx, a, b);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(gf, y_gelu);
|
||||
ggml_build_forward_expand(gf, y_geglu);
|
||||
ggml_graph_compute_with_ctx(ctx, gf, 1);
|
||||
|
||||
int bad = 0;
|
||||
double max_err = 0.0;
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
const double r0 = gelu_quick_ref(x[i]);
|
||||
const double r1 = r0 * g[i];
|
||||
const float y0 = ((const float *) y_gelu->data)[i];
|
||||
const float y1 = ((const float *) y_geglu->data)[i];
|
||||
if (std::isfinite(r0) && std::isfinite(y0)) {
|
||||
max_err = std::fmax(max_err, std::fabs((double) y0 - r0) / (1.0 + std::fabs(r0)));
|
||||
}
|
||||
if (!matches(y0, r0)) {
|
||||
if (bad < 3) {
|
||||
printf(" gelu_quick(%g) = %g, expected %g\n", x[i], y0, r0);
|
||||
}
|
||||
bad++;
|
||||
}
|
||||
if (!matches(y1, r1)) {
|
||||
if (bad < 3) {
|
||||
printf(" geglu_quick(%g, %g) = %g, expected %g\n", x[i], g[i], y1, r1);
|
||||
}
|
||||
bad++;
|
||||
}
|
||||
}
|
||||
printf("gelu_quick n=%6d: max err %.2e, %d mismatches %s\n", (int) n, max_err, bad, bad == 0 ? "OK" : "FAIL");
|
||||
|
||||
ggml_free(ctx);
|
||||
return bad;
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
const std::vector<float> x = make_inputs();
|
||||
std::vector<float> g(x.size());
|
||||
for (float & v : g) {
|
||||
v = frand();
|
||||
}
|
||||
|
||||
int fails = 0;
|
||||
|
||||
// every input in one call
|
||||
fails += check(x.data(), g.data(), (int64_t) x.size()) != 0;
|
||||
|
||||
// short lengths end on the edge values and leave a tail after the vector loop
|
||||
const int64_t lengths[] = { 1, 2, 3, 5, 7, 8, 9, 15, 16, 17, 23, 31, 33 };
|
||||
for (int64_t n : lengths) {
|
||||
const int64_t off = (int64_t) x.size() - n;
|
||||
fails += check(x.data() + off, g.data() + off, n) != 0;
|
||||
}
|
||||
|
||||
printf(fails == 0 ? "all gelu_quick checks passed\n" : "%d gelu_quick checks FAILED\n", fails);
|
||||
return fails == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
#include "gguf-model-data.h"
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
#define TEST_ASSERT(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
fprintf(stderr, "FAIL: %s (line %d): %s\n", #cond, __LINE__, msg); \
|
||||
return 1; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
int main() {
|
||||
fprintf(stderr, "=== test-gguf-model-data ===\n");
|
||||
|
||||
// Fetch Qwen3-0.6B Q8_0 metadata
|
||||
auto result = gguf_fetch_model_meta("ggml-org/Qwen3-0.6B-GGUF", "Q8_0");
|
||||
|
||||
if (!result.has_value()) {
|
||||
fprintf(stderr, "SKIP: could not fetch model metadata (no network or HTTP disabled)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
const auto & model = result.value();
|
||||
|
||||
fprintf(stderr, "Architecture: %s\n", model.architecture.c_str());
|
||||
fprintf(stderr, "n_embd: %u\n", model.n_embd);
|
||||
fprintf(stderr, "n_ff: %u\n", model.n_ff);
|
||||
fprintf(stderr, "n_vocab: %u\n", model.n_vocab);
|
||||
fprintf(stderr, "n_layer: %u\n", model.n_layer);
|
||||
fprintf(stderr, "n_head: %u\n", model.n_head);
|
||||
fprintf(stderr, "n_head_kv: %u\n", model.n_head_kv);
|
||||
fprintf(stderr, "n_expert: %u\n", model.n_expert);
|
||||
fprintf(stderr, "n_embd_head_k: %u\n", model.n_embd_head_k);
|
||||
fprintf(stderr, "n_embd_head_v: %u\n", model.n_embd_head_v);
|
||||
fprintf(stderr, "tensors: %zu\n", model.tensors.size());
|
||||
|
||||
// Verify architecture
|
||||
TEST_ASSERT(model.architecture == "qwen3", "expected architecture 'qwen3'");
|
||||
|
||||
// Verify key dimensions (Qwen3-0.6B)
|
||||
TEST_ASSERT(model.n_layer == 28, "expected n_layer == 28");
|
||||
TEST_ASSERT(model.n_embd == 1024, "expected n_embd == 1024");
|
||||
TEST_ASSERT(model.n_head == 16, "expected n_head == 16");
|
||||
TEST_ASSERT(model.n_head_kv == 8, "expected n_head_kv == 8");
|
||||
TEST_ASSERT(model.n_expert == 0, "expected n_expert == 0 (not MoE)");
|
||||
TEST_ASSERT(model.n_vocab == 151936, "expected n_vocab == 151936");
|
||||
|
||||
// Verify tensor count
|
||||
TEST_ASSERT(model.tensors.size() == 311, "expected tensor count == 311");
|
||||
|
||||
// Verify known tensor names exist
|
||||
bool found_attn_q = false;
|
||||
bool found_token_embd = false;
|
||||
bool found_output_norm = false;
|
||||
for (const auto & t : model.tensors) {
|
||||
if (t.name == "blk.0.attn_q.weight") {
|
||||
found_attn_q = true;
|
||||
}
|
||||
if (t.name == "token_embd.weight") {
|
||||
found_token_embd = true;
|
||||
}
|
||||
if (t.name == "output_norm.weight") {
|
||||
found_output_norm = true;
|
||||
}
|
||||
}
|
||||
TEST_ASSERT(found_attn_q, "expected tensor 'blk.0.attn_q.weight'");
|
||||
TEST_ASSERT(found_token_embd, "expected tensor 'token_embd.weight'");
|
||||
TEST_ASSERT(found_output_norm, "expected tensor 'output_norm.weight'");
|
||||
|
||||
// Verify token_embd.weight shape
|
||||
for (const auto & t : model.tensors) {
|
||||
if (t.name == "token_embd.weight") {
|
||||
TEST_ASSERT(t.ne[0] == 1024, "expected token_embd.weight ne[0] == 1024");
|
||||
TEST_ASSERT(t.n_dims == 2, "expected token_embd.weight to be 2D");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Test that second call uses cache (just call again, it should work)
|
||||
auto result2 = gguf_fetch_model_meta("ggml-org/Qwen3-0.6B-GGUF", "Q8_0");
|
||||
TEST_ASSERT(result2.has_value(), "cached fetch should succeed");
|
||||
TEST_ASSERT(result2->tensors.size() == model.tensors.size(), "cached result should match");
|
||||
|
||||
// Test a split MoE model without specifying quant (should default to Q8_0)
|
||||
auto result3 = gguf_fetch_model_meta("ggml-org/GLM-4.6V-GGUF");
|
||||
if (!result3.has_value()) {
|
||||
fprintf(stderr, "SKIP: could not fetch GLM-4.6V metadata (no network?)\n");
|
||||
return 0;
|
||||
}
|
||||
const auto & model3 = result3.value();
|
||||
|
||||
fprintf(stderr, "Architecture: %s\n", model3.architecture.c_str());
|
||||
fprintf(stderr, "n_embd: %u\n", model3.n_embd);
|
||||
fprintf(stderr, "n_ff: %u\n", model3.n_ff);
|
||||
fprintf(stderr, "n_vocab: %u\n", model3.n_vocab);
|
||||
fprintf(stderr, "n_layer: %u\n", model3.n_layer);
|
||||
fprintf(stderr, "n_head: %u\n", model3.n_head);
|
||||
fprintf(stderr, "n_head_kv: %u\n", model3.n_head_kv);
|
||||
fprintf(stderr, "n_expert: %u\n", model3.n_expert);
|
||||
fprintf(stderr, "n_embd_head_k: %u\n", model3.n_embd_head_k);
|
||||
fprintf(stderr, "n_embd_head_v: %u\n", model3.n_embd_head_v);
|
||||
fprintf(stderr, "tensors: %zu\n", model3.tensors.size());
|
||||
|
||||
// Verify architecture
|
||||
TEST_ASSERT(model3.architecture == "glm4moe", "expected architecture 'glm4moe'");
|
||||
|
||||
// Verify key dimensions (GLM-4.6V)
|
||||
TEST_ASSERT(model3.n_layer == 46, "expected n_layer == 46");
|
||||
TEST_ASSERT(model3.n_embd == 4096, "expected n_embd == 4096");
|
||||
TEST_ASSERT(model3.n_head == 96, "expected n_head == 96");
|
||||
TEST_ASSERT(model3.n_head_kv == 8, "expected n_head_kv == 8");
|
||||
TEST_ASSERT(model3.n_expert == 128, "expected n_expert == 128 (MoE)");
|
||||
TEST_ASSERT(model3.n_vocab == 151552, "expected n_vocab == 151552");
|
||||
|
||||
// Verify tensor count
|
||||
TEST_ASSERT(model3.tensors.size() == 780, "expected tensor count == 780");
|
||||
|
||||
// Test a hybrid-attention model with array-valued head counts
|
||||
auto result4 = gguf_fetch_model_meta("ggml-org/Step-3.5-Flash-GGUF", "Q4_K");
|
||||
if (!result4.has_value()) {
|
||||
fprintf(stderr, "FAIL: could not fetch Step-3.5-Flash metadata\n");
|
||||
return 1;
|
||||
}
|
||||
const auto & model4 = result4.value();
|
||||
|
||||
fprintf(stderr, "Architecture: %s\n", model4.architecture.c_str());
|
||||
fprintf(stderr, "n_embd: %u\n", model4.n_embd);
|
||||
fprintf(stderr, "n_ff: %u\n", model4.n_ff);
|
||||
fprintf(stderr, "n_vocab: %u\n", model4.n_vocab);
|
||||
fprintf(stderr, "n_layer: %u\n", model4.n_layer);
|
||||
fprintf(stderr, "n_head: %u\n", model4.n_head);
|
||||
fprintf(stderr, "n_head_kv: %u\n", model4.n_head_kv);
|
||||
fprintf(stderr, "n_expert: %u\n", model4.n_expert);
|
||||
fprintf(stderr, "n_embd_head_k: %u\n", model4.n_embd_head_k);
|
||||
fprintf(stderr, "n_embd_head_v: %u\n", model4.n_embd_head_v);
|
||||
fprintf(stderr, "tensors: %zu\n", model4.tensors.size());
|
||||
|
||||
TEST_ASSERT(model4.architecture == "step35", "expected architecture 'step35'");
|
||||
|
||||
TEST_ASSERT(model4.n_layer == 45, "expected n_layer == 45");
|
||||
TEST_ASSERT(model4.n_embd == 4096, "expected n_embd == 4096");
|
||||
TEST_ASSERT(model4.n_ff == 11264, "expected n_ff == 11264");
|
||||
TEST_ASSERT(model4.n_head == 64, "expected n_head == 64 (first element of per-layer array)");
|
||||
TEST_ASSERT(model4.n_head_kv == 8, "expected n_head_kv == 8 (first element of per-layer array)");
|
||||
TEST_ASSERT(model4.n_expert == 288, "expected n_expert == 288");
|
||||
TEST_ASSERT(model4.n_embd_head_k == 128, "expected n_embd_head_k == 128");
|
||||
TEST_ASSERT(model4.n_embd_head_v == 128, "expected n_embd_head_v == 128");
|
||||
TEST_ASSERT(model4.n_vocab == 128896, "expected n_vocab == 128896");
|
||||
TEST_ASSERT(model4.tensors.size() == 754, "expected tensor count == 754");
|
||||
|
||||
fprintf(stderr, "=== ALL TESTS PASSED ===\n");
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,598 @@
|
||||
#ifdef NDEBUG
|
||||
#undef NDEBUG
|
||||
#endif
|
||||
|
||||
#include "llama.h"
|
||||
|
||||
// TODO: shold not include libllama sources
|
||||
#include "../src/llama-grammar.h"
|
||||
|
||||
#include <cassert>
|
||||
|
||||
static const char * type_str(llama_gretype type) {
|
||||
switch (type) {
|
||||
case LLAMA_GRETYPE_CHAR: return "LLAMA_GRETYPE_CHAR";
|
||||
case LLAMA_GRETYPE_CHAR_NOT: return "LLAMA_GRETYPE_CHAR_NOT";
|
||||
case LLAMA_GRETYPE_CHAR_ALT: return "LLAMA_GRETYPE_CHAR_ALT";
|
||||
case LLAMA_GRETYPE_CHAR_RNG_UPPER: return "LLAMA_GRETYPE_CHAR_RNG_UPPER";
|
||||
case LLAMA_GRETYPE_RULE_REF: return "LLAMA_GRETYPE_RULE_REF";
|
||||
case LLAMA_GRETYPE_ALT: return "LLAMA_GRETYPE_ALT";
|
||||
case LLAMA_GRETYPE_END: return "LLAMA_GRETYPE_END";
|
||||
default: return "?";
|
||||
}
|
||||
}
|
||||
|
||||
static void verify_parsing(const char *grammar_bytes, const std::vector<std::pair<std::string, uint32_t>> expected, const std::vector<llama_grammar_element> &expected_rules) {
|
||||
uint32_t index = 0;
|
||||
llama_grammar_parser parsed_grammar;
|
||||
parsed_grammar.parse(grammar_bytes);
|
||||
|
||||
std::map<uint32_t, std::string> symbol_names;
|
||||
for (auto it = parsed_grammar.symbol_ids.begin(); it != parsed_grammar.symbol_ids.end(); ++it) {
|
||||
symbol_names[it->second] = it->first;
|
||||
}
|
||||
|
||||
auto print_all = [&]() {
|
||||
fprintf(stderr, " verify_parsing(R\"\"\"(%s)\"\"\", {\n", grammar_bytes);
|
||||
for (auto it = parsed_grammar.symbol_ids.begin(); it != parsed_grammar.symbol_ids.end(); ++it) {
|
||||
fprintf(stderr, " {\"%s\", %u},\n", it->first.c_str(), it->second);
|
||||
}
|
||||
fprintf(stderr, " }, {\n");
|
||||
for (size_t i_rule = 0; i_rule < parsed_grammar.rules.size(); i_rule++) {
|
||||
fprintf(stderr, " // %s (index %zu)\n", symbol_names[i_rule].c_str(), i_rule);
|
||||
auto & rule = parsed_grammar.rules[i_rule];
|
||||
for (uint32_t i = 0; i < rule.size(); i++) {
|
||||
std::string rule_str;
|
||||
fprintf(stderr, " {%s, ", type_str(rule[i].type));
|
||||
if (rule[i].type == LLAMA_GRETYPE_CHAR || rule[i].type == LLAMA_GRETYPE_CHAR_ALT ||
|
||||
rule[i].type == LLAMA_GRETYPE_CHAR_NOT || rule[i].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {
|
||||
char c = rule[i].value;
|
||||
if (c == '\n') {
|
||||
fprintf(stderr, "'\\n'");
|
||||
} else if (c == '\t') {
|
||||
fprintf(stderr, "'\\t'");
|
||||
} else if (c == '\r') {
|
||||
fprintf(stderr, "'\\r'");
|
||||
} else if (c == '\0') {
|
||||
fprintf(stderr, "'\\0'");
|
||||
} else {
|
||||
fprintf(stderr, "'%c'", c);
|
||||
}
|
||||
} else if (rule[i].type == LLAMA_GRETYPE_RULE_REF) {
|
||||
fprintf(stderr, "/* %s */ %u", symbol_names[rule[i].value].c_str(), rule[i].value);
|
||||
} else {
|
||||
fprintf(stderr, "%u", rule[i].value);
|
||||
}
|
||||
fprintf(stderr, "},\n");
|
||||
}
|
||||
}
|
||||
fprintf(stderr, " });\n");
|
||||
};
|
||||
|
||||
if (getenv("TEST_GRAMMAR_PARSER_PRINT_ALL")) {
|
||||
print_all();
|
||||
fprintf(stderr, "\n");
|
||||
return;
|
||||
}
|
||||
|
||||
fprintf(stderr, "Testing grammar:%s\n", grammar_bytes);
|
||||
|
||||
if (parsed_grammar.symbol_ids.size() != expected.size()) {
|
||||
fprintf(stderr, "Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n");
|
||||
print_all();
|
||||
assert(parsed_grammar.symbol_ids.size() == expected.size());
|
||||
}
|
||||
|
||||
for (auto it = parsed_grammar.symbol_ids.begin(); it != parsed_grammar.symbol_ids.end(); ++it)
|
||||
{
|
||||
std::string key = it->first;
|
||||
uint32_t value = it->second;
|
||||
std::pair<std::string, uint32_t> expected_pair = expected[index];
|
||||
|
||||
// pretty print error message before asserting
|
||||
if (expected_pair.first != key || expected_pair.second != value)
|
||||
{
|
||||
fprintf(stderr, "index: %u\n", index);
|
||||
fprintf(stderr, "expected_pair: %s, %u\n", expected_pair.first.c_str(), expected_pair.second);
|
||||
fprintf(stderr, "actual_pair: %s, %u\n", key.c_str(), value);
|
||||
fprintf(stderr, "expected_pair != actual_pair\n");
|
||||
fprintf(stderr, "Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n");
|
||||
print_all();
|
||||
}
|
||||
|
||||
assert(expected_pair.first == key && expected_pair.second == value);
|
||||
|
||||
index++;
|
||||
}
|
||||
|
||||
index = 0;
|
||||
for (auto rule : parsed_grammar.rules)
|
||||
{
|
||||
// compare rule to expected rule
|
||||
for (uint32_t i = 0; i < rule.size(); i++)
|
||||
{
|
||||
llama_grammar_element element = rule[i];
|
||||
llama_grammar_element expected_element = expected_rules[index];
|
||||
|
||||
// pretty print error message before asserting
|
||||
if (expected_element.type != element.type || expected_element.value != element.value)
|
||||
{
|
||||
fprintf(stderr, "index: %u\n", index);
|
||||
fprintf(stderr, "expected_element: %s, %u\n", type_str(expected_element.type), expected_element.value);
|
||||
fprintf(stderr, "actual_element: %s, %u\n", type_str(element.type), element.value);
|
||||
fprintf(stderr, "expected_element != actual_element\n");
|
||||
fprintf(stderr, "all elements:\n");
|
||||
fprintf(stderr, "Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n");
|
||||
print_all();
|
||||
}
|
||||
|
||||
assert(expected_element.type == element.type && expected_element.value == element.value);
|
||||
index++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void verify_failure(const char * grammar_bytes) {
|
||||
fprintf(stderr, "Testing expected failure:%s\n", grammar_bytes);
|
||||
llama_grammar_parser result;
|
||||
result.parse(grammar_bytes);
|
||||
assert(result.rules.empty() && "should have failed");
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
verify_failure(R"""(
|
||||
root ::= "a"{,}"
|
||||
)""");
|
||||
|
||||
verify_failure(R"""(
|
||||
root ::= (((((([^x]*){0,99}){0,99}){0,99}){0,99}){0,99}){0,99}
|
||||
)""");
|
||||
|
||||
verify_failure(R"""(
|
||||
root ::= "a"{,10}"
|
||||
)""");
|
||||
|
||||
verify_failure(R"""(
|
||||
root ::= "a"{5000}
|
||||
)""");
|
||||
|
||||
verify_failure(R"""(
|
||||
root ::= "a"{5000,}
|
||||
)""");
|
||||
|
||||
verify_failure(R"""(
|
||||
root ::= "a"{5000,6000}
|
||||
)""");
|
||||
|
||||
// Recursion depth limit exceeded (Issue #29495)
|
||||
{
|
||||
std::string deep_grammar = "root ::= ";
|
||||
int n_depth = 300;
|
||||
for (int i = 0; i < n_depth; ++i) {
|
||||
deep_grammar += "(";
|
||||
}
|
||||
deep_grammar += "\"x\"";
|
||||
for (int i = 0; i < n_depth; ++i) {
|
||||
deep_grammar += ")";
|
||||
}
|
||||
verify_failure(deep_grammar.c_str());
|
||||
}
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"{0,5000}
|
||||
)""", {
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_1 (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"{3,5000}
|
||||
)""", {
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_1 (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"
|
||||
)""", {
|
||||
{"root", 0},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a" | [bdx-z] | [^1-3]
|
||||
)""", {
|
||||
{"root", 0},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_CHAR, 'b'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, 'd'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, 'x'},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, 'z'},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_CHAR_NOT, '1'},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, '3'},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= a+
|
||||
a ::= "a"
|
||||
)""", {
|
||||
{"a", 1},
|
||||
{"root", 0},
|
||||
{"root_2", 2},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* a */ 1},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_2 */ 2},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// a (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_2 (index 2)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* a */ 1},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_2 */ 2},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"+
|
||||
)""", {
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_1 (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= a?
|
||||
a ::= "a"
|
||||
)""", {
|
||||
{"a", 1},
|
||||
{"root", 0},
|
||||
{"root_2", 2},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_2 */ 2},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// a (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_2 (index 2)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* a */ 1},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"?
|
||||
)""", {
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_1 (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= a*
|
||||
a ::= "a"
|
||||
)""", {
|
||||
{"a", 1},
|
||||
{"root", 0},
|
||||
{"root_2", 2},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_2 */ 2},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// a (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_2 (index 2)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* a */ 1},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_2 */ 2},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"*
|
||||
)""", {
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_1 (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"{2}
|
||||
)""", {
|
||||
{"root", 0},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"{2,}
|
||||
)""", {
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_1 (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"{ 4}
|
||||
)""", {
|
||||
{"root", 0},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= "a"{2,4}
|
||||
)""", {
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
{"root_2", 2},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_2 */ 2},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_1 (index 1)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_2 (index 2)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= (expr "=" term "\n")+
|
||||
expr ::= term ([-+*/] term)*
|
||||
term ::= [0-9]+
|
||||
)""", {
|
||||
{"expr", 2},
|
||||
{"expr_5", 5},
|
||||
{"expr_6", 6},
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
{"root_4", 4},
|
||||
{"term", 3},
|
||||
{"term_7", 7},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_4 */ 4},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_1 (index 1)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* expr */ 2},
|
||||
{LLAMA_GRETYPE_CHAR, '='},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* term */ 3},
|
||||
{LLAMA_GRETYPE_CHAR, '\n'},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// expr (index 2)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* term */ 3},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* expr_6 */ 6},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// term (index 3)
|
||||
{LLAMA_GRETYPE_CHAR, '0'},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, '9'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* term_7 */ 7},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_4 (index 4)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_4 */ 4},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// expr_5 (index 5)
|
||||
{LLAMA_GRETYPE_CHAR, '-'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '+'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '*'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '/'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* term */ 3},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// expr_6 (index 6)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* expr_5 */ 5},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* expr_6 */ 6},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// term_7 (index 7)
|
||||
{LLAMA_GRETYPE_CHAR, '0'},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, '9'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* term_7 */ 7},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
verify_parsing(R"""(
|
||||
root ::= (expr "=" ws term "\n")+
|
||||
expr ::= term ([-+*/] term)*
|
||||
term ::= ident | num | "(" ws expr ")" ws
|
||||
ident ::= [a-z] [a-z0-9_]* ws
|
||||
num ::= [0-9]+ ws
|
||||
ws ::= [ \t\n]*
|
||||
)""", {
|
||||
{"expr", 2},
|
||||
{"expr_6", 6},
|
||||
{"expr_7", 7},
|
||||
{"ident", 8},
|
||||
{"ident_10", 10},
|
||||
{"num", 9},
|
||||
{"num_11", 11},
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
{"root_5", 5},
|
||||
{"term", 4},
|
||||
{"ws", 3},
|
||||
{"ws_12", 12},
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_5 */ 5},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_1 (index 1)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* expr */ 2},
|
||||
{LLAMA_GRETYPE_CHAR, '='},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ws */ 3},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* term */ 4},
|
||||
{LLAMA_GRETYPE_CHAR, '\n'},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// expr (index 2)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* term */ 4},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* expr_7 */ 7},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// ws (index 3)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ws_12 */ 12},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// term (index 4)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ident */ 8},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* num */ 9},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_CHAR, '('},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ws */ 3},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* expr */ 2},
|
||||
{LLAMA_GRETYPE_CHAR, ')'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ws */ 3},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// root_5 (index 5)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_1 */ 1},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* root_5 */ 5},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// expr_6 (index 6)
|
||||
{LLAMA_GRETYPE_CHAR, '-'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '+'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '*'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '/'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* term */ 4},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// expr_7 (index 7)
|
||||
{LLAMA_GRETYPE_RULE_REF, /* expr_6 */ 6},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* expr_7 */ 7},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// ident (index 8)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, 'z'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ident_10 */ 10},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ws */ 3},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// num (index 9)
|
||||
{LLAMA_GRETYPE_CHAR, '0'},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, '9'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* num_11 */ 11},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ws */ 3},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// ident_10 (index 10)
|
||||
{LLAMA_GRETYPE_CHAR, 'a'},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, 'z'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '0'},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, '9'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '_'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ident_10 */ 10},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// num_11 (index 11)
|
||||
{LLAMA_GRETYPE_CHAR, '0'},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, '9'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* num_11 */ 11},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
// ws_12 (index 12)
|
||||
{LLAMA_GRETYPE_CHAR, ' '},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '\t'},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, '\n'},
|
||||
{LLAMA_GRETYPE_RULE_REF, /* ws_12 */ 12},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
// <[1000]> = "<think>"
|
||||
// <[1001]> = "</think>"
|
||||
verify_parsing(R"""(
|
||||
root ::= <[1000]> !<[1001]> <[1001]>
|
||||
)""", {
|
||||
{"root", 0}
|
||||
}, {
|
||||
// root (index 0)
|
||||
{LLAMA_GRETYPE_TOKEN, 1000},
|
||||
{LLAMA_GRETYPE_TOKEN_NOT, 1001},
|
||||
{LLAMA_GRETYPE_TOKEN, 1001},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
});
|
||||
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
+1623
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,513 @@
|
||||
#include "json-schema.h"
|
||||
#include "json.h"
|
||||
#include "testing.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
static common_chat_schema_document parse(const std::string & schema) {
|
||||
return common_chat_schema_from_json(common_json::parse(schema));
|
||||
}
|
||||
|
||||
// the node as T, aborting the current test when it is some other kind
|
||||
template <typename T>
|
||||
static const T & as(testing & t, const common_chat_schema * node, const char * what) {
|
||||
const T * typed = dynamic_cast<const T *>(node);
|
||||
if (!t.assert_true(std::string(what) + " has the expected kind", typed != nullptr)) {
|
||||
throw std::runtime_error(std::string(what) + " has the wrong kind");
|
||||
}
|
||||
return *typed;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static const T & root(testing & t, const common_chat_schema_document & doc) {
|
||||
return as<T>(t, doc.root.get(), "root");
|
||||
}
|
||||
|
||||
static void assert_error(testing & t, const std::string & schema, const std::string & needle) {
|
||||
try {
|
||||
parse(schema);
|
||||
t.assert_true(schema + " is rejected", false);
|
||||
} catch (const std::runtime_error & e) {
|
||||
std::string what = e.what();
|
||||
t.assert_true(schema + " -> " + what, what.find(needle) != std::string::npos);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_any(testing & t) {
|
||||
t.test("empty schema", [](testing & t) {
|
||||
auto doc = parse("{}");
|
||||
root<common_chat_schema_any>(t, doc);
|
||||
t.assert_true("no refs", doc.refs.empty());
|
||||
});
|
||||
|
||||
t.test("keywords that do not imply a type", [](testing & t) {
|
||||
auto doc = parse(R"({"description": "x", "format": "email", "additionalProperties": true})");
|
||||
root<common_chat_schema_any>(t, doc);
|
||||
});
|
||||
}
|
||||
|
||||
static void test_primitives(testing & t) {
|
||||
t.test("null, boolean, number", [](testing & t) {
|
||||
auto doc_null = parse(R"({"type": "null"})");
|
||||
root<common_chat_schema_null>(t, doc_null);
|
||||
auto doc_bool = parse(R"({"type": "boolean"})");
|
||||
root<common_chat_schema_boolean>(t, doc_bool);
|
||||
auto doc_num = parse(R"({"type": "number", "minimum": 1, "maximum": 2})");
|
||||
root<common_chat_schema_number>(t, doc_num);
|
||||
});
|
||||
}
|
||||
|
||||
static void test_integer(testing & t) {
|
||||
t.test("unbounded", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "integer"})");
|
||||
const auto & i = root<common_chat_schema_integer>(t, doc);
|
||||
t.assert_equal("minimum", INT64_MIN, i.minimum);
|
||||
t.assert_equal("maximum", INT64_MAX, i.maximum);
|
||||
});
|
||||
|
||||
t.test("inclusive bounds", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "integer", "minimum": -5, "maximum": 10})");
|
||||
const auto & i = root<common_chat_schema_integer>(t, doc);
|
||||
t.assert_equal("minimum", -5, i.minimum);
|
||||
t.assert_equal("maximum", 10, i.maximum);
|
||||
});
|
||||
|
||||
t.test("exclusive bounds are folded", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "integer", "exclusiveMinimum": 0, "exclusiveMaximum": 10})");
|
||||
const auto & i = root<common_chat_schema_integer>(t, doc);
|
||||
t.assert_equal("minimum", 1, i.minimum);
|
||||
t.assert_equal("maximum", 9, i.maximum);
|
||||
});
|
||||
|
||||
t.test("fractional bounds round inwards", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "integer", "minimum": 1.5, "exclusiveMaximum": 9.5})");
|
||||
const auto & i = root<common_chat_schema_integer>(t, doc);
|
||||
t.assert_equal("minimum", 2, i.minimum);
|
||||
t.assert_equal("maximum", 9, i.maximum);
|
||||
});
|
||||
}
|
||||
|
||||
static void test_string(testing & t) {
|
||||
t.test("defaults", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "string"})");
|
||||
const auto & s = root<common_chat_schema_string>(t, doc);
|
||||
t.assert_equal("pattern", "", s.pattern);
|
||||
t.assert_equal("format", common_chat_schema::FORMAT_NONE, s.format);
|
||||
t.assert_equal("min_length", 0, s.min_length);
|
||||
t.assert_equal("max_length", -1, s.max_length);
|
||||
});
|
||||
|
||||
t.test("all keywords are kept", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "string", "pattern": "^[a-z]+$", "format": "date", "minLength": 2, "maxLength": 8})");
|
||||
const auto & s = root<common_chat_schema_string>(t, doc);
|
||||
t.assert_equal("pattern", "^[a-z]+$", s.pattern);
|
||||
t.assert_equal("format", common_chat_schema::FORMAT_DATE, s.format);
|
||||
t.assert_equal("min_length", 2, s.min_length);
|
||||
t.assert_equal("max_length", 8, s.max_length);
|
||||
});
|
||||
|
||||
t.test("formats", [](testing & t) {
|
||||
auto expect = [&](const char * format, common_chat_schema::string_format expected) {
|
||||
auto doc = parse(std::string(R"({"type": "string", "format": ")") + format + "\"}");
|
||||
t.assert_equal(format, expected, root<common_chat_schema_string>(t, doc).format);
|
||||
};
|
||||
expect("time", common_chat_schema::FORMAT_TIME);
|
||||
expect("date-time", common_chat_schema::FORMAT_DATE_TIME);
|
||||
expect("uuid", common_chat_schema::FORMAT_UUID);
|
||||
expect("uuid5", common_chat_schema::FORMAT_UUID);
|
||||
expect("email", common_chat_schema::FORMAT_NONE);
|
||||
});
|
||||
|
||||
t.test("pattern, length and known format imply a string", [](testing & t) {
|
||||
auto doc_pattern = parse(R"({"pattern": "^a$"})");
|
||||
t.assert_equal("pattern", "^a$", root<common_chat_schema_string>(t, doc_pattern).pattern);
|
||||
auto doc_length = parse(R"({"minLength": 1, "maxLength": 3})");
|
||||
t.assert_equal("min_length", 1, root<common_chat_schema_string>(t, doc_length).min_length);
|
||||
t.assert_equal("max_length", 3, root<common_chat_schema_string>(t, doc_length).max_length);
|
||||
auto doc_format = parse(R"({"format": "uuid"})");
|
||||
t.assert_equal("format", common_chat_schema::FORMAT_UUID, root<common_chat_schema_string>(t, doc_format).format);
|
||||
});
|
||||
}
|
||||
|
||||
static void test_array(testing & t) {
|
||||
t.test("items with bounds", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "array", "items": {"type": "integer"}, "minItems": 1, "maxItems": 3})");
|
||||
const auto & a = root<common_chat_schema_array>(t, doc);
|
||||
as<common_chat_schema_integer>(t, a.items.get(), "items");
|
||||
t.assert_equal("min_items", 1, a.min_items);
|
||||
t.assert_equal("max_items", 3, a.max_items);
|
||||
});
|
||||
|
||||
t.test("no items", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "array"})");
|
||||
const auto & a = root<common_chat_schema_array>(t, doc);
|
||||
as<common_chat_schema_any>(t, a.items.get(), "items");
|
||||
t.assert_equal("min_items", 0, a.min_items);
|
||||
t.assert_equal("max_items", -1, a.max_items);
|
||||
});
|
||||
|
||||
t.test("items imply an array", [](testing & t) {
|
||||
auto doc = parse(R"({"items": {"type": "string"}})");
|
||||
const auto & a = root<common_chat_schema_array>(t, doc);
|
||||
as<common_chat_schema_string>(t, a.items.get(), "items");
|
||||
});
|
||||
}
|
||||
|
||||
static void test_tuple(testing & t) {
|
||||
t.test("prefixItems", [](testing & t) {
|
||||
auto doc = parse(R"({"prefixItems": [{"type": "string"}, {"type": "number"}]})");
|
||||
const auto & tup = root<common_chat_schema_tuple>(t, doc);
|
||||
t.assert_equal("size", (size_t) 2, tup.items.size());
|
||||
as<common_chat_schema_string>(t, tup.items[0].get(), "items[0]");
|
||||
as<common_chat_schema_number>(t, tup.items[1].get(), "items[1]");
|
||||
});
|
||||
|
||||
t.test("items as an array", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "array", "items": [{"type": "boolean"}]})");
|
||||
const auto & tup = root<common_chat_schema_tuple>(t, doc);
|
||||
t.assert_equal("size", (size_t) 1, tup.items.size());
|
||||
as<common_chat_schema_boolean>(t, tup.items[0].get(), "items[0]");
|
||||
});
|
||||
}
|
||||
|
||||
static void test_object(testing & t) {
|
||||
t.test("type alone accepts any object", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "object"})");
|
||||
const auto & o = root<common_chat_schema_object>(t, doc);
|
||||
t.assert_true("no properties", o.properties.empty());
|
||||
as<common_chat_schema_any>(t, o.additional_properties.get(), "additional_properties");
|
||||
});
|
||||
|
||||
t.test("properties", [](testing & t) {
|
||||
auto doc = parse(R"({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"b": {"type": "string"},
|
||||
"a": {"type": "integer"},
|
||||
"c": {"type": "boolean"}
|
||||
},
|
||||
"required": ["a", "c"]
|
||||
})");
|
||||
const auto & o = root<common_chat_schema_object>(t, doc);
|
||||
t.assert_equal("size", (size_t) 3, o.properties.size());
|
||||
t.assert_equal("order", "b", o.properties[0].name);
|
||||
t.assert_equal("order", "a", o.properties[1].name);
|
||||
t.assert_equal("order", "c", o.properties[2].name);
|
||||
t.assert_true("b optional", !o.properties[0].required);
|
||||
t.assert_true("a required", o.properties[1].required);
|
||||
t.assert_true("c required", o.properties[2].required);
|
||||
as<common_chat_schema_string>(t, o.properties[0].schema.get(), "b");
|
||||
as<common_chat_schema_integer>(t, o.properties[1].schema.get(), "a");
|
||||
as<common_chat_schema_boolean>(t, o.properties[2].schema.get(), "c");
|
||||
t.assert_true("closed", o.additional_properties == nullptr);
|
||||
});
|
||||
|
||||
t.test("unknown required entries are ignored", [](testing & t) {
|
||||
auto doc = parse(R"({"properties": {"a": {}}, "required": ["a", "zzz", 1]})");
|
||||
const auto & o = root<common_chat_schema_object>(t, doc);
|
||||
t.assert_equal("size", (size_t) 1, o.properties.size());
|
||||
t.assert_true("a required", o.properties[0].required);
|
||||
});
|
||||
|
||||
t.test("additionalProperties false implies an object", [](testing & t) {
|
||||
auto doc = parse(R"({"additionalProperties": false})");
|
||||
const auto & o = root<common_chat_schema_object>(t, doc);
|
||||
t.assert_true("no properties", o.properties.empty());
|
||||
t.assert_true("closed", o.additional_properties == nullptr);
|
||||
});
|
||||
|
||||
t.test("additionalProperties schema", [](testing & t) {
|
||||
auto doc = parse(R"({"properties": {"a": {}}, "additionalProperties": {"type": "integer", "minimum": 0}})");
|
||||
const auto & o = root<common_chat_schema_object>(t, doc);
|
||||
t.assert_equal("size", (size_t) 1, o.properties.size());
|
||||
const auto & v = as<common_chat_schema_integer>(t, o.additional_properties.get(), "additional_properties");
|
||||
t.assert_equal("minimum", 0, v.minimum);
|
||||
});
|
||||
|
||||
t.test("nested", [](testing & t) {
|
||||
auto doc = parse(R"({"properties": {"inner": {"properties": {"leaf": {"type": "null"}}, "required": ["leaf"]}}})");
|
||||
const auto & o = root<common_chat_schema_object>(t, doc);
|
||||
const auto & inner = as<common_chat_schema_object>(t, o.properties[0].schema.get(), "inner");
|
||||
t.assert_equal("leaf name", "leaf", inner.properties[0].name);
|
||||
t.assert_true("leaf required", inner.properties[0].required);
|
||||
as<common_chat_schema_null>(t, inner.properties[0].schema.get(), "leaf");
|
||||
});
|
||||
}
|
||||
|
||||
static void test_const_enum(testing & t) {
|
||||
t.test("const", [](testing & t) {
|
||||
auto doc = parse(R"({"const": {"a": [1, null]}})");
|
||||
t.assert_equal("value", R"({"a":[1,null]})", root<common_chat_schema_const>(t, doc).value.dump());
|
||||
});
|
||||
|
||||
t.test("enum", [](testing & t) {
|
||||
auto doc = parse(R"({"enum": ["a", 1, null, true]})");
|
||||
const auto & e = root<common_chat_schema_enum>(t, doc);
|
||||
t.assert_equal("size", (size_t) 4, e.values.size());
|
||||
t.assert_equal("values[0]", "\"a\"", e.values[0].dump());
|
||||
t.assert_equal("values[1]", "1", e.values[1].dump());
|
||||
t.assert_equal("values[2]", "null", e.values[2].dump());
|
||||
t.assert_equal("values[3]", "true", e.values[3].dump());
|
||||
});
|
||||
|
||||
t.test("const wins over enum, enum wins over type", [](testing & t) {
|
||||
auto doc_enum = parse(R"({"type": "integer", "enum": [1, 2]})");
|
||||
root<common_chat_schema_enum>(t, doc_enum);
|
||||
auto doc_const = parse(R"({"type": "string", "const": "x", "enum": ["y"]})");
|
||||
t.assert_equal("value", "\"x\"", root<common_chat_schema_const>(t, doc_const).value.dump());
|
||||
});
|
||||
}
|
||||
|
||||
static void test_any_of(testing & t) {
|
||||
t.test("anyOf and oneOf", [](testing & t) {
|
||||
auto doc_any = parse(R"({"anyOf": [{"type": "string"}, {"type": "number"}]})");
|
||||
const auto & u = root<common_chat_schema_any_of>(t, doc_any);
|
||||
t.assert_equal("size", (size_t) 2, u.children.size());
|
||||
as<common_chat_schema_string>(t, u.children[0].get(), "children[0]");
|
||||
as<common_chat_schema_number>(t, u.children[1].get(), "children[1]");
|
||||
|
||||
auto doc_one = parse(R"({"oneOf": [{"type": "null"}]})");
|
||||
const auto & o = root<common_chat_schema_any_of>(t, doc_one);
|
||||
t.assert_equal("size", (size_t) 1, o.children.size());
|
||||
as<common_chat_schema_null>(t, o.children[0].get(), "children[0]");
|
||||
});
|
||||
|
||||
t.test("oneOf wins over anyOf and type", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "string", "oneOf": [{"type": "null"}], "anyOf": [{"type": "number"}, {"type": "boolean"}]})");
|
||||
const auto & u = root<common_chat_schema_any_of>(t, doc);
|
||||
t.assert_equal("size", (size_t) 1, u.children.size());
|
||||
as<common_chat_schema_null>(t, u.children[0].get(), "children[0]");
|
||||
});
|
||||
|
||||
t.test("type array expands with sibling keywords", [](testing & t) {
|
||||
auto doc = parse(R"({"type": ["string", "null", "integer"], "minLength": 2, "minimum": 5})");
|
||||
const auto & u = root<common_chat_schema_any_of>(t, doc);
|
||||
t.assert_equal("size", (size_t) 3, u.children.size());
|
||||
t.assert_equal("min_length", 2, as<common_chat_schema_string>(t, u.children[0].get(), "children[0]").min_length);
|
||||
as<common_chat_schema_null>(t, u.children[1].get(), "children[1]");
|
||||
t.assert_equal("minimum", 5, as<common_chat_schema_integer>(t, u.children[2].get(), "children[2]").minimum);
|
||||
});
|
||||
}
|
||||
|
||||
static void test_all_of(testing & t) {
|
||||
t.test("components", [](testing & t) {
|
||||
auto doc = parse(R"({"allOf": [{"properties": {"a": {}}}, {"anyOf": [{"properties": {"b": {}}}, {"type": "null"}]}]})");
|
||||
const auto & all = root<common_chat_schema_all_of>(t, doc);
|
||||
t.assert_equal("size", (size_t) 2, all.children.size());
|
||||
as<common_chat_schema_object>(t, all.children[0].get(), "children[0]");
|
||||
as<common_chat_schema_any_of>(t, all.children[1].get(), "children[1]");
|
||||
|
||||
auto doc_typed = parse(R"({"type": "object", "allOf": [{"properties": {"a": {}}}]})");
|
||||
root<common_chat_schema_all_of>(t, doc_typed);
|
||||
});
|
||||
|
||||
t.test("properties win over allOf", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "object", "properties": {"a": {}}, "allOf": [{"properties": {"b": {}}}]})");
|
||||
t.assert_equal("size", (size_t) 1, root<common_chat_schema_object>(t, doc).properties.size());
|
||||
});
|
||||
|
||||
t.test("other types ignore allOf", [](testing & t) {
|
||||
auto doc = parse(R"({"type": "integer", "allOf": [{"minimum": 1}]})");
|
||||
root<common_chat_schema_integer>(t, doc);
|
||||
});
|
||||
}
|
||||
|
||||
static void test_ref(testing & t) {
|
||||
t.test("target is owned by the document", [](testing & t) {
|
||||
auto doc = parse(R"({"$ref": "#/$defs/t", "type": "string", "$defs": {"t": {"type": "boolean"}}})");
|
||||
const auto & r = root<common_chat_schema_ref>(t, doc);
|
||||
t.assert_equal("ref", "#/$defs/t", r.ref);
|
||||
t.assert_equal("refs", (size_t) 1, doc.refs.size());
|
||||
t.assert_true("target", r.target != nullptr && r.target == doc.refs.at("#/$defs/t").get());
|
||||
as<common_chat_schema_boolean>(t, r.target, "target");
|
||||
});
|
||||
|
||||
t.test("definitions", [](testing & t) {
|
||||
auto doc = parse(R"({"properties": {"a": {"$ref": "#/definitions/t"}}, "definitions": {"t": {"type": "number"}}})");
|
||||
const auto & o = root<common_chat_schema_object>(t, doc);
|
||||
const auto & r = as<common_chat_schema_ref>(t, o.properties[0].schema.get(), "a");
|
||||
as<common_chat_schema_number>(t, r.target, "target");
|
||||
});
|
||||
|
||||
t.test("recursive", [](testing & t) {
|
||||
auto doc = parse(R"({
|
||||
"$ref": "#/$defs/node",
|
||||
"$defs": {
|
||||
"node": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"value": {"type": "number"},
|
||||
"next": {"$ref": "#/$defs/node"}
|
||||
},
|
||||
"required": ["value"]
|
||||
}
|
||||
}
|
||||
})");
|
||||
const auto & r = root<common_chat_schema_ref>(t, doc);
|
||||
const auto & node = as<common_chat_schema_object>(t, r.target, "node");
|
||||
t.assert_equal("properties", (size_t) 2, node.properties.size());
|
||||
const auto & next = as<common_chat_schema_ref>(t, node.properties[1].schema.get(), "next");
|
||||
t.assert_true("cycle", next.target == r.target);
|
||||
t.assert_equal("refs", (size_t) 1, doc.refs.size());
|
||||
});
|
||||
|
||||
t.test("pointer through an array", [](testing & t) {
|
||||
auto doc = parse(R"({"oneOf": [{"type": "null"}, {"$ref": "#/oneOf/0"}]})");
|
||||
const auto & u = root<common_chat_schema_any_of>(t, doc);
|
||||
const auto & r = as<common_chat_schema_ref>(t, u.children[1].get(), "children[1]");
|
||||
as<common_chat_schema_null>(t, r.target, "target");
|
||||
});
|
||||
|
||||
t.test("targets survive moving the document", [](testing & t) {
|
||||
auto parsed = parse(R"({"items": {"$ref": "#/$defs/t"}, "$defs": {"t": {"type": "null"}}})");
|
||||
common_chat_schema_document doc = std::move(parsed);
|
||||
const auto & a = root<common_chat_schema_array>(t, doc);
|
||||
const auto & r = as<common_chat_schema_ref>(t, a.items.get(), "items");
|
||||
t.assert_true("target", r.target == doc.refs.at("#/$defs/t").get());
|
||||
as<common_chat_schema_null>(t, r.target, "target");
|
||||
});
|
||||
}
|
||||
|
||||
static void test_may_be_string(testing & t) {
|
||||
auto check = [](testing & t, const std::string & schema, bool expected) {
|
||||
t.assert_equal(schema, expected, parse(schema).root->may_be_string());
|
||||
};
|
||||
|
||||
t.test("leaves", [&](testing & t) {
|
||||
check(t, R"({"type": "string"})", true);
|
||||
check(t, R"({"type": "integer"})", false);
|
||||
check(t, R"({"minLength": 1})", true);
|
||||
check(t, R"({"pattern": "^[a-z]+$"})", true);
|
||||
check(t, R"({"const": "hello"})", true);
|
||||
check(t, R"({"const": 123})", false);
|
||||
check(t, R"({"enum": [1, "a", null]})", true);
|
||||
check(t, R"({"enum": [1, 2, 3]})", false);
|
||||
});
|
||||
|
||||
t.test("composites", [&](testing & t) {
|
||||
check(t, R"({"type": ["integer", "string"]})", true);
|
||||
check(t, R"({"anyOf": [{"type": "integer"}, {"type": "boolean"}]})", false);
|
||||
check(t, R"({"allOf": [{"type": "string"}, {"minLength": 1}]})", true);
|
||||
check(t, R"({"allOf": [{"type": "string"}, {"type": "integer"}]})", false);
|
||||
check(t, R"({"allOf": [{"minLength": 1}, {"maxLength": 2}]})", true);
|
||||
});
|
||||
|
||||
t.test("ref", [&](testing & t) {
|
||||
check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"anyOf": [{"$ref": "#/$defs/n"}, {"type": "string"}]}}})", true);
|
||||
check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"$ref": "#/$defs/n"}}})", false);
|
||||
check(t, R"({"anyOf": [{"$ref": "#/$defs/a"}, {"$ref": "#/$defs/b"}], "$defs": {"a": {"allOf": [{"$ref": "#/$defs/b"}, {"type": "integer"}]}, "b": {"type": "string"}}})", true);
|
||||
});
|
||||
}
|
||||
|
||||
// e.g. {number, integer}, in type order
|
||||
static std::string dump(const common_chat_schema::type_set & types) {
|
||||
static const common_chat_schema::value_type order[] = { common_chat_schema::TYPE_NULL, common_chat_schema::TYPE_BOOLEAN, common_chat_schema::TYPE_NUMBER,
|
||||
common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_ARRAY,
|
||||
common_chat_schema::TYPE_OBJECT };
|
||||
std::string out;
|
||||
for (auto type : order) {
|
||||
if (types.has(type)) {
|
||||
out += (out.empty() ? "" : ", ") + std::string(common_chat_schema::type_name(type));
|
||||
}
|
||||
}
|
||||
return "{" + out + "}";
|
||||
}
|
||||
|
||||
static void test_value_types(testing & t) {
|
||||
auto check = [](testing & t, const std::string & schema, const common_chat_schema::type_set & expected) {
|
||||
t.assert_equal(schema, dump(expected), dump(parse(schema).root->value_types()));
|
||||
};
|
||||
|
||||
t.test("leaves", [&](testing & t) {
|
||||
check(t, R"({"type": "string"})", { common_chat_schema::TYPE_STRING });
|
||||
check(t, R"({"type": "number"})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER });
|
||||
check(t, R"({"description": "x"})", common_chat_schema::type_set::all());
|
||||
check(t, R"({"properties": {"a": {"type": "string"}}})", { common_chat_schema::TYPE_OBJECT });
|
||||
check(t, R"({"items": {"type": "string"}})", { common_chat_schema::TYPE_ARRAY });
|
||||
check(t, R"({"const": 1.5})", { common_chat_schema::TYPE_NUMBER });
|
||||
check(t, R"({"enum": [1, "a", null]})", { common_chat_schema::TYPE_INTEGER, common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL });
|
||||
});
|
||||
|
||||
t.test("any_of is the union, all_of is the intersection", [&](testing & t) {
|
||||
check(t, R"({"type": ["string", "null"]})", { common_chat_schema::TYPE_STRING, common_chat_schema::TYPE_NULL });
|
||||
check(t, R"({"allOf": [{"type": ["string", "number"]}, {"type": ["number", "object"]}]})", { common_chat_schema::TYPE_NUMBER, common_chat_schema::TYPE_INTEGER });
|
||||
check(t, R"({"allOf": [{"type": "string"}, {"type": "integer"}]})", {});
|
||||
});
|
||||
|
||||
t.test("ref", [&](testing & t) {
|
||||
check(t, R"({"$ref": "#/$defs/n", "$defs": {"n": {"anyOf": [{"$ref": "#/$defs/n"}, {"type": "string"}]}}})",
|
||||
{ common_chat_schema::TYPE_STRING });
|
||||
});
|
||||
}
|
||||
|
||||
static void test_errors(testing & t) {
|
||||
t.test("not a schema", [](testing & t) {
|
||||
assert_error(t, R"([])", "#: schema must be an object");
|
||||
});
|
||||
|
||||
t.test("type", [](testing & t) {
|
||||
assert_error(t, R"({"type": 5})", "#: type must be a string or an array of strings");
|
||||
assert_error(t, R"({"type": []})", "#: type must not be empty");
|
||||
assert_error(t, R"({"type": ["string", "bad"]})", "#/type/1: unrecognized type bad");
|
||||
});
|
||||
|
||||
t.test("ref", [](testing & t) {
|
||||
assert_error(t, R"({"$ref": 5})", "#: $ref must be a string");
|
||||
assert_error(t, R"({"$ref": "https://example.com/x.json"})", "#: unsupported $ref https://example.com/x.json");
|
||||
assert_error(t, R"({"$ref": ""})", "#: unsupported $ref ,");
|
||||
assert_error(t, R"({"$ref": "#"})", "#: unsupported $ref #,");
|
||||
assert_error(t, R"({"$defs": {}, "$ref": "#/$defs/missing"})", "#: cannot resolve $ref #/$defs/missing, missing not found");
|
||||
assert_error(t, R"({"oneOf": [{}], "$ref": "#/oneOf/1"})", "#: cannot resolve $ref #/oneOf/1, 1 is out of range");
|
||||
assert_error(t, R"({"$defs": {"a": {"$ref": "#/$defs/a/nope"}}, "$ref": "#/$defs/a"})", "#/$defs/a: cannot resolve $ref #/$defs/a/nope, nope not found");
|
||||
});
|
||||
|
||||
t.test("alternatives", [](testing & t) {
|
||||
assert_error(t, R"({"oneOf": []})", "#/oneOf: must not be empty");
|
||||
assert_error(t, R"({"anyOf": {}})", "#/anyOf: must be an array of schemas");
|
||||
assert_error(t, R"({"anyOf": [{"type": "string"}, {"items": {"type": "x"}}]})", "#/anyOf/1/items: unrecognized type x");
|
||||
});
|
||||
|
||||
t.test("keywords", [](testing & t) {
|
||||
assert_error(t, R"({"enum": []})", "#: enum must be a non-empty array");
|
||||
assert_error(t, R"({"type": "string", "pattern": 5})", "#: pattern must be a string");
|
||||
assert_error(t, R"({"type": "string", "minLength": -1})", "#: minLength must be a non-negative integer");
|
||||
assert_error(t, R"({"type": "integer", "minimum": "1"})", "#: minimum must be a number");
|
||||
assert_error(t, R"({"type": "array", "maxItems": 1.5})", "#: maxItems must be a non-negative integer");
|
||||
assert_error(t, R"({"properties": []})", "#: properties must be an object");
|
||||
assert_error(t, R"({"properties": {"a": {"type": "nope"}}})", "#/properties/a: unrecognized type nope");
|
||||
assert_error(t, R"({"additionalProperties": null})", "#: additionalProperties must be a boolean or a schema");
|
||||
});
|
||||
}
|
||||
|
||||
int main(int argc, char * argv[]) {
|
||||
testing t(std::cout);
|
||||
if (argc >= 2) {
|
||||
t.set_filter(argv[1]);
|
||||
}
|
||||
|
||||
const char * verbose = getenv("LLAMA_TEST_VERBOSE");
|
||||
if (verbose) {
|
||||
t.verbose = std::string(verbose) == "1";
|
||||
}
|
||||
|
||||
t.test("any", test_any);
|
||||
t.test("primitives", test_primitives);
|
||||
t.test("integer", test_integer);
|
||||
t.test("string", test_string);
|
||||
t.test("array", test_array);
|
||||
t.test("tuple", test_tuple);
|
||||
t.test("object", test_object);
|
||||
t.test("const and enum", test_const_enum);
|
||||
t.test("any_of", test_any_of);
|
||||
t.test("all_of", test_all_of);
|
||||
t.test("ref", test_ref);
|
||||
t.test("may_be_string", test_may_be_string);
|
||||
t.test("value_types", test_value_types);
|
||||
t.test("errors", test_errors);
|
||||
|
||||
return t.summary();
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,550 @@
|
||||
//
|
||||
// test-llama-context - exercises the state handling of llama_context
|
||||
//
|
||||
|
||||
#include "llama-context/test-case.h"
|
||||
#include "llama-context/test-log.h"
|
||||
#include "llama-context/test-runner.h"
|
||||
#include "llama-context/test-util.h"
|
||||
|
||||
#include "arg.h"
|
||||
#include "common.h"
|
||||
#include "log.h"
|
||||
#include "llama-cpp.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <clocale>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <mutex>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
// the state file of a model, named after it so that models running in parallel do not collide
|
||||
static std::string test_state_path(const std::string & model_name) {
|
||||
return "dump_state." + model_name + ".bin";
|
||||
}
|
||||
|
||||
// --name VALUE or --name=VALUE: 0 when this argument is not the option, 1 when it took a value,
|
||||
// -1 when the value is missing or empty
|
||||
static int test_opt(char ** argv, int & i, int argc, const char * name, std::string & value) {
|
||||
const size_t len = std::strlen(name);
|
||||
|
||||
if (std::strncmp(argv[i], name, len) != 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (argv[i][len] == '=') {
|
||||
value = argv[i] + len + 1;
|
||||
return value.empty() ? -1 : 1;
|
||||
}
|
||||
|
||||
if (argv[i][len] != '\0') {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (i + 1 >= argc) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
value = argv[++i];
|
||||
return value.empty() ? -1 : 1;
|
||||
}
|
||||
|
||||
// tokenize the prompt, or generate random tokens for models without a tokenizer
|
||||
static llama_tokens prepare_tokens(llama_model * model, const common_params & params) {
|
||||
llama_tokens tokens;
|
||||
|
||||
if (params.prompt.empty()) {
|
||||
// the prompt must not exceed n_batch and n_ctx of cases
|
||||
constexpr int n_prompt_max = 100;
|
||||
const int n_prompt = std::min(params.n_batch, n_prompt_max);
|
||||
|
||||
// this path is useful for model files that do not have a tokenizer
|
||||
TLOG_INF("%s: no prompt provided, generating %d random tokens\n", __func__, n_prompt);
|
||||
|
||||
const auto * vocab = llama_model_get_vocab(model);
|
||||
const auto n_vocab = llama_vocab_n_tokens(vocab);
|
||||
|
||||
std::mt19937 rng(params.sampling.seed);
|
||||
std::uniform_int_distribution<llama_token> dist(0, n_vocab - 1);
|
||||
for (int i = 0; i < n_prompt; i++) {
|
||||
tokens.push_back(dist(rng));
|
||||
}
|
||||
} else {
|
||||
TLOG_INF("%s: tokenizing prompt '%s'\n", __func__, params.prompt.c_str());
|
||||
|
||||
auto ctx = llama_context_ptr{llama_init_from_model(model, common_context_params_to_llama(params))};
|
||||
tokens = common_tokenize(ctx.get(), params.prompt, true);
|
||||
}
|
||||
|
||||
TLOG_INF("%s: the input prompt is %d tokens\n", __func__, (int) tokens.size());
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
static std::vector<std::string> collect_models(const std::string & models_dir) {
|
||||
std::vector<std::string> models;
|
||||
|
||||
for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {
|
||||
if (entry.is_regular_file() && entry.path().extension() == ".gguf") {
|
||||
models.push_back(entry.path().string());
|
||||
}
|
||||
}
|
||||
std::sort(models.begin(), models.end());
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
// one row of the result table
|
||||
struct model_result {
|
||||
std::string name;
|
||||
std::string log_path;
|
||||
std::vector<case_run> runs;
|
||||
bool failed = false;
|
||||
};
|
||||
|
||||
// run the selected cases for one model and return one verdict per case
|
||||
//
|
||||
// safe to call from a worker thread: the log capture and the state file are either per thread or
|
||||
// named after the model
|
||||
//
|
||||
// report prints the verdict of every case as it finishes, capture_log keeps the log of the run in
|
||||
// a file - table mode always captures, so that a failing model can be dumped after the table;
|
||||
// single model mode only does when --logs asks for it
|
||||
static model_result run_model(const std::string & model_path,
|
||||
const common_params & params,
|
||||
const std::vector<const ctx_test *> & cases,
|
||||
const std::string & logs_dir,
|
||||
bool report,
|
||||
bool capture_log) {
|
||||
model_result res;
|
||||
|
||||
res.name = std::filesystem::path(model_path).filename().string();
|
||||
res.log_path = test_log_path(logs_dir, res.name);
|
||||
|
||||
if (capture_log) {
|
||||
// capture the model from the moment it is loaded, so that even a load failure has a log
|
||||
test_log_capture(res.log_path);
|
||||
}
|
||||
|
||||
struct common_params model_params = params;
|
||||
model_params.model.path = model_path;
|
||||
|
||||
// a model that cannot be loaded is a failure of every case, not a skip - the cases never
|
||||
// run, so their verdicts are recorded here
|
||||
auto llama_init = common_init_from_params(model_params, true);
|
||||
if (llama_init->model() == nullptr) {
|
||||
TLOG_ERR("%s: failed to init model '%s'\n", __func__, model_path.c_str());
|
||||
|
||||
for (const auto * tc : cases) {
|
||||
case_run run;
|
||||
run.name = tc->name();
|
||||
run.status = test_status::FAIL;
|
||||
run.reason = "the model failed to load";
|
||||
res.runs.push_back(std::move(run));
|
||||
}
|
||||
} else {
|
||||
GGML_ASSERT(llama_init->context() == nullptr);
|
||||
|
||||
model_run mr;
|
||||
mr.model = llama_init->model();
|
||||
mr.name = res.name;
|
||||
// the cases share a working directory, so state files are named after the model
|
||||
mr.params = std::move(model_params);
|
||||
mr.params.out_file = test_state_path(res.name);
|
||||
mr.tokens = prepare_tokens(mr.model, mr.params);
|
||||
|
||||
res.runs = run_cases(mr, cases, report);
|
||||
}
|
||||
|
||||
if (capture_log) {
|
||||
test_log_capture("");
|
||||
}
|
||||
|
||||
std::remove(test_state_path(res.name).c_str()); // the cases leave the session file behind
|
||||
|
||||
res.failed = std::any_of(res.runs.begin(), res.runs.end(),
|
||||
[](const case_run & run) { return run.status == test_status::FAIL; });
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
static void print_usage(int /* argc */, char ** argv) {
|
||||
LOG("\nexample usage:\n");
|
||||
LOG("\n %s -m your_model.gguf\n", argv[0]);
|
||||
LOG("\n %s --models tests/test-models\n", argv[0]);
|
||||
LOG("\n %s --models tests/test-models --test 'rollback|replay'\n", argv[0]);
|
||||
LOG("\n %s --models tests/test-models -j 8\n", argv[0]);
|
||||
LOG("\n %s -m your_model.gguf -lv 5\n", argv[0]);
|
||||
LOG("\nspecial flags:\n");
|
||||
LOG("\n --models DIR run every .gguf in DIR and print a result table\n");
|
||||
LOG("\n --test REGEX run only the cases whose name matches REGEX\n");
|
||||
LOG("\n --list list the registered cases and exit\n");
|
||||
LOG("\n -j N, --threads N run N models at a time in --models mode (default: min(nproc, 4))\n");
|
||||
LOG("\n --logs DIR keep the log of each model in DIR instead of discarding it\n");
|
||||
LOG("\n --logs-on-fail always after the table, print the log of every failed model (default)\n");
|
||||
LOG("\n --logs-on-fail never never print the logs of the failed models\n");
|
||||
LOG("\n");
|
||||
}
|
||||
|
||||
static void print_cases() {
|
||||
printf("\ncases:\n");
|
||||
|
||||
for (const auto & tc : test_cases()) {
|
||||
printf(" %-11s %-5s %s\n", tc->name(), tc->group() == TEST_GROUP_RS ? "[rs]" : "[kv]", tc->description());
|
||||
}
|
||||
|
||||
printf("\nthe [rs] cases are also run against a cache pre-filled with a known pattern\n\n");
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
|
||||
common_params params;
|
||||
params.prompt = "";
|
||||
params.out_file = "dump_state.bin";
|
||||
params.sampling.seed = 1234;
|
||||
|
||||
common_init();
|
||||
|
||||
// extract our own options before handing the rest to the common parser
|
||||
std::string models_dir;
|
||||
std::string logs_dir;
|
||||
std::string test_filter;
|
||||
|
||||
bool list_cases = false;
|
||||
bool dump_logs_on_fail = true;
|
||||
|
||||
int parallel_workers = 0; // 0 = auto, same flag as in test-backend-ops
|
||||
|
||||
auto parse_parallel_workers = [&](const char * flag, const char * value) {
|
||||
try {
|
||||
parallel_workers = std::stoi(value);
|
||||
} catch (const std::exception &) {
|
||||
fprintf(stderr, "%s: expected a number, got '%s'\n", flag, value);
|
||||
return false;
|
||||
}
|
||||
if (parallel_workers < 1) {
|
||||
fprintf(stderr, "%s: expected a positive number, got %d\n", flag, parallel_workers);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
std::vector<char *> filtered_argv;
|
||||
filtered_argv.push_back(argv[0]);
|
||||
for (int i = 1; i < argc; i++) {
|
||||
std::string value;
|
||||
|
||||
const int res_models = test_opt(argv, i, argc, "--models", value);
|
||||
if (res_models != 0) {
|
||||
if (res_models < 0) {
|
||||
fprintf(stderr, "--models requires a directory argument\n");
|
||||
return 1;
|
||||
}
|
||||
models_dir = value;
|
||||
continue;
|
||||
}
|
||||
|
||||
const int res_logs = test_opt(argv, i, argc, "--logs", value);
|
||||
if (res_logs != 0) {
|
||||
if (res_logs < 0) {
|
||||
fprintf(stderr, "--logs requires a directory argument\n");
|
||||
return 1;
|
||||
}
|
||||
logs_dir = value;
|
||||
continue;
|
||||
}
|
||||
|
||||
const int res_test = test_opt(argv, i, argc, "--test", value);
|
||||
if (res_test != 0) {
|
||||
if (res_test < 0) {
|
||||
fprintf(stderr, "--test requires a regular expression argument\n");
|
||||
return 1;
|
||||
}
|
||||
test_filter = value;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (test_opt(argv, i, argc, "--logs-on-fail", value) != 0) {
|
||||
if (value == "always") {
|
||||
dump_logs_on_fail = true;
|
||||
} else if (value == "never") {
|
||||
dump_logs_on_fail = false;
|
||||
} else {
|
||||
fprintf(stderr, "--logs-on-fail: expected 'always' or 'never', got '%s'\n", value.c_str());
|
||||
return 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// -j is the flag of test-backend-ops, --threads is its long form
|
||||
if (std::strcmp(argv[i], "-j") == 0) {
|
||||
if (++i >= argc) {
|
||||
fprintf(stderr, "-j requires a number argument\n");
|
||||
return 1;
|
||||
}
|
||||
if (!parse_parallel_workers("-j", argv[i])) {
|
||||
return 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const int res_threads = test_opt(argv, i, argc, "--threads", value);
|
||||
if (res_threads != 0) {
|
||||
if (res_threads < 0) {
|
||||
fprintf(stderr, "--threads requires a number argument\n");
|
||||
return 1;
|
||||
}
|
||||
if (!parse_parallel_workers("--threads", value.c_str())) {
|
||||
return 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (std::strcmp(argv[i], "--list") == 0) {
|
||||
list_cases = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
filtered_argv.push_back(argv[i]);
|
||||
}
|
||||
filtered_argv.push_back(nullptr);
|
||||
const int fargc = (int) filtered_argv.size() - 1;
|
||||
|
||||
if (list_cases) {
|
||||
print_cases();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// in --models mode there is no single model; set a placeholder so the common parser's
|
||||
// "--model is required" check passes (each model is set individually inside the loop)
|
||||
if (!models_dir.empty()) {
|
||||
params.model.path = models_dir;
|
||||
}
|
||||
|
||||
if (!common_params_parse(fargc, filtered_argv.data(), params, LLAMA_EXAMPLE_COMMON, print_usage)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// one model per thread: fitting the params to the free memory would swap in a log callback
|
||||
// that points to the stack of the thread doing the fit, which is only safe while no other
|
||||
// thread is inside llama - and the models of this test are far too small to need it
|
||||
params.fit_params = false;
|
||||
|
||||
const std::vector<const ctx_test *> cases = select_cases(test_filter);
|
||||
if (cases.empty()) {
|
||||
fprintf(stderr, "--test '%s' selected no cases, see --list\n", test_filter.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.n_predict < 0) {
|
||||
params.n_predict = 16;
|
||||
}
|
||||
|
||||
// capture the logs of each model to a file, so that they can be dumped when a model fails:
|
||||
// the table stays clean, but the details of a failure stay close at hand
|
||||
if (!logs_dir.empty() && !std::filesystem::create_directories(logs_dir)) {
|
||||
fprintf(stderr, "failed to create the log directory '%s'\n", logs_dir.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
// route the log of every thread to its own capture - without a capture of its own, a thread
|
||||
// writes to the console as usual
|
||||
llama_log_set(test_log_callback, nullptr);
|
||||
|
||||
llama_backend_init();
|
||||
|
||||
// run the full suite for a single model
|
||||
if (models_dir.empty()) {
|
||||
const int32_t n_threads = params.cpuparams.n_threads;
|
||||
LOG("threads: %d\n", n_threads);
|
||||
|
||||
const model_result res = run_model(params.model.path, params, cases, logs_dir,
|
||||
true /* report */, !logs_dir.empty());
|
||||
|
||||
size_t n_pass = 0;
|
||||
size_t n_skip = 0;
|
||||
size_t n_fail = 0;
|
||||
|
||||
for (const auto & run : res.runs) {
|
||||
switch (run.status) {
|
||||
case test_status::PASS: n_pass++; break;
|
||||
case test_status::SKIP: n_skip++; break;
|
||||
case test_status::FAIL: n_fail++; break;
|
||||
}
|
||||
}
|
||||
|
||||
fprintf(stderr, "\ncases : %zu\n", res.runs.size());
|
||||
fprintf(stderr, "passed : %zu\n", n_pass);
|
||||
fprintf(stderr, "skipped : %zu\n", n_skip);
|
||||
fprintf(stderr, "failed : %zu\n", n_fail);
|
||||
|
||||
return n_fail == 0 ? 0 : 1;
|
||||
}
|
||||
|
||||
// table mode: run the suite for every model in the directory
|
||||
if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {
|
||||
TLOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
const auto models = collect_models(models_dir);
|
||||
if (models.empty()) {
|
||||
TLOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
const unsigned n_hw_2 = std::thread::hardware_concurrency() / 2;
|
||||
const size_t n_auto = std::max<size_t>(1, std::min<size_t>(n_hw_2, 4));
|
||||
const size_t n_pool = parallel_workers > 0
|
||||
? std::min<size_t>((size_t) parallel_workers, models.size())
|
||||
: n_auto;
|
||||
|
||||
const int32_t n_threads = std::max<int32_t>(1, n_hw_2 / n_pool);
|
||||
|
||||
// a column is as wide as its case name, but never narrower than the widest verdict
|
||||
const size_t verdict_width = std::max({ test_status_width(test_status::PASS),
|
||||
test_status_width(test_status::FAIL),
|
||||
test_status_width(test_status::SKIP) });
|
||||
|
||||
auto col_width = [&](const char * name) { return (int) std::max(strlen(name), verdict_width); };
|
||||
|
||||
size_t name_width = 5; // "Model"
|
||||
for (const auto & model_path : models) {
|
||||
name_width = std::max(name_width, std::filesystem::path(model_path).filename().string().size());
|
||||
}
|
||||
|
||||
// silence everything but the table itself (LOG has verbosity LOG_LEVEL_OUTPUT = 0)
|
||||
common_log_set_verbosity_thold(0);
|
||||
|
||||
if (!test_filter.empty()) {
|
||||
LOG("filter: %s\n", test_filter.c_str());
|
||||
}
|
||||
LOG("models in parallel: %zu, threads per model: %d\n", n_pool, n_threads);
|
||||
common_log_flush(common_log_main());
|
||||
|
||||
LOG("%-*s", (int) name_width, "Model");
|
||||
for (const auto & tc : cases) {
|
||||
LOG(" %-*s", col_width(tc->name()), tc->name());
|
||||
}
|
||||
LOG("\n");
|
||||
common_log_flush(common_log_main());
|
||||
|
||||
std::vector<size_t> n_pass(cases.size(), 0);
|
||||
std::vector<size_t> n_skip(cases.size(), 0);
|
||||
std::vector<size_t> n_fail(cases.size(), 0);
|
||||
|
||||
size_t n_model_fail = 0;
|
||||
|
||||
// the models are independent of each other, so they run on several threads - but the table
|
||||
// is printed in model order, with the log of a failed model right after its row, exactly as
|
||||
// if the models had run one by one: a worker stores its result and then prints every row
|
||||
// that became printable in order, under the print lock
|
||||
std::vector<model_result> results(models.size());
|
||||
std::vector<char> ready(models.size(), 0);
|
||||
|
||||
std::mutex pool_mutex;
|
||||
size_t next_print = 0;
|
||||
|
||||
auto print_ready = [&]() {
|
||||
std::lock_guard<std::mutex> lock(pool_mutex);
|
||||
|
||||
while (next_print < results.size() && ready[next_print]) {
|
||||
const model_result & res = results[next_print];
|
||||
|
||||
LOG("%-*s", (int) name_width, res.name.c_str());
|
||||
|
||||
for (size_t i = 0; i < res.runs.size(); i++) {
|
||||
const int pad = (int) (col_width(cases[i]->name()) - test_status_width(res.runs[i].status));
|
||||
|
||||
LOG(" %s%*s", test_status_str(res.runs[i].status), pad, "");
|
||||
|
||||
switch (res.runs[i].status) {
|
||||
case test_status::PASS: n_pass[i]++; break;
|
||||
case test_status::FAIL: n_fail[i]++; break;
|
||||
case test_status::SKIP: n_skip[i]++; break;
|
||||
}
|
||||
}
|
||||
LOG("\n");
|
||||
common_log_flush(common_log_main());
|
||||
|
||||
// a log is discarded when it was printed in full and nobody asked to keep it
|
||||
bool keep_log = !logs_dir.empty();
|
||||
|
||||
if (res.failed) {
|
||||
n_model_fail++;
|
||||
|
||||
// dump the log of a failed model once its row is printed - interleaving it with
|
||||
// the rows would make the table unreadable
|
||||
if (dump_logs_on_fail) {
|
||||
LOG("\n=== logs: %s (%s) ===\n", res.name.c_str(), res.log_path.c_str());
|
||||
common_log_flush(common_log_main());
|
||||
|
||||
// a dump that had to be cut short leaves the file behind, the table cannot
|
||||
// show the whole log
|
||||
keep_log = keep_log || !test_log_dump(res.log_path);
|
||||
}
|
||||
}
|
||||
|
||||
if (!keep_log) {
|
||||
std::remove(res.log_path.c_str());
|
||||
}
|
||||
|
||||
++next_print;
|
||||
}
|
||||
};
|
||||
|
||||
std::atomic<size_t> next_model(0);
|
||||
|
||||
auto worker = [&]() {
|
||||
// no verdict lines here - the table carries the verdicts, the reasons are logged to the
|
||||
// file of the model
|
||||
for (size_t i = next_model.fetch_add(1); i < results.size(); i = next_model.fetch_add(1)) {
|
||||
results[i] = run_model(models[i], params, cases, logs_dir,
|
||||
false /* report */, true /* capture_log */);
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(pool_mutex);
|
||||
ready[i] = 1;
|
||||
}
|
||||
|
||||
print_ready();
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<std::thread> pool;
|
||||
pool.reserve(n_pool > 0 ? n_pool - 1 : 0);
|
||||
|
||||
for (size_t i = 1; i < n_pool; i++) {
|
||||
pool.emplace_back(worker);
|
||||
}
|
||||
|
||||
worker(); // the main thread runs models like any other worker
|
||||
|
||||
for (auto & t : pool) {
|
||||
t.join();
|
||||
}
|
||||
|
||||
print_ready(); // the last worker can have stored its result while the others were joining
|
||||
|
||||
common_log_set_verbosity_thold(LOG_DEFAULT_LLAMA);
|
||||
common_log_flush(common_log_main());
|
||||
|
||||
for (size_t i = 0; i < cases.size(); i++) {
|
||||
if (n_fail[i] == 0 && n_skip[i] == 0) {
|
||||
continue;
|
||||
}
|
||||
LOG_INF("%s: %-13s %zu passed, %zu skipped, %zu failed (of %zu)\n",
|
||||
__func__, cases[i]->name(), n_pass[i], n_skip[i], n_fail[i], models.size());
|
||||
}
|
||||
LOG_INF("%s: models: %zu passed, %zu failed (of %zu)\n", __func__, models.size() - n_model_fail, n_model_fail, models.size());
|
||||
|
||||
return n_model_fail == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
#ifdef NDEBUG
|
||||
#undef NDEBUG
|
||||
#endif
|
||||
|
||||
#include "llama.h"
|
||||
|
||||
#include "../src/llama-grammar.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <stdexcept>
|
||||
|
||||
int main()
|
||||
{
|
||||
llama_grammar_parser parsed_grammar;
|
||||
|
||||
std::vector<std::pair<std::string, uint32_t>> expected = {
|
||||
{"expr", 2},
|
||||
{"expr_6", 6},
|
||||
{"expr_7", 7},
|
||||
{"ident", 8},
|
||||
{"ident_10", 10},
|
||||
{"num", 9},
|
||||
{"num_11", 11},
|
||||
{"root", 0},
|
||||
{"root_1", 1},
|
||||
{"root_5", 5},
|
||||
{"term", 4},
|
||||
{"ws", 3},
|
||||
{"ws_12", 12},
|
||||
};
|
||||
|
||||
std::vector<std::vector<llama_grammar_element>> expected_rules = {
|
||||
{{LLAMA_GRETYPE_RULE_REF, 5}, {LLAMA_GRETYPE_END, 0}},
|
||||
{
|
||||
{LLAMA_GRETYPE_RULE_REF, 2},
|
||||
{LLAMA_GRETYPE_CHAR, 61},
|
||||
{LLAMA_GRETYPE_RULE_REF, 3},
|
||||
{LLAMA_GRETYPE_RULE_REF, 4},
|
||||
{LLAMA_GRETYPE_CHAR, 10},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
},
|
||||
{{LLAMA_GRETYPE_RULE_REF, 4}, {LLAMA_GRETYPE_RULE_REF, 7}, {LLAMA_GRETYPE_END, 0}},
|
||||
{{LLAMA_GRETYPE_RULE_REF, 12}, {LLAMA_GRETYPE_END, 0}},
|
||||
{
|
||||
{LLAMA_GRETYPE_RULE_REF, 8},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_RULE_REF, 9},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_CHAR, 40},
|
||||
{LLAMA_GRETYPE_RULE_REF, 3},
|
||||
{LLAMA_GRETYPE_RULE_REF, 2},
|
||||
{LLAMA_GRETYPE_CHAR, 41},
|
||||
{LLAMA_GRETYPE_RULE_REF, 3},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
},
|
||||
{{LLAMA_GRETYPE_RULE_REF, 1}, {LLAMA_GRETYPE_RULE_REF, 5}, {LLAMA_GRETYPE_ALT, 0}, {LLAMA_GRETYPE_RULE_REF, 1}, {LLAMA_GRETYPE_END, 0}},
|
||||
{
|
||||
{LLAMA_GRETYPE_CHAR, 45},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, 43},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, 42},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, 47},
|
||||
{LLAMA_GRETYPE_RULE_REF, 4},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
},
|
||||
{{LLAMA_GRETYPE_RULE_REF, 6}, {LLAMA_GRETYPE_RULE_REF, 7}, {LLAMA_GRETYPE_ALT, 0}, {LLAMA_GRETYPE_END, 0}},
|
||||
{
|
||||
{LLAMA_GRETYPE_CHAR, 97},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, 122},
|
||||
{LLAMA_GRETYPE_RULE_REF, 10},
|
||||
{LLAMA_GRETYPE_RULE_REF, 3},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
},
|
||||
{{LLAMA_GRETYPE_RULE_REF, 11}, {LLAMA_GRETYPE_RULE_REF, 3}, {LLAMA_GRETYPE_END, 0}},
|
||||
{
|
||||
{LLAMA_GRETYPE_CHAR, 97},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, 122},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, 48},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, 57},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, 95},
|
||||
{LLAMA_GRETYPE_RULE_REF, 10},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
},
|
||||
{
|
||||
{LLAMA_GRETYPE_CHAR, 48},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, 57},
|
||||
{LLAMA_GRETYPE_RULE_REF, 11},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_CHAR, 48},
|
||||
{LLAMA_GRETYPE_CHAR_RNG_UPPER, 57},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
},
|
||||
{
|
||||
{LLAMA_GRETYPE_CHAR, 32},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, 9},
|
||||
{LLAMA_GRETYPE_CHAR_ALT, 10},
|
||||
{LLAMA_GRETYPE_RULE_REF, 12},
|
||||
{LLAMA_GRETYPE_ALT, 0},
|
||||
{LLAMA_GRETYPE_END, 0},
|
||||
},
|
||||
};
|
||||
|
||||
for (auto pair : expected)
|
||||
{
|
||||
parsed_grammar.symbol_ids[pair.first] = pair.second;
|
||||
}
|
||||
|
||||
for (auto rule : expected_rules)
|
||||
{
|
||||
parsed_grammar.rules.emplace_back();
|
||||
for (auto element : rule)
|
||||
{
|
||||
parsed_grammar.rules.back().push_back(element);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<const llama_grammar_element *> grammar_rules(parsed_grammar.c_rules());
|
||||
|
||||
llama_grammar * grammar = llama_grammar_init_impl(nullptr, grammar_rules.data(), grammar_rules.size(), parsed_grammar.symbol_ids.at("root"));
|
||||
if (grammar == nullptr) {
|
||||
throw std::runtime_error("Failed to initialize llama_grammar");
|
||||
}
|
||||
|
||||
std::vector<std::vector<llama_grammar_element>> expected_stacks = {
|
||||
{
|
||||
{LLAMA_GRETYPE_CHAR, 61},
|
||||
{LLAMA_GRETYPE_RULE_REF, 7},
|
||||
{LLAMA_GRETYPE_CHAR, 40},
|
||||
},
|
||||
{
|
||||
{LLAMA_GRETYPE_CHAR, 61},
|
||||
{LLAMA_GRETYPE_RULE_REF, 7},
|
||||
{LLAMA_GRETYPE_RULE_REF, 3},
|
||||
{LLAMA_GRETYPE_CHAR, 48},
|
||||
},
|
||||
{
|
||||
{LLAMA_GRETYPE_CHAR, 61},
|
||||
{LLAMA_GRETYPE_RULE_REF, 7},
|
||||
{LLAMA_GRETYPE_RULE_REF, 3},
|
||||
{LLAMA_GRETYPE_CHAR, 48},
|
||||
},
|
||||
{
|
||||
{LLAMA_GRETYPE_CHAR, 61},
|
||||
{LLAMA_GRETYPE_RULE_REF, 7},
|
||||
{LLAMA_GRETYPE_CHAR, 97},
|
||||
},
|
||||
{
|
||||
{LLAMA_GRETYPE_RULE_REF, 5},
|
||||
{LLAMA_GRETYPE_CHAR, 61},
|
||||
{LLAMA_GRETYPE_RULE_REF, 7},
|
||||
{LLAMA_GRETYPE_CHAR, 40},
|
||||
},
|
||||
{
|
||||
{LLAMA_GRETYPE_RULE_REF, 5},
|
||||
{LLAMA_GRETYPE_CHAR, 61},
|
||||
{LLAMA_GRETYPE_RULE_REF, 7},
|
||||
{LLAMA_GRETYPE_RULE_REF, 3},
|
||||
{LLAMA_GRETYPE_CHAR, 48},
|
||||
},
|
||||
{
|
||||
{LLAMA_GRETYPE_RULE_REF, 5},
|
||||
{LLAMA_GRETYPE_CHAR, 61},
|
||||
{LLAMA_GRETYPE_RULE_REF, 7},
|
||||
{LLAMA_GRETYPE_RULE_REF, 3},
|
||||
{LLAMA_GRETYPE_CHAR, 48},
|
||||
},
|
||||
{
|
||||
{LLAMA_GRETYPE_RULE_REF, 5},
|
||||
{LLAMA_GRETYPE_CHAR, 61},
|
||||
{LLAMA_GRETYPE_RULE_REF, 7},
|
||||
{LLAMA_GRETYPE_CHAR, 97},
|
||||
}};
|
||||
|
||||
auto index = 0;
|
||||
for (const llama_grammar_stack & stack : llama_grammar_get_stacks(grammar))
|
||||
{
|
||||
// compare stack to expected_stack
|
||||
for (uint32_t i = 0; i < stack.size(); i++)
|
||||
{
|
||||
const llama_grammar_element * element = stack[i];
|
||||
const llama_grammar_element & expected_element = expected_stacks[index][i];
|
||||
|
||||
// pretty print error message before asserting
|
||||
if (expected_element.type != element->type || expected_element.value != element->value)
|
||||
{
|
||||
fprintf(stderr, "index: %d\n", index);
|
||||
fprintf(stderr, "expected_element: %d, %u\n", expected_element.type, expected_element.value);
|
||||
fprintf(stderr, "actual_element: %d, %u\n", element->type, element->value);
|
||||
fprintf(stderr, "expected_element != actual_element\n");
|
||||
}
|
||||
|
||||
assert(expected_element.type == element->type && expected_element.value == element->value);
|
||||
}
|
||||
index++;
|
||||
}
|
||||
|
||||
std::vector<llama_grammar_candidate> next_candidates;
|
||||
next_candidates.resize(23);
|
||||
|
||||
for (size_t i = 0; i < 23; ++i)
|
||||
{
|
||||
uint32_t *cp = new uint32_t[2]; // dynamically allocate memory for code_point
|
||||
cp[0] = 37 + i;
|
||||
cp[1] = 0;
|
||||
next_candidates[i] = {i, cp, {}, 0};
|
||||
}
|
||||
|
||||
std::vector<std::vector<std::pair<uint32_t, uint16_t>>> expected_reject = {
|
||||
{
|
||||
{0, 37},
|
||||
{1, 38},
|
||||
{2, 39},
|
||||
{4, 41},
|
||||
{5, 42},
|
||||
{6, 43},
|
||||
{7, 44},
|
||||
{8, 45},
|
||||
{9, 46},
|
||||
{10, 47},
|
||||
{11, 48},
|
||||
{12, 49},
|
||||
{13, 50},
|
||||
{14, 51},
|
||||
{15, 52},
|
||||
{16, 53},
|
||||
{17, 54},
|
||||
{18, 55},
|
||||
{19, 56},
|
||||
{20, 57},
|
||||
{21, 58},
|
||||
{22, 59},
|
||||
{23, 60},
|
||||
},
|
||||
{
|
||||
{0, 37},
|
||||
{1, 38},
|
||||
{2, 39},
|
||||
{3, 40},
|
||||
{4, 41},
|
||||
{5, 42},
|
||||
{6, 43},
|
||||
{7, 44},
|
||||
{8, 45},
|
||||
{9, 46},
|
||||
{10, 47},
|
||||
{21, 58},
|
||||
{22, 59},
|
||||
{23, 60},
|
||||
},
|
||||
{
|
||||
{0, 37},
|
||||
{1, 38},
|
||||
{2, 39},
|
||||
{3, 40},
|
||||
{4, 41},
|
||||
{5, 42},
|
||||
{6, 43},
|
||||
{7, 44},
|
||||
{8, 45},
|
||||
{9, 46},
|
||||
{10, 47},
|
||||
{21, 58},
|
||||
{22, 59},
|
||||
{23, 60},
|
||||
},
|
||||
{
|
||||
{0, 37},
|
||||
{1, 38},
|
||||
{2, 39},
|
||||
{3, 40},
|
||||
{4, 41},
|
||||
{5, 42},
|
||||
{6, 43},
|
||||
{7, 44},
|
||||
{8, 45},
|
||||
{9, 46},
|
||||
{10, 47},
|
||||
{11, 48},
|
||||
{12, 49},
|
||||
{13, 50},
|
||||
{14, 51},
|
||||
{15, 52},
|
||||
{16, 53},
|
||||
{17, 54},
|
||||
{18, 55},
|
||||
{19, 56},
|
||||
{20, 57},
|
||||
{21, 58},
|
||||
{22, 59},
|
||||
},
|
||||
{
|
||||
{0, 37},
|
||||
{1, 38},
|
||||
{2, 39},
|
||||
{4, 41},
|
||||
{5, 42},
|
||||
{6, 43},
|
||||
{7, 44},
|
||||
{8, 45},
|
||||
{9, 46},
|
||||
{10, 47},
|
||||
{11, 48},
|
||||
{12, 49},
|
||||
{13, 50},
|
||||
{14, 51},
|
||||
{15, 52},
|
||||
{16, 53},
|
||||
{17, 54},
|
||||
{18, 55},
|
||||
{19, 56},
|
||||
{20, 57},
|
||||
{21, 58},
|
||||
{22, 59},
|
||||
{23, 60},
|
||||
},
|
||||
{
|
||||
{0, 37},
|
||||
{1, 38},
|
||||
{2, 39},
|
||||
{3, 40},
|
||||
{4, 41},
|
||||
{5, 42},
|
||||
{6, 43},
|
||||
{7, 44},
|
||||
{8, 45},
|
||||
{9, 46},
|
||||
{10, 47},
|
||||
{21, 58},
|
||||
{22, 59},
|
||||
{23, 60},
|
||||
},
|
||||
{
|
||||
{0, 37},
|
||||
{1, 38},
|
||||
{2, 39},
|
||||
{3, 40},
|
||||
{4, 41},
|
||||
{5, 42},
|
||||
{6, 43},
|
||||
{7, 44},
|
||||
{8, 45},
|
||||
{9, 46},
|
||||
{10, 47},
|
||||
{21, 58},
|
||||
{22, 59},
|
||||
{23, 60},
|
||||
},
|
||||
{
|
||||
{0, 37},
|
||||
{1, 38},
|
||||
{2, 39},
|
||||
{3, 40},
|
||||
{4, 41},
|
||||
{5, 42},
|
||||
{6, 43},
|
||||
{7, 44},
|
||||
{8, 45},
|
||||
{9, 46},
|
||||
{10, 47},
|
||||
{11, 48},
|
||||
{12, 49},
|
||||
{13, 50},
|
||||
{14, 51},
|
||||
{15, 52},
|
||||
{16, 53},
|
||||
{17, 54},
|
||||
{18, 55},
|
||||
{19, 56},
|
||||
{20, 57},
|
||||
{21, 58},
|
||||
{22, 59},
|
||||
},
|
||||
};
|
||||
|
||||
std::vector<llama_grammar_candidate> rejects = llama_grammar_reject_candidates_for_stack(llama_grammar_get_rules(grammar), llama_grammar_get_stacks(grammar)[0], next_candidates);
|
||||
|
||||
std::vector<std::vector<llama_grammar_candidate>> all_rejects;
|
||||
|
||||
for (std::size_t count = 0; count < llama_grammar_get_stacks(grammar).size(); ++count)
|
||||
{
|
||||
rejects = llama_grammar_reject_candidates_for_stack(llama_grammar_get_rules(grammar), llama_grammar_get_stacks(grammar)[count], next_candidates);
|
||||
all_rejects.push_back(rejects);
|
||||
}
|
||||
|
||||
index = 0;
|
||||
for (auto rej : all_rejects)
|
||||
{
|
||||
for (uint32_t i = 0; i < rej.size(); i++)
|
||||
{
|
||||
auto element = rej[i];
|
||||
auto expected_element = expected_reject[index][i];
|
||||
assert(element.index == expected_element.first && *element.code_points == expected_element.second);
|
||||
}
|
||||
index++;
|
||||
}
|
||||
|
||||
for (auto &candidate : next_candidates)
|
||||
{
|
||||
delete[] candidate.code_points;
|
||||
candidate.code_points = nullptr;
|
||||
}
|
||||
|
||||
llama_grammar_free_impl(grammar);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
#include "log.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <thread>
|
||||
|
||||
int main() {
|
||||
const int n_thread = 8;
|
||||
|
||||
std::thread threads[n_thread];
|
||||
for (int i = 0; i < n_thread; i++) {
|
||||
threads[i] = std::thread([i]() {
|
||||
const int n_msg = 1000;
|
||||
|
||||
for (int j = 0; j < n_msg; j++) {
|
||||
const int log_type = std::rand() % 4;
|
||||
|
||||
switch (log_type) {
|
||||
case 0: LOG_INF("Thread %d: %d\n", i, j); break;
|
||||
case 1: LOG_WRN("Thread %d: %d\n", i, j); break;
|
||||
case 2: LOG_ERR("Thread %d: %d\n", i, j); break;
|
||||
case 3: LOG_DBG("Thread %d: %d\n", i, j); break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (rand () % 10 < 5) {
|
||||
common_log_set_timestamps(common_log_main(), rand() % 2);
|
||||
common_log_set_prefix (common_log_main(), rand() % 2);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_thread; i++) {
|
||||
threads[i].join();
|
||||
}
|
||||
|
||||
common_log_flush(common_log_main());
|
||||
// We explicitly free the logger singleton to avoid hanging on Windows
|
||||
// related to timing issues of thread startup and DLL teardown
|
||||
common_log_free(common_log_main());
|
||||
return 0;
|
||||
}
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
# Array of models to iterate over
|
||||
declare -a params=(
|
||||
"Gemma2ForCausalLM 64"
|
||||
"LlamaForCausalLM 64"
|
||||
"Phi3ForCausalLM 64"
|
||||
)
|
||||
|
||||
MODELS_REPO=lora-tests
|
||||
MODELS_REPO_URL=https://huggingface.co/ggml-org/$MODELS_REPO
|
||||
COMMIT=c26d5fb85b4070a9e9c4e65d132c783b98086890
|
||||
|
||||
# Clone the Hugging Face repository if the directory does not exist
|
||||
if [ ! -d "$MODELS_REPO" ]; then
|
||||
echo "Cloning the Hugging Face repository..."
|
||||
git clone $MODELS_REPO_URL --depth 1
|
||||
cd $MODELS_REPO
|
||||
git fetch --depth=1 origin $COMMIT
|
||||
git reset --hard $COMMIT
|
||||
cd -
|
||||
else
|
||||
echo "Repository already exists. Skipping clone."
|
||||
fi
|
||||
|
||||
# Array to store results to print
|
||||
results=()
|
||||
|
||||
trim_leading_whitespace() {
|
||||
local input_string="$1"
|
||||
echo "${input_string#"${input_string%%[![:space:]]*}"}"
|
||||
}
|
||||
|
||||
extract_starting_substring() {
|
||||
local reference_string="$1"
|
||||
local target_string="$2"
|
||||
|
||||
local target_length=${#target_string}
|
||||
echo "${reference_string:0:$target_length}"
|
||||
}
|
||||
|
||||
get_first_word() {
|
||||
local input_string="$1"
|
||||
read -r first_word _ <<< "$input_string"
|
||||
echo "$first_word"
|
||||
}
|
||||
|
||||
# Load the expected strings
|
||||
EXPECTED_BASE_FULL=$(cat $MODELS_REPO/data/pale_blue_dot.txt)
|
||||
EXPECTED_LORA_FULL=$(cat $MODELS_REPO/data/bohemian_rhapsody.txt)
|
||||
EXPECTED_BASE_FIRST_WORD=$(get_first_word "$EXPECTED_BASE_FULL")
|
||||
EXPECTED_LORA_FIRST_WORD=$(get_first_word "$EXPECTED_LORA_FULL")
|
||||
|
||||
run_conversion_and_inference_lora() {
|
||||
local model_name=$1
|
||||
local hidden_size=$2
|
||||
|
||||
echo -e "\n\n-------- RUNNING TEST FOR MODEL $model_name --------\n\n"
|
||||
|
||||
# Convert safetensors to gguf
|
||||
echo "Running convert_hf_to_gguf.py for $model_name with hidden_size $hidden_size..."
|
||||
python convert_hf_to_gguf.py $MODELS_REPO/$model_name/hidden_size=$hidden_size/base \
|
||||
--outfile $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \
|
||||
--outtype f32
|
||||
|
||||
echo -e "\n\n---------------------------\n\n"
|
||||
echo "Running convert_lora_to_gguf.py for $model_name with hidden_size $hidden_size..."
|
||||
python3 convert_lora_to_gguf.py $MODELS_REPO/$model_name/hidden_size=$hidden_size/lora \
|
||||
--base $MODELS_REPO/$model_name/hidden_size=$hidden_size/base \
|
||||
--outtype f32
|
||||
|
||||
echo -e "\n\n---------------------------\n\n"
|
||||
echo "Running llama-export-lora with lora for $model_name with hidden_size $hidden_size..."
|
||||
./llama-export-lora \
|
||||
-m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \
|
||||
-o $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32-lora-merged.gguf \
|
||||
--lora $MODELS_REPO/$model_name/hidden_size=$hidden_size/lora/Lora-F32-LoRA.gguf
|
||||
|
||||
# Run inference
|
||||
echo -e "\n\n---------------------------\n\n"
|
||||
echo "Running llama-completion without lora for $model_name with hidden_size $hidden_size..."
|
||||
OUTPUT_BASE=$(./llama-completion -no-cnv -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \
|
||||
-p "$EXPECTED_BASE_FIRST_WORD" -n 50 --seed 42 --temp 0)
|
||||
|
||||
echo -e "\n\n---------------------------\n\n"
|
||||
echo "Running llama-completion with hot lora for $model_name with hidden_size $hidden_size..."
|
||||
OUTPUT_LORA_HOT=$(./llama-completion -no-cnv -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \
|
||||
--lora $MODELS_REPO/$model_name/hidden_size=$hidden_size/lora/Lora-F32-LoRA.gguf \
|
||||
-p "$EXPECTED_LORA_FIRST_WORD" -n 50 --seed 42 --temp 0)
|
||||
|
||||
echo -e "\n\n---------------------------\n\n"
|
||||
echo "Running llama-completion with merged lora for $model_name with hidden_size $hidden_size..."
|
||||
OUTPUT_LORA_MERGED=$(./llama-completion -no-cnv -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32-lora-merged.gguf \
|
||||
-p "$EXPECTED_LORA_FIRST_WORD" -n 50 --seed 42 --temp 0)
|
||||
|
||||
# Remove any initial white space
|
||||
OUTPUT_BASE=$(trim_leading_whitespace "$OUTPUT_BASE")
|
||||
OUTPUT_LORA_HOT=$(trim_leading_whitespace "$OUTPUT_LORA_HOT")
|
||||
OUTPUT_LORA_MERGED=$(trim_leading_whitespace "$OUTPUT_LORA_MERGED")
|
||||
# Extract the corresponding substring from full string
|
||||
EXPECTED_BASE=$(extract_starting_substring "$EXPECTED_BASE_FULL" "$OUTPUT_BASE")
|
||||
EXPECTED_LORA=$(extract_starting_substring "$EXPECTED_LORA_FULL" "$OUTPUT_LORA_HOT")
|
||||
|
||||
# Assert output equals the expected output
|
||||
if [[ "$OUTPUT_BASE" != "$EXPECTED_BASE" ]]; then
|
||||
echo "Error: $model_name OUTPUT_BASE does not start with the expected string."
|
||||
echo -e "Out=$OUTPUT_BASE\n\nExp=$EXPECTED_BASE"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$OUTPUT_LORA_HOT" != "$EXPECTED_LORA" ]]; then
|
||||
echo "Error: $model_name OUTPUT_LORA_HOT does not start with the expected string."
|
||||
echo -e "Out=$OUTPUT_LORA_HOT\n\nExp=$EXPECTED_LORA"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$OUTPUT_LORA_MERGED" != "$EXPECTED_LORA" ]]; then
|
||||
echo "Error: $model_name OUTPUT_LORA_MERGED does not start with the expected string."
|
||||
echo -e "Out=$OUTPUT_LORA_MERGED\n\nExp=$EXPECTED_LORA"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Store the results
|
||||
results+=("
|
||||
\n\033[1mResults for $model_name with hidden_size $hidden_size:\033[0m
|
||||
\n\033[32m • Base:\n$OUTPUT_BASE
|
||||
\n\033[34m • Lora hot:\n$OUTPUT_LORA_HOT
|
||||
\n\033[36m • Lora merged:\n$OUTPUT_LORA_MERGED
|
||||
\n \033[0m
|
||||
")
|
||||
|
||||
echo "All tests passed for $model_name with hidden_size $hidden_size!"
|
||||
}
|
||||
|
||||
# Run test for each model
|
||||
for param in "${params[@]}"; do
|
||||
run_conversion_and_inference_lora $param
|
||||
done
|
||||
|
||||
# Print results
|
||||
echo -e "\n\n---------------------------\n\n"
|
||||
echo -e "\n\033[1mSummary of All Results:\033[0m"
|
||||
for result in "${results[@]}"; do
|
||||
echo -e "$result"
|
||||
done
|
||||
@@ -0,0 +1,285 @@
|
||||
// checks the MUL_MAT+ADD packs of the Metal graph reorder (devices, pack readers, src1 rows), and that the encoder fuses
|
||||
// no MUL_MAT+ADD the reorder leaves unpacked
|
||||
#include "ggml.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml-impl.h"
|
||||
#include "ggml-metal-common.h"
|
||||
#include "ggml-metal-device.h"
|
||||
#include "ggml-metal-fusion.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <vector>
|
||||
|
||||
static constexpr int n_tensors = 64;
|
||||
static constexpr float scale = 2.0f;
|
||||
static constexpr float norm_eps = 1e-6f;
|
||||
// a mat-mul the few-row MMA kernels take: K a multiple of their 64-weight step, 2..16 src1 rows
|
||||
static constexpr int64_t n_k = 64;
|
||||
static constexpr int64_t n_m = 16;
|
||||
static constexpr int64_t n_rows = 8;
|
||||
// src1 row counts below, inside and above the 2..16 rows of the few-row MMA kernels
|
||||
static const std::vector<int64_t> batch_rows = { 1, 4, 9, 64, 512 };
|
||||
|
||||
struct device_case {
|
||||
const char * name;
|
||||
|
||||
bool has_native_simdgroup_mm;
|
||||
bool packed;
|
||||
};
|
||||
|
||||
// a device with probed simdgroup matrices that are native (MTLGPUFamilyApple7+) only if has_native_simdgroup_mm
|
||||
static ggml_metal_device_props device_props(bool has_native_simdgroup_mm) {
|
||||
ggml_metal_device_props props = {};
|
||||
props.has_simdgroup_mm = has_native_simdgroup_mm;
|
||||
return props;
|
||||
}
|
||||
|
||||
// the index of t in the nodes of graph, -1 if absent
|
||||
static int node_index(ggml_cgraph * graph, const ggml_tensor * t) {
|
||||
for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
|
||||
if (ggml_graph_node(graph, i) == t) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
// the reordered positions of the tracked nodes
|
||||
static std::vector<int> node_positions(ggml_cgraph * graph, const std::vector<ggml_tensor *> & tracked) {
|
||||
std::vector<int> res;
|
||||
for (const ggml_tensor * t : tracked) {
|
||||
res.push_back(node_index(graph, t));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// allocates the tensors of ctx in a CPU buffer marked as weights, as the model loader does
|
||||
static ggml_backend_buffer_t alloc_weights(ggml_context * ctx) {
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_dev_buffer_type(ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU));
|
||||
ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);
|
||||
ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
// what the add sums with the mat-mul
|
||||
enum add_operand_kind {
|
||||
ADD_RESIDUAL, // a same-shape activation
|
||||
ADD_BIAS, // a one-row bias weight
|
||||
ADD_BIAS_VIEW, // a view of a bias weight
|
||||
};
|
||||
|
||||
static const char * add_operand_name(add_operand_kind kind) {
|
||||
switch (kind) {
|
||||
case ADD_RESIDUAL: return "residual";
|
||||
case ADD_BIAS: return "bias";
|
||||
case ADD_BIAS_VIEW: return "bias view";
|
||||
}
|
||||
return "?";
|
||||
}
|
||||
|
||||
static ggml_tensor * new_add_operand(ggml_context * ctx, ggml_context * ctx_w, add_operand_kind kind, int64_t rows) {
|
||||
switch (kind) {
|
||||
case ADD_RESIDUAL: return ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_m, rows);
|
||||
case ADD_BIAS: return ggml_new_tensor_2d(ctx_w, GGML_TYPE_F32, n_m, 1);
|
||||
case ADD_BIAS_VIEW: return ggml_reshape_2d(ctx, ggml_new_tensor_1d(ctx_w, GGML_TYPE_F32, n_m), n_m, 1);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// the reordered positions of a mat-mul with rows src1 rows, the add of an operand of kind to it and an independent
|
||||
// mat-mul, which the reorder runs between the first mat-mul and the add unless the pair is packed
|
||||
static std::vector<int> reorder_mul_mat_add(int64_t rows, add_operand_kind kind) {
|
||||
ggml_init_params params = { n_tensors*ggml_tensor_overhead() + ggml_graph_overhead(), nullptr, true };
|
||||
ggml_context * ctx = ggml_init(params);
|
||||
ggml_context * ctx_w = ggml_init(params);
|
||||
ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_k, rows);
|
||||
ggml_tensor * mm = ggml_mul_mat(ctx, ggml_new_tensor_2d(ctx_w, GGML_TYPE_F32, n_k, n_m), x);
|
||||
ggml_tensor * add = ggml_add(ctx, mm, new_add_operand(ctx, ctx_w, kind, rows));
|
||||
ggml_tensor * other = ggml_mul_mat(ctx, ggml_new_tensor_2d(ctx_w, GGML_TYPE_F32, n_k, n_m), x);
|
||||
ggml_backend_buffer_t weights = alloc_weights(ctx_w);
|
||||
|
||||
ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
ggml_build_forward_expand(graph, add);
|
||||
ggml_build_forward_expand(graph, other);
|
||||
ggml_graph_optimize(graph);
|
||||
const std::vector<int> res = node_positions(graph, { mm, add, other });
|
||||
|
||||
ggml_backend_buffer_free(weights);
|
||||
ggml_free(ctx_w);
|
||||
ggml_free(ctx);
|
||||
return res;
|
||||
}
|
||||
|
||||
// true if the independent mat-mul of reorder_mul_mat_add does not run between the mat-mul and the add
|
||||
static bool add_packed(const std::vector<int> & pos) {
|
||||
return !(pos[0] < pos[2] && pos[2] < pos[1]);
|
||||
}
|
||||
|
||||
static bool check_pack(const device_case & c) {
|
||||
const std::vector<int> pos = reorder_mul_mat_add(n_rows, ADD_RESIDUAL);
|
||||
const bool packed = add_packed(pos);
|
||||
|
||||
const bool ok = packed == c.packed;
|
||||
std::printf("%s: mat-mul and add packed %d (expected %d): %s\n", c.name, packed, c.packed, ok ? "OK" : "FAIL");
|
||||
return ok;
|
||||
}
|
||||
|
||||
static int run_pack_cases(const std::vector<device_case> & cases) {
|
||||
int failures = 0;
|
||||
for (const device_case & c : cases) {
|
||||
failures += check_pack(c) ? 0 : 1;
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
// true if every row count in rows gives the reorder of the first one, on a device that fuses MUL_MAT+ADD
|
||||
static bool same_reorder_for_rows(const std::vector<int64_t> & rows, add_operand_kind kind) {
|
||||
const std::vector<int> first = reorder_mul_mat_add(rows.front(), kind);
|
||||
for (auto r = rows.begin() + 1; r != rows.end(); ++r) {
|
||||
if (reorder_mul_mat_add(*r, kind) != first) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// the pack must not depend on the batch size, or graphs with the same nodes get another allocation per ubatch size,
|
||||
// and a bias add, which the encoder never fuses, must stay free to run next to independent nodes
|
||||
static bool check_row_independent_pack(add_operand_kind kind) {
|
||||
const bool same = same_reorder_for_rows(batch_rows, kind);
|
||||
const bool packed = add_packed(reorder_mul_mat_add(batch_rows.front(), kind));
|
||||
const bool expected = kind == ADD_RESIDUAL;
|
||||
const bool ok = same && packed == expected;
|
||||
std::printf("MUL_MAT+ADD of a %s, reorder independent of src1 rows %d, packed %d (expected %d): %s\n",
|
||||
add_operand_name(kind), same, packed, expected, ok ? "OK" : "FAIL");
|
||||
return ok;
|
||||
}
|
||||
|
||||
static int run_row_independent_pack_cases() {
|
||||
int failures = 0;
|
||||
for (add_operand_kind kind : { ADD_RESIDUAL, ADD_BIAS, ADD_BIAS_VIEW }) {
|
||||
failures += check_row_independent_pack(kind) ? 0 : 1;
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
// true if the encoder's check fuses a few-row MUL_MAT with a same-shape residual, which is a weight if weight_res;
|
||||
// the check compares Metal buffer ranges, so the tensors live in Metal buffers
|
||||
static bool encoder_fuses_mul_mat_add(bool weight_res) {
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_dev_buffer_type(ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU));
|
||||
ggml_init_params params = { n_tensors*ggml_tensor_overhead(), nullptr, true };
|
||||
ggml_context * ctx = ggml_init(params);
|
||||
ggml_context * ctx_w = ggml_init(params);
|
||||
ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_k, n_rows);
|
||||
ggml_tensor * mm = ggml_mul_mat(ctx, ggml_new_tensor_2d(ctx_w, GGML_TYPE_F32, n_k, n_m), x);
|
||||
ggml_tensor * add = ggml_add(ctx, mm, ggml_new_tensor_2d(weight_res ? ctx_w : ctx, GGML_TYPE_F32, n_m, n_rows));
|
||||
ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft);
|
||||
ggml_backend_buffer_t weights = ggml_backend_alloc_ctx_tensors_from_buft(ctx_w, buft);
|
||||
ggml_backend_buffer_set_usage(weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
|
||||
ggml_init_params params_gf = { ggml_graph_overhead(), nullptr, true };
|
||||
ggml_context * ctx_gf = ggml_init(params_gf);
|
||||
ggml_cgraph * gf = ggml_new_graph(ctx_gf);
|
||||
ggml_build_forward_expand(gf, add);
|
||||
|
||||
const int idxs[] = { 0, 1 };
|
||||
int n_fused = 1;
|
||||
const ggml_metal_fusion * fusion = ggml_metal_fusion_next(gf, idxs, 2, 0, GGML_METAL_FUSION_FULL, &n_fused);
|
||||
const bool fused = fusion != nullptr && ggml_metal_fusion_get_id(fusion) == GGML_METAL_FUSION_MUL_MAT_ADD;
|
||||
ggml_free(ctx_gf);
|
||||
|
||||
ggml_backend_buffer_free(weights);
|
||||
ggml_backend_buffer_free(buffer);
|
||||
ggml_free(ctx_w);
|
||||
ggml_free(ctx);
|
||||
return fused;
|
||||
}
|
||||
|
||||
// the encoder may fuse only what the reorder packs, so it must leave a same-shape residual alone when it is a weight
|
||||
static bool check_encoder_skips_weight_residual() {
|
||||
const bool fuses_residual = encoder_fuses_mul_mat_add(false);
|
||||
const bool fuses_weight = encoder_fuses_mul_mat_add(true);
|
||||
const bool ok = fuses_residual && !fuses_weight;
|
||||
std::printf("encoder fuses MUL_MAT+ADD of a residual %d (expected 1), of a weight %d (expected 0): %s\n",
|
||||
fuses_residual, fuses_weight, ok ? "OK" : "FAIL");
|
||||
return ok;
|
||||
}
|
||||
|
||||
static ggml_context * graph_ctx() {
|
||||
ggml_init_params params = { n_tensors*ggml_tensor_overhead() + ggml_graph_overhead(), nullptr, true };
|
||||
return ggml_init(params);
|
||||
}
|
||||
|
||||
static void expand_all(ggml_cgraph * graph, const std::vector<ggml_tensor *> & outputs) {
|
||||
for (ggml_tensor * t : outputs) {
|
||||
ggml_build_forward_expand(graph, t);
|
||||
}
|
||||
}
|
||||
|
||||
// the graph of outputs in build order, reordered for a device that fuses MUL_MAT+ADD
|
||||
static ggml_cgraph * optimized_graph(ggml_context * ctx, const std::vector<ggml_tensor *> & outputs) {
|
||||
ggml_cgraph * graph = ggml_new_graph(ctx);
|
||||
expand_all(graph, outputs);
|
||||
ggml_graph_optimize(graph);
|
||||
return graph;
|
||||
}
|
||||
|
||||
// the position of t among the nodes the encoder runs (views are skipped), -1 if absent
|
||||
static int encoded_index(ggml_cgraph * graph, const ggml_tensor * t) {
|
||||
int res = 0;
|
||||
for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
|
||||
const ggml_tensor * node = ggml_graph_node(graph, i);
|
||||
if (node == t) {
|
||||
return res;
|
||||
}
|
||||
res += ggml_op_is_empty(node->op) ? 0 : 1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
// true if the encoder runs the nodes back to back in this order, so they fuse
|
||||
static bool encoded_in_a_row(ggml_cgraph * graph, const std::vector<ggml_tensor *> & nodes) {
|
||||
const int first = encoded_index(graph, nodes[0]);
|
||||
for (size_t j = 1; j < nodes.size(); ++j) {
|
||||
if (encoded_index(graph, nodes[j]) != first + (int) j) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool report_reader(const char * name, bool packed, bool reader_after) {
|
||||
const bool ok = packed && reader_after;
|
||||
std::printf("%s: packed %d, reader after the write %d (expected 1, 1): %s\n", name, packed, reader_after, ok ? "OK" : "FAIL");
|
||||
return ok;
|
||||
}
|
||||
|
||||
// x feeds a MUL_MAT+ADD pack chained with RMS_NORM+MUL, and c reads the residual sum h, an output inside the pack
|
||||
static bool check_chained_pack_reader() {
|
||||
ggml_context * ctx = graph_ctx();
|
||||
ggml_tensor * x = ggml_scale(ctx, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_k, n_rows), scale);
|
||||
ggml_tensor * mm = ggml_mul_mat(ctx, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_k, n_m), x);
|
||||
ggml_tensor * h = ggml_add(ctx, mm, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_m, n_rows));
|
||||
ggml_tensor * n = ggml_rms_norm(ctx, h, norm_eps);
|
||||
ggml_tensor * m = ggml_mul(ctx, n, ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_m));
|
||||
ggml_tensor * c = ggml_scale(ctx, h, scale);
|
||||
|
||||
ggml_cgraph * graph = optimized_graph(ctx, { m, c });
|
||||
const bool packed = encoded_in_a_row(graph, { mm, h, n, m });
|
||||
const bool after = encoded_index(graph, c) > encoded_index(graph, h);
|
||||
ggml_free(ctx);
|
||||
|
||||
return report_reader("reader of a chained MUL_MAT+ADD sum", packed, after);
|
||||
}
|
||||
|
||||
int main() {
|
||||
const std::vector<device_case> devices = {
|
||||
{ "native simdgroup matrices", true, true },
|
||||
{ "probed or no simdgroup matrices", false, true },
|
||||
};
|
||||
const int failures = run_pack_cases(devices) + (check_chained_pack_reader() ? 0 : 1) + run_row_independent_pack_cases() +
|
||||
(check_encoder_skips_weight_residual() ? 0 : 1);
|
||||
|
||||
return failures == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#include "llama.h"
|
||||
#include "common.h"
|
||||
|
||||
#include <cstdlib>
|
||||
|
||||
int main(int argc, char *argv[] ) {
|
||||
auto * model_path = common_get_model_or_exit(argc, argv);
|
||||
auto * file = fopen(model_path, "r");
|
||||
if (file == nullptr) {
|
||||
fprintf(stderr, "no model at '%s' found\n", model_path);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
fprintf(stderr, "using '%s'\n", model_path);
|
||||
fclose(file);
|
||||
|
||||
llama_backend_init();
|
||||
auto params = llama_model_params{};
|
||||
params.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
params.progress_callback = [](float progress, void * ctx){
|
||||
(void) ctx;
|
||||
return progress > 0.50;
|
||||
};
|
||||
auto * model = llama_model_load_from_file(model_path, params);
|
||||
llama_backend_free();
|
||||
return model == nullptr ? EXIT_SUCCESS : EXIT_FAILURE;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,172 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include "mtmd.h"
|
||||
#include "mtmd-helper.h"
|
||||
|
||||
int main(void) {
|
||||
printf("\n\nTesting libmtmd C API...\n");
|
||||
printf("--------\n\n");
|
||||
|
||||
struct mtmd_context_params params = mtmd_context_params_default();
|
||||
printf("Default image marker: %s\n", params.image_marker);
|
||||
|
||||
mtmd_input_chunks * chunks = mtmd_test_create_input_chunks();
|
||||
|
||||
if (!chunks) {
|
||||
fprintf(stderr, "Failed to create input chunks\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
// simple test for the helper
|
||||
size_t n_tokens_total = mtmd_helper_get_n_tokens(chunks);
|
||||
printf("Total tokens in chunks: %zu\n", n_tokens_total);
|
||||
assert(n_tokens_total > 0);
|
||||
|
||||
size_t n_chunks = mtmd_input_chunks_size(chunks);
|
||||
printf("Number of chunks: %zu\n", n_chunks);
|
||||
assert(n_chunks > 0);
|
||||
|
||||
for (size_t i = 0; i < n_chunks; i++) {
|
||||
const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i);
|
||||
assert(chunk != NULL);
|
||||
enum mtmd_input_chunk_type type = mtmd_input_chunk_get_type(chunk);
|
||||
printf("Chunk %zu type: %d\n", i, type);
|
||||
|
||||
if (type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
|
||||
size_t n_tokens;
|
||||
const llama_token * tokens = mtmd_input_chunk_get_tokens_text(chunk, &n_tokens);
|
||||
printf(" Text chunk with %zu tokens\n", n_tokens);
|
||||
assert(tokens != NULL);
|
||||
assert(n_tokens > 0);
|
||||
for (size_t j = 0; j < n_tokens; j++) {
|
||||
assert(tokens[j] >= 0);
|
||||
printf(" > Token %zu: %d\n", j, tokens[j]);
|
||||
}
|
||||
|
||||
} else if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {
|
||||
const mtmd_image_tokens * image_tokens = mtmd_input_chunk_get_tokens_image(chunk);
|
||||
size_t n_tokens = mtmd_image_tokens_get_n_tokens(image_tokens);
|
||||
// get position of the last token, which should be (nx - 1, ny - 1)
|
||||
struct mtmd_decoder_pos pos = mtmd_image_tokens_get_decoder_pos(image_tokens, 0, n_tokens - 1);
|
||||
size_t nx = pos.x + 1;
|
||||
size_t ny = pos.y + 1;
|
||||
const char * id = mtmd_image_tokens_get_id(image_tokens);
|
||||
assert(n_tokens > 0);
|
||||
assert(nx > 0);
|
||||
assert(ny > 0);
|
||||
assert(id != NULL);
|
||||
printf(" Image chunk with %zu tokens\n", n_tokens);
|
||||
printf(" Image size: %zu x %zu\n", nx, ny);
|
||||
printf(" Image ID: %s\n", id);
|
||||
}
|
||||
}
|
||||
|
||||
// test chunk save/load round-trip
|
||||
for (size_t i = 0; i < n_chunks; i++) {
|
||||
const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i);
|
||||
assert(chunk != NULL);
|
||||
enum mtmd_input_chunk_type type = mtmd_input_chunk_get_type(chunk);
|
||||
|
||||
// query the required buffer size (out_buf == NULL)
|
||||
size_t expected_len = 0;
|
||||
int32_t rc = mtmd_input_chunk_save(chunk, NULL, 0, &expected_len);
|
||||
printf(" Chunk %zu: save query rc = %d, expected_len = %zu\n", i, rc, expected_len);
|
||||
assert(rc == 0);
|
||||
assert(expected_len > 0);
|
||||
|
||||
// saving into a too-small buffer must fail, not crash
|
||||
char tiny_buf[1];
|
||||
rc = mtmd_input_chunk_save(chunk, tiny_buf, sizeof(tiny_buf), NULL);
|
||||
printf(" Chunk %zu: save into too-small buffer rc = %d (expect non-zero)\n", i, rc);
|
||||
assert(rc != 0);
|
||||
|
||||
// save into a properly-sized buffer
|
||||
char * buf = (char *) malloc(expected_len);
|
||||
assert(buf != NULL);
|
||||
rc = mtmd_input_chunk_save(chunk, buf, expected_len, NULL);
|
||||
assert(rc == 0);
|
||||
|
||||
// loading from a truncated buffer must fail gracefully, not crash
|
||||
if (expected_len > 1) {
|
||||
mtmd_input_chunk * bad = mtmd_input_chunk_load(buf, expected_len - 1);
|
||||
printf(" Chunk %zu: load from truncated buffer = %p (expect NULL)\n", i, (void *) bad);
|
||||
assert(bad == NULL);
|
||||
}
|
||||
|
||||
// load it back
|
||||
mtmd_input_chunk * loaded = mtmd_input_chunk_load(buf, expected_len);
|
||||
assert(loaded != NULL);
|
||||
|
||||
// metadata must match the original chunk
|
||||
assert(mtmd_input_chunk_get_type(loaded) == type);
|
||||
assert(mtmd_input_chunk_get_n_tokens(loaded) == mtmd_input_chunk_get_n_tokens(chunk));
|
||||
assert(mtmd_input_chunk_get_n_pos(loaded) == mtmd_input_chunk_get_n_pos(chunk));
|
||||
|
||||
if (type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
|
||||
size_t n_tok_orig, n_tok_loaded;
|
||||
const llama_token * tok_orig = mtmd_input_chunk_get_tokens_text(chunk, &n_tok_orig);
|
||||
const llama_token * tok_loaded = mtmd_input_chunk_get_tokens_text(loaded, &n_tok_loaded);
|
||||
printf(" Chunk %zu: loaded %zu text tokens (orig %zu), first token %d (orig %d)\n",
|
||||
i, n_tok_loaded, n_tok_orig,
|
||||
n_tok_loaded > 0 ? tok_loaded[0] : -1,
|
||||
n_tok_orig > 0 ? tok_orig[0] : -1);
|
||||
assert(n_tok_orig == n_tok_loaded);
|
||||
for (size_t j = 0; j < n_tok_orig; j++) {
|
||||
assert(tok_orig[j] == tok_loaded[j]);
|
||||
}
|
||||
} else if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE || type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {
|
||||
const char * id_orig = mtmd_input_chunk_get_id(chunk);
|
||||
const char * id_loaded = mtmd_input_chunk_get_id(loaded);
|
||||
printf(" Chunk %zu: loaded id '%s' (orig '%s')\n", i, id_loaded, id_orig);
|
||||
assert(id_orig != NULL && id_loaded != NULL);
|
||||
assert(strcmp(id_orig, id_loaded) == 0);
|
||||
}
|
||||
|
||||
mtmd_input_chunk_free(loaded);
|
||||
free(buf);
|
||||
}
|
||||
printf("Chunk save/load round-trip OK\n");
|
||||
|
||||
// test input validation of mtmd_tokenize_from_parts()
|
||||
// invalid parts are rejected before the ctx is used, so NULL ctx is OK here
|
||||
{
|
||||
mtmd_input_chunks * out = mtmd_input_chunks_init();
|
||||
mtmd_bitmap * bmp = mtmd_bitmap_init(4, 4, NULL); // placeholder bitmap
|
||||
struct mtmd_input_text txt = { "hello", 5, false, false };
|
||||
struct mtmd_input_text txt_null = { NULL, 0, false, false };
|
||||
|
||||
struct mtmd_input_part part_both = { &txt, bmp };
|
||||
struct mtmd_input_part part_neither = { NULL, NULL };
|
||||
struct mtmd_input_part part_null_text = { &txt_null, NULL };
|
||||
const mtmd_input_part * parts[1];
|
||||
int32_t rc;
|
||||
|
||||
parts[0] = &part_both;
|
||||
rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false);
|
||||
printf("tokenize part with both text and bitmap rc = %d (expect 1)\n", rc);
|
||||
assert(rc == 1);
|
||||
|
||||
parts[0] = &part_neither;
|
||||
rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false);
|
||||
printf("tokenize part with neither text nor bitmap rc = %d (expect 1)\n", rc);
|
||||
assert(rc == 1);
|
||||
|
||||
parts[0] = &part_null_text;
|
||||
rc = mtmd_tokenize_from_parts(NULL, out, parts, 1, false);
|
||||
printf("tokenize part with null text pointer rc = %d (expect 1)\n", rc);
|
||||
assert(rc == 1);
|
||||
|
||||
mtmd_bitmap_free(bmp);
|
||||
mtmd_input_chunks_free(out);
|
||||
}
|
||||
|
||||
// Free the chunks
|
||||
mtmd_input_chunks_free(chunks);
|
||||
|
||||
printf("\n\nDONE: test libmtmd C API...\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
#include "testing.h"
|
||||
|
||||
#include "mtmd-audio.h"
|
||||
#include "mtmd-image.h"
|
||||
#include "mtmd-internal.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
// this test file contains:
|
||||
// 1. test cases for mtmd helpers
|
||||
// 2. test cases for internal mtmd components
|
||||
// internal headers can be included here
|
||||
|
||||
struct test_registry {
|
||||
using fn_t = void (*)(testing &);
|
||||
|
||||
struct entry {
|
||||
std::string name;
|
||||
fn_t fn;
|
||||
};
|
||||
|
||||
static std::vector<entry> & all() {
|
||||
static std::vector<entry> entries;
|
||||
return entries;
|
||||
}
|
||||
|
||||
test_registry(const char * name, fn_t fn) {
|
||||
all().push_back({ name, fn });
|
||||
}
|
||||
};
|
||||
|
||||
#define MAKE_TEST(name) \
|
||||
static void name(testing & t); \
|
||||
static const test_registry test_registry_ ## name(#name, &name); \
|
||||
static void name(testing & t)
|
||||
|
||||
|
||||
//
|
||||
// mtmd_image
|
||||
//
|
||||
|
||||
MAKE_TEST(test_image_preprocessor_longest_edge) {
|
||||
clip_hparams hparams;
|
||||
hparams.image_longest_edge = 1540;
|
||||
hparams.patch_size = 14;
|
||||
hparams.n_merge = 2;
|
||||
|
||||
const std::vector<std::pair<clip_image_size, clip_image_size>> cases = {
|
||||
{ { 463, 463 }, { 476, 476 } },
|
||||
{ { 463, 300 }, { 476, 308 } },
|
||||
{ { 448, 280 }, { 448, 280 } },
|
||||
{ { 2000, 1000 }, { 1540, 784 } },
|
||||
};
|
||||
|
||||
for (const auto & [input, expected] : cases) {
|
||||
const clip_image_size actual = mtmd_image_preprocessor_longest_edge::get_target_size(hparams, input);
|
||||
const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);
|
||||
t.assert_equal(name + " width", expected.width, actual.width);
|
||||
t.assert_equal(name + " height", expected.height, actual.height);
|
||||
}
|
||||
}
|
||||
|
||||
MAKE_TEST(test_image_preprocessor_dyn_size) {
|
||||
// { patch_size, image size, expected size }
|
||||
// a side that is an even number plus .5 merged patches rounds down, like Python round() in smart_resize
|
||||
const std::vector<std::tuple<int, clip_image_size, clip_image_size>> cases = {
|
||||
{ 16, { 768, 400 }, { 768, 384 } },
|
||||
{ 16, { 800, 336 }, { 800, 320 } },
|
||||
{ 16, { 400, 400 }, { 384, 384 } },
|
||||
{ 16, { 640, 240 }, { 640, 256 } },
|
||||
{ 14, { 966, 560 }, { 952, 560 } },
|
||||
};
|
||||
|
||||
for (const auto & [patch_size, input, expected] : cases) {
|
||||
clip_hparams hparams;
|
||||
hparams.patch_size = patch_size;
|
||||
hparams.n_merge = 2;
|
||||
hparams.image_round_half_even = true;
|
||||
hparams.set_limit_image_tokens(8, 4096);
|
||||
|
||||
const clip_image_size actual = mtmd_image_preprocessor_dyn_size::get_target_size(hparams, input);
|
||||
const std::string name = std::to_string(input.width) + "x" + std::to_string(input.height);
|
||||
t.assert_equal(name + " width", expected.width, actual.width);
|
||||
t.assert_equal(name + " height", expected.height, actual.height);
|
||||
}
|
||||
}
|
||||
|
||||
static std::string image_size_to_str(const clip_image_size & size) {
|
||||
return std::to_string(size.width) + "x" + std::to_string(size.height);
|
||||
}
|
||||
|
||||
static std::string resize_algo_to_str(resize_algo algo) {
|
||||
std::string str;
|
||||
switch (algo & ~RESIZE_ALGO_FLAG_TORCH) {
|
||||
case RESIZE_ALGO_BILINEAR: str = "bilinear"; break;
|
||||
case RESIZE_ALGO_BICUBIC: str = "bicubic"; break;
|
||||
case RESIZE_ALGO_LANCZOS: str = "lanczos"; break;
|
||||
default: str = "unknown"; break;
|
||||
}
|
||||
if (algo & RESIZE_ALGO_FLAG_TORCH) {
|
||||
str += "_torch";
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
// pseudo-random pixels: s = s * 1103515245 + 12345 from s = 1, one byte (s >> 16) & 255 per channel
|
||||
static clip_image_u8 make_test_image(const clip_image_size & size) {
|
||||
clip_image_u8 img;
|
||||
img.set_size(size, false);
|
||||
uint32_t s = 1;
|
||||
auto next = [&s]() {
|
||||
s = s * 1103515245u + 12345u;
|
||||
return static_cast<uint8_t>(s >> 16);
|
||||
};
|
||||
for (int y = 0; y < size.height; y++) {
|
||||
for (int x = 0; x < size.width; x++) {
|
||||
img.set_pixel(x, y, { next(), next(), next() });
|
||||
}
|
||||
}
|
||||
return img;
|
||||
}
|
||||
|
||||
// FNV-1a of the pixels
|
||||
static uint32_t hash_pixels(const clip_image_u8 & img, uint32_t hash = 2166136261u) {
|
||||
for (uint8_t v : img.get_ro_buf()) {
|
||||
hash = (hash ^ v) * 16777619u;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
MAKE_TEST(test_image_preprocessor_lfm2) {
|
||||
clip_hparams hparams;
|
||||
hparams.patch_size = 16;
|
||||
hparams.n_merge = 2;
|
||||
hparams.set_limit_image_tokens(64, 256);
|
||||
hparams.preproc_min_tiles = 2;
|
||||
hparams.preproc_max_tiles = 10;
|
||||
hparams.preproc_tile_size = 512;
|
||||
hparams.preproc_max_pixels_tolerance = 2.0f;
|
||||
|
||||
// { image size, expected tiling }
|
||||
const std::vector<std::pair<clip_image_size, bool>> cases = {
|
||||
{ { 704, 704 }, false },
|
||||
// 720 / (patch_size * n_merge) is exactly 22.5, so this only matches HF
|
||||
// if round_by_factor rounds half to even (22) instead of away from zero (23)
|
||||
{ { 720, 720 }, false },
|
||||
{ { 736, 736 }, true },
|
||||
{ { 1024, 977 }, true },
|
||||
{ { 1056, 384 }, false },
|
||||
};
|
||||
|
||||
for (const auto & [size, expected] : cases) {
|
||||
const bool actual = mtmd_image_preprocessor_lfm2::should_tile(hparams, size);
|
||||
|
||||
t.assert_equal(
|
||||
"tiling for " + std::to_string(size.width) + "x" + std::to_string(size.height),
|
||||
std::string(expected ? "tiled" : "single"),
|
||||
std::string(actual ? "tiled" : "single"));
|
||||
}
|
||||
|
||||
// { image size, expected size from HF smart_resize }
|
||||
const std::vector<std::pair<clip_image_size, clip_image_size>> resize_cases = {
|
||||
{ { 400, 600 }, { 384, 608 } }, // 400 / 32 = 12.5 rounds half to even
|
||||
{ { 480, 336 }, { 480, 320 } },
|
||||
{ { 64, 169 }, { 160, 416 } }, // 169 * beta / 32 is exactly 13 in double
|
||||
{ { 1024, 768 }, { 576, 416 } },
|
||||
};
|
||||
|
||||
for (const auto & [size, expected] : resize_cases) {
|
||||
t.assert_equal("smart_resize for " + image_size_to_str(size), image_size_to_str(expected),
|
||||
image_size_to_str(mtmd_image_preprocessor_lfm2::smart_resize(hparams, size)));
|
||||
}
|
||||
|
||||
// { image size, expected grid from HF }
|
||||
const std::vector<std::pair<clip_image_size, clip_image_size>> grid_cases = {
|
||||
{ { 720, 1600 }, { 2, 4 } }, // 0.45 is an exact tie between 2/4 and 2/5 in double, not in float
|
||||
{ { 420, 1440 }, { 1, 3 } },
|
||||
{ { 255, 2160 }, { 1, 8 } },
|
||||
{ { 1024, 768 }, { 3, 2 } },
|
||||
};
|
||||
|
||||
for (const auto & [size, expected] : grid_cases) {
|
||||
t.assert_equal("grid for " + image_size_to_str(size), image_size_to_str(expected),
|
||||
image_size_to_str(mtmd_image_preprocessor_lfm2::get_grid_layout(hparams, size.height, size.width)));
|
||||
}
|
||||
|
||||
// do_image_splitting=false
|
||||
hparams.preproc_min_tiles = 1;
|
||||
hparams.preproc_max_tiles = 1;
|
||||
t.assert_equal("tiling for 1024x977 with max_tiles=1", std::string("single"),
|
||||
std::string(mtmd_image_preprocessor_lfm2::should_tile(hparams, { 1024, 977 }) ? "tiled" : "single"));
|
||||
}
|
||||
|
||||
MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) {
|
||||
clip_hparams hparams;
|
||||
hparams.patch_size = 16;
|
||||
hparams.n_merge = 2;
|
||||
hparams.set_limit_image_tokens(64, 256);
|
||||
hparams.preproc_min_tiles = 2;
|
||||
hparams.preproc_max_tiles = 10;
|
||||
hparams.preproc_tile_size = 512;
|
||||
hparams.preproc_max_pixels_tolerance = 2.0f;
|
||||
// Lfm2VlImageProcessor stretches the image to the tile grid
|
||||
hparams.image_pad_rf = PAD_NONE;
|
||||
|
||||
// { resize algo, image size, expected hash of the tiles and the thumbnail from Lfm2VlImageProcessor (torchvision 0.29, native uint8 resize: arm64, or x86 with AVX2/AVX512) }
|
||||
const std::vector<std::tuple<resize_algo, clip_image_size, uint32_t>> cases = {
|
||||
{ RESIZE_ALGO_BILINEAR, { 620, 410 }, 0xf5e9c941 }, // 608x416
|
||||
{ RESIZE_ALGO_BILINEAR, { 2000, 1500 }, 0x3331f56c }, // 3x2 tiles, 576x416 thumbnail
|
||||
{ RESIZE_ALGO_BICUBIC, { 620, 410 }, 0x2b34eac9 },
|
||||
{ RESIZE_ALGO_BICUBIC, { 2000, 1500 }, 0xb035d942 },
|
||||
};
|
||||
|
||||
for (const auto & [base_algo, size, expected] : cases) {
|
||||
const resize_algo algo = static_cast<resize_algo>(base_algo | RESIZE_ALGO_FLAG_TORCH);
|
||||
hparams.image_resize_algo_rf = algo;
|
||||
hparams.image_resize_algo_ov = algo;
|
||||
const mtmd_image_preprocessor_lfm2 preproc(hparams);
|
||||
const auto sliced = preproc.slice_image(make_test_image(size), preproc.get_slice_instructions(size));
|
||||
|
||||
uint32_t hash = 2166136261u;
|
||||
for (const auto & slice : sliced.slices) {
|
||||
hash = hash_pixels(slice, hash);
|
||||
}
|
||||
hash = hash_pixels(sliced.overview, hash);
|
||||
t.assert_equal("pixels of " + std::to_string(size.width) + "x" + std::to_string(size.height) + " with resize algo " + resize_algo_to_str(algo), expected, hash);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd temporal merge
|
||||
//
|
||||
|
||||
MAKE_TEST(test_temporal_merge_grouping) {
|
||||
std::vector<mtmd::bitmap_ptr> pool; // keeps the bitmaps alive until the end of the test
|
||||
|
||||
// spec chars:
|
||||
// v = video frame, w = video frame of another size, a = audio, i = plain image, t = text
|
||||
auto make_parts = [&pool](const std::string & spec) {
|
||||
std::vector<mtmd_internal_part> parts;
|
||||
for (char c : spec) {
|
||||
if (c == 't') {
|
||||
parts.push_back({ "hello", nullptr });
|
||||
continue;
|
||||
}
|
||||
mtmd_bitmap * bm = nullptr;
|
||||
switch (c) {
|
||||
case 'v': bm = mtmd_bitmap_init(100, 100, nullptr); break;
|
||||
case 'w': bm = mtmd_bitmap_init(200, 200, nullptr); break;
|
||||
case 'a': bm = mtmd_bitmap_init_from_audio(100, nullptr); break;
|
||||
case 'i': bm = mtmd_bitmap_init(100, 100, nullptr); break;
|
||||
default: throw std::runtime_error(std::string("unknown spec char: ") + c);
|
||||
}
|
||||
mtmd_bitmap_set_mergeable(bm, c != 'i');
|
||||
pool.emplace_back(bm);
|
||||
parts.push_back({ "", bm });
|
||||
}
|
||||
return parts;
|
||||
};
|
||||
|
||||
// { parts, n_merge, expected size of each group }
|
||||
const std::vector<std::tuple<std::string, int, std::string>> cases = {
|
||||
{ "vv", 2, "2" },
|
||||
{ "vvv", 2, "21" },
|
||||
{ "vvvv", 2, "22" },
|
||||
{ "vvi", 2, "21" },
|
||||
{ "tvvt", 2, "2" },
|
||||
{ "vtv", 2, "11" }, // text in between breaks the merge
|
||||
{ "vw", 2, "11" }, // different sizes cannot be merged
|
||||
{ "aa", 2, "11" }, // audio is never merged
|
||||
{ "ii", 2, "11" }, // two unrelated images must stay separated
|
||||
{ "iv", 2, "11" },
|
||||
{ "vi", 2, "11" },
|
||||
{ "vv", 1, "11" }, // model without temporal merge
|
||||
};
|
||||
|
||||
for (const auto & [spec, n_merge, expected] : cases) {
|
||||
auto parts = make_parts(spec);
|
||||
auto groups = mtmd_group_mergeable_bitmaps(parts, n_merge);
|
||||
|
||||
std::string actual;
|
||||
for (const auto & group : groups) {
|
||||
actual += std::to_string(group.size());
|
||||
}
|
||||
|
||||
const std::string name = "\"" + spec + "\" with n_merge=" + std::to_string(n_merge);
|
||||
t.assert_equal("groups for " + name, expected, actual);
|
||||
|
||||
size_t n_bitmap_parts = 0;
|
||||
for (const auto & p : parts) {
|
||||
n_bitmap_parts += p.bitmap != nullptr ? 1 : 0;
|
||||
}
|
||||
t.assert_equal("remaining bitmap parts for " + name, groups.size(), n_bitmap_parts);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_audio
|
||||
//
|
||||
|
||||
MAKE_TEST(test_audio_preprocessor_conformer) {
|
||||
clip_hparams hparams;
|
||||
hparams.n_mel_bins = 128;
|
||||
hparams.audio_sample_rate = 16000;
|
||||
hparams.audio_n_fft = 512;
|
||||
hparams.audio_window_len = 400;
|
||||
hparams.audio_hop_len = 160;
|
||||
|
||||
// 0.4 s of tones, 0.2 s of silence, 0.4 s of a quiet tone
|
||||
const double pi = 3.14159265358979323846;
|
||||
const int sr = hparams.audio_sample_rate;
|
||||
std::vector<float> samples(sr, 0.0f);
|
||||
for (int i = 0; i < sr; i++) {
|
||||
const double ts = (double) i / sr;
|
||||
if (i < 0.4 * sr) {
|
||||
samples[i] = (float) (0.3 * std::sin(2 * pi * 300 * ts) + 0.2 * std::sin(2 * pi * 1200 * ts) + 0.1 * std::sin(2 * pi * 3500 * ts));
|
||||
} else if (i >= 0.6 * sr) {
|
||||
samples[i] = (float) (1e-3 * std::sin(2 * pi * 800 * ts));
|
||||
}
|
||||
}
|
||||
|
||||
mtmd_audio_preprocessor_conformer preproc(hparams);
|
||||
preproc.initialize();
|
||||
std::vector<mtmd_audio_mel> mels;
|
||||
if (!t.assert_true("preprocess", preproc.preprocess(samples.data(), samples.size(), mels))) {
|
||||
return;
|
||||
}
|
||||
const mtmd_audio_mel & mel = mels[0];
|
||||
t.assert_equal("n_len", (int64_t) 101, mel.n_len);
|
||||
|
||||
// reference values from NeMo AudioToMelSpectrogramPreprocessor (liquid-audio 1.3.0)
|
||||
// { mel bin, frame, value }
|
||||
const std::vector<std::tuple<int, int, float>> cases = {
|
||||
{ 8, 0, 3.6104f }, // tones
|
||||
{ 24, 50, -0.6371f }, // silence
|
||||
{ 24, 60, -0.0476f }, // silence -> quiet tone
|
||||
{ 100, 20, -0.0913f }, // tones, high bin
|
||||
};
|
||||
for (const auto & [bin, frame, expected] : cases) {
|
||||
const float actual = mel.data[bin * mel.n_len + frame];
|
||||
t.assert_true("mel[" + std::to_string(bin) + "][" + std::to_string(frame) + "] = " + std::to_string(actual) + ", expected " + std::to_string(expected), std::fabs(actual - expected) < 1e-3f);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// main
|
||||
//
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
testing t(std::cout);
|
||||
t.verbose = true;
|
||||
|
||||
// usage: test-mtmd-impl [filter_regex]
|
||||
for (int i = 1; i < argc; i++) {
|
||||
t.set_filter(argv[i]);
|
||||
}
|
||||
|
||||
for (const auto & e : test_registry::all()) {
|
||||
t.test(e.name, e.fn);
|
||||
}
|
||||
|
||||
return t.summary();
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,25 @@
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
|
||||
#include "peg-parser/tests.h"
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
testing t(std::cout);
|
||||
if (argc >= 2) {
|
||||
t.set_filter(argv[1]);
|
||||
}
|
||||
|
||||
const char * verbose = getenv("LLAMA_TEST_VERBOSE");
|
||||
if (verbose) {
|
||||
t.verbose = std::string(verbose) == "1";
|
||||
}
|
||||
|
||||
t.test("basic", test_basic);
|
||||
t.test("unicode", test_unicode);
|
||||
t.test("json", test_json_parser);
|
||||
t.test("gbnf", test_gbnf_generation);
|
||||
t.test("python-dict", test_python_dict_parser);
|
||||
|
||||
return t.summary();
|
||||
}
|
||||
@@ -0,0 +1,520 @@
|
||||
#include "../src/llama-ext.h"
|
||||
#include "ggml-cpp.h"
|
||||
#include "gguf-model-data.h"
|
||||
#include "llama.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ftype name <-> enum mapping
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct ftype_name_entry {
|
||||
const char * name;
|
||||
llama_ftype ftype;
|
||||
};
|
||||
|
||||
static const ftype_name_entry ftype_name_table[] = {
|
||||
{ "F32", LLAMA_FTYPE_ALL_F32 },
|
||||
{ "F16", LLAMA_FTYPE_MOSTLY_F16 },
|
||||
{ "BF16", LLAMA_FTYPE_MOSTLY_BF16 },
|
||||
{ "Q4_0", LLAMA_FTYPE_MOSTLY_Q4_0 },
|
||||
{ "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1 },
|
||||
{ "Q5_0", LLAMA_FTYPE_MOSTLY_Q5_0 },
|
||||
{ "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1 },
|
||||
{ "Q8_0", LLAMA_FTYPE_MOSTLY_Q8_0 },
|
||||
{ "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K },
|
||||
{ "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S },
|
||||
{ "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S },
|
||||
{ "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M },
|
||||
{ "Q3_K_L", LLAMA_FTYPE_MOSTLY_Q3_K_L },
|
||||
{ "Q4_K_S", LLAMA_FTYPE_MOSTLY_Q4_K_S },
|
||||
{ "Q4_K_M", LLAMA_FTYPE_MOSTLY_Q4_K_M },
|
||||
{ "Q5_K_S", LLAMA_FTYPE_MOSTLY_Q5_K_S },
|
||||
{ "Q5_K_M", LLAMA_FTYPE_MOSTLY_Q5_K_M },
|
||||
{ "Q6_K", LLAMA_FTYPE_MOSTLY_Q6_K },
|
||||
{ "IQ1_S", LLAMA_FTYPE_MOSTLY_IQ1_S },
|
||||
{ "IQ1_M", LLAMA_FTYPE_MOSTLY_IQ1_M },
|
||||
{ "IQ2_XXS", LLAMA_FTYPE_MOSTLY_IQ2_XXS },
|
||||
{ "IQ2_XS", LLAMA_FTYPE_MOSTLY_IQ2_XS },
|
||||
{ "IQ2_S", LLAMA_FTYPE_MOSTLY_IQ2_S },
|
||||
{ "IQ2_M", LLAMA_FTYPE_MOSTLY_IQ2_M },
|
||||
{ "IQ3_XXS", LLAMA_FTYPE_MOSTLY_IQ3_XXS },
|
||||
{ "IQ3_XS", LLAMA_FTYPE_MOSTLY_IQ3_XS },
|
||||
{ "IQ3_S", LLAMA_FTYPE_MOSTLY_IQ3_S },
|
||||
{ "IQ3_M", LLAMA_FTYPE_MOSTLY_IQ3_M },
|
||||
{ "IQ4_NL", LLAMA_FTYPE_MOSTLY_IQ4_NL },
|
||||
{ "IQ4_XS", LLAMA_FTYPE_MOSTLY_IQ4_XS },
|
||||
{ "TQ1_0", LLAMA_FTYPE_MOSTLY_TQ1_0 },
|
||||
{ "TQ2_0", LLAMA_FTYPE_MOSTLY_TQ2_0 },
|
||||
{ "MXFP4_MOE", LLAMA_FTYPE_MOSTLY_MXFP4_MOE },
|
||||
{ "NVFP4", LLAMA_FTYPE_MOSTLY_NVFP4 },
|
||||
};
|
||||
|
||||
static llama_ftype llama_ftype_from_name(const char * name) {
|
||||
for (const auto & e : ftype_name_table) {
|
||||
if (strcmp(name, e.name) == 0) {
|
||||
return e.ftype;
|
||||
}
|
||||
}
|
||||
return (llama_ftype) -1;
|
||||
}
|
||||
|
||||
static const char * llama_ftype_to_name(llama_ftype ftype) {
|
||||
for (const auto & e : ftype_name_table) {
|
||||
if (e.ftype == ftype) {
|
||||
return e.name;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ggml_type name lookup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static ggml_type ggml_type_from_name(const std::string & name) {
|
||||
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
|
||||
const char * tname = ggml_type_name((ggml_type) i);
|
||||
if (tname && name == tname) {
|
||||
return (ggml_type) i;
|
||||
}
|
||||
}
|
||||
return GGML_TYPE_COUNT;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// File parser for snapshot files (quant type schemas)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct snapshot_section {
|
||||
llama_ftype ftype;
|
||||
ggml_type default_type;
|
||||
std::vector<std::pair<std::string, ggml_type>> overrides;
|
||||
};
|
||||
|
||||
// This function is pretty ugly, but it's a trade-off of readable snapshot files
|
||||
// versus readable parsing code
|
||||
static bool parse_snapshot_file(const std::string & path, std::vector<snapshot_section> & sections) {
|
||||
std::ifstream f(path);
|
||||
if (!f.good()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
snapshot_section * cur = nullptr;
|
||||
std::string line;
|
||||
|
||||
while (std::getline(f, line)) {
|
||||
if (line.empty() || line[0] == '#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// section header: [FTYPE_NAME] default_type
|
||||
if (line[0] == '[') {
|
||||
auto close = line.find(']');
|
||||
if (close == std::string::npos) {
|
||||
fprintf(stderr, "parse error: missing ] in '%s'\n", line.c_str());
|
||||
return false;
|
||||
}
|
||||
std::string ftype_str = line.substr(1, close - 1);
|
||||
std::string default_str;
|
||||
size_t pos = close + 1;
|
||||
while (pos < line.size() && line[pos] == ' ') {
|
||||
pos++;
|
||||
}
|
||||
default_str = line.substr(pos);
|
||||
|
||||
llama_ftype ftype = llama_ftype_from_name(ftype_str.c_str());
|
||||
if ((int) ftype < 0) {
|
||||
fprintf(stderr, "parse error: unknown ftype '%s'\n", ftype_str.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
ggml_type dtype = ggml_type_from_name(default_str);
|
||||
if (dtype == GGML_TYPE_COUNT) {
|
||||
fprintf(stderr, "parse error: unknown default type '%s'\n", default_str.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
sections.push_back({ ftype, dtype, {} });
|
||||
cur = §ions.back();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!cur) {
|
||||
fprintf(stderr, "parse error: tensor line before any section: '%s'\n", line.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto sp = line.rfind(' ');
|
||||
if (sp == std::string::npos) {
|
||||
fprintf(stderr, "parse error: no space in tensor line: '%s'\n", line.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string tname = line.substr(0, sp);
|
||||
std::string ttype = line.substr(sp + 1);
|
||||
|
||||
ggml_type gt = ggml_type_from_name(ttype);
|
||||
if (gt == GGML_TYPE_COUNT) {
|
||||
fprintf(stderr, "parse error: unknown type '%s' for tensor '%s'\n", ttype.c_str(), tname.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
cur->overrides.push_back({ tname, gt });
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Remote model support using gguf-model-data.cpp
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct remote_model_spec {
|
||||
const char * repo;
|
||||
const char * quant;
|
||||
};
|
||||
|
||||
// Get model name from repo: strip org prefix, strip -GGUF suffix,
|
||||
// and strip anything up to and including first '_' (e.g. "deepseek-ai_DeepSeek-V3.1").
|
||||
static std::string model_name_from_repo(const char * repo) {
|
||||
std::string s(repo);
|
||||
|
||||
auto slash = s.find('/');
|
||||
if (slash != std::string::npos) {
|
||||
s = s.substr(slash + 1);
|
||||
}
|
||||
|
||||
const std::string suffix = "-GGUF";
|
||||
if (s.size() >= suffix.size() && s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0) {
|
||||
s = s.substr(0, s.size() - suffix.size());
|
||||
}
|
||||
|
||||
auto underscore = s.find('_');
|
||||
if (underscore != std::string::npos) {
|
||||
s = s.substr(underscore + 1);
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
static std::string snapshot_file_from_name(const std::string & name) {
|
||||
std::string lower = name;
|
||||
for (auto & c : lower) {
|
||||
c = std::tolower(c);
|
||||
}
|
||||
return lower;
|
||||
}
|
||||
|
||||
static const remote_model_spec model_specs[] = {
|
||||
{ "ggml-org/Qwen3-0.6B-GGUF", "Q8_0" },
|
||||
{ "ggml-org/GLM-4.6V-GGUF", "Q8_0" },
|
||||
{ "ggml-org/Step-3.5-Flash-GGUF", "Q4_K" },
|
||||
{ "ggml-org/Qwen3-Coder-Next-GGUF", "Q8_0" },
|
||||
{ "ggml-org/Qwen3-14B-GGUF", "Q8_0" },
|
||||
{ "ggml-org/NVIDIA-Nemotron-3-Nano-30B-A3B-GGUF", "Q8_0" },
|
||||
{ "ggml-org/gpt-oss-120b-GGUF", "mxfp4" },
|
||||
{ "ggml-org/gemma-3-4b-it-GGUF", "Q8_0" },
|
||||
{ "bartowski/Meta-Llama-3.1-70B-Instruct-GGUF", "Q4_K_M" },
|
||||
{ "bartowski/deepseek-ai_DeepSeek-V3.1-GGUF", "IQ1_M" },
|
||||
//{ "bartowski/Qwen_Qwen3.5-397B-A17B-GGUF", "IQ1_S" }, // TODO: swap with ggml-org if/when it's released
|
||||
{ "ggml-org/Qwen3.6-27B-GGUF", "Q8_0" },
|
||||
};
|
||||
|
||||
static const int n_model_specs = (int) (sizeof(model_specs) / sizeof(model_specs[0]));
|
||||
|
||||
static llama_model * build_mock_model_from_remote(const gguf_remote_model & remote) {
|
||||
llama_quant_model_desc desc = {};
|
||||
desc.architecture = remote.architecture.c_str();
|
||||
desc.n_embd = remote.n_embd;
|
||||
desc.n_ff = remote.n_ff;
|
||||
desc.n_layer = remote.n_layer;
|
||||
desc.n_head = remote.n_head;
|
||||
desc.n_head_kv = remote.n_head_kv;
|
||||
desc.n_expert = remote.n_expert;
|
||||
desc.n_embd_head_k = remote.n_embd_head_k;
|
||||
desc.n_embd_head_v = remote.n_embd_head_v;
|
||||
return llama_quant_model_from_metadata(&desc);
|
||||
}
|
||||
|
||||
// Single ggml context holding all quantizable tensors for a model.
|
||||
struct mock_tensors {
|
||||
ggml_context_ptr ctx;
|
||||
std::vector<ggml_tensor *> tensors;
|
||||
};
|
||||
|
||||
static mock_tensors build_mock_tensors(const quantize_state_impl * qs, const gguf_remote_model & remote) {
|
||||
const size_t ctx_size = remote.tensors.size() * ggml_tensor_overhead();
|
||||
struct ggml_init_params params = { ctx_size, nullptr, true };
|
||||
ggml_context_ptr ctx(ggml_init(params));
|
||||
|
||||
std::vector<ggml_tensor *> result;
|
||||
|
||||
for (const auto & t : remote.tensors) {
|
||||
ggml_tensor * gt = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F32, t.ne[0], t.ne[1], t.ne[2], t.ne[3]);
|
||||
ggml_set_name(gt, t.name.c_str());
|
||||
if (llama_quant_tensor_allows_quantization(qs, gt)) {
|
||||
result.push_back(gt);
|
||||
}
|
||||
}
|
||||
|
||||
// sort by layer index then name, matching llama_model_loader::weight_name_comparer
|
||||
std::sort(result.begin(), result.end(), [](const ggml_tensor * a, const ggml_tensor * b) {
|
||||
int a_layer = -1, b_layer = -1;
|
||||
sscanf(a->name, "blk.%d.", &a_layer);
|
||||
sscanf(b->name, "blk.%d.", &b_layer);
|
||||
if (a_layer != b_layer) {
|
||||
return a_layer < b_layer;
|
||||
}
|
||||
return strcmp(a->name, b->name) < 0;
|
||||
});
|
||||
|
||||
return { std::move(ctx), std::move(result) };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generate mode: regenerate all snapshot files
|
||||
// Use this when either adding new models or modifying quants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static std::string generate_snapshot(const std::string & name,
|
||||
const gguf_remote_model & remote,
|
||||
quantize_state_impl * qs,
|
||||
mock_tensors & mt) {
|
||||
std::ostringstream out;
|
||||
|
||||
out << "# Model: " << name << "\n";
|
||||
out << "# n_embd=" << remote.n_embd << ", n_ff=" << remote.n_ff << ", n_vocab=" << remote.n_vocab
|
||||
<< ", n_layer=" << remote.n_layer << ", n_head=" << remote.n_head << ", n_head_kv=" << remote.n_head_kv;
|
||||
if (remote.n_expert > 0) {
|
||||
out << ", n_expert=" << remote.n_expert;
|
||||
}
|
||||
out << "\n";
|
||||
|
||||
for (int i = 0; i < LLAMA_FTYPE_GUESSED; i++) {
|
||||
llama_ftype ft = (llama_ftype) i;
|
||||
ggml_type default_type = llama_ftype_get_default_type(ft);
|
||||
if (default_type == GGML_TYPE_COUNT) {
|
||||
continue;
|
||||
}
|
||||
const char * fname = llama_ftype_to_name(ft);
|
||||
if (!fname) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::vector<ggml_type> result_types(mt.tensors.size());
|
||||
llama_quant_compute_types(qs, ft, mt.tensors.data(), result_types.data(), mt.tensors.size());
|
||||
|
||||
out << "\n[" << fname << "] " << ggml_type_name(default_type) << "\n";
|
||||
for (size_t j = 0; j < mt.tensors.size(); j++) {
|
||||
if (result_types[j] != default_type) {
|
||||
out << ggml_get_name(mt.tensors[j]) << " " << ggml_type_name(result_types[j]) << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return out.str();
|
||||
}
|
||||
|
||||
static int run_generate(const std::string & snapshot_dir) {
|
||||
fprintf(stderr, "This will overwrite all snapshot files in:\n %s\n", snapshot_dir.c_str());
|
||||
fprintf(stderr, "Continue? [y/N] ");
|
||||
int ch = fgetc(stdin);
|
||||
if (ch != 'y' && ch != 'Y') {
|
||||
fprintf(stderr, "Aborted.\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
fprintf(stderr, "\n");
|
||||
|
||||
int n_written = 0;
|
||||
|
||||
for (int m = 0; m < n_model_specs; m++) {
|
||||
const auto & spec = model_specs[m];
|
||||
std::string name = model_name_from_repo(spec.repo);
|
||||
|
||||
fprintf(stderr, "Fetching model metadata for %s from %s...\n", name.c_str(), spec.repo);
|
||||
auto result = gguf_fetch_model_meta(spec.repo, spec.quant);
|
||||
if (!result.has_value()) {
|
||||
fprintf(stderr, "ERROR: could not fetch model metadata for %s\n", name.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
const auto & remote = result.value();
|
||||
llama_model * model = build_mock_model_from_remote(remote);
|
||||
llama_model_quantize_params qparams = llama_model_quantize_default_params();
|
||||
quantize_state_impl * qs = llama_quant_init(model, &qparams);
|
||||
auto mt = build_mock_tensors(qs, remote);
|
||||
|
||||
std::string content = generate_snapshot(name, remote, qs, mt);
|
||||
std::string path = snapshot_dir + "/" + snapshot_file_from_name(name) + ".schema";
|
||||
|
||||
std::ofstream f(path);
|
||||
if (!f.good()) {
|
||||
fprintf(stderr, "ERROR: could not write %s\n", path.c_str());
|
||||
llama_quant_free(qs);
|
||||
llama_model_free(model);
|
||||
return 1;
|
||||
}
|
||||
f << content;
|
||||
n_written++;
|
||||
fprintf(stderr, " wrote %s\n", path.c_str());
|
||||
llama_quant_free(qs);
|
||||
llama_model_free(model);
|
||||
}
|
||||
|
||||
fprintf(stderr, "%d files written\n", n_written);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test mode: compare against snapshot files
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static bool run_test_section(quantize_state_impl * qs, mock_tensors & mt, const snapshot_section & section) {
|
||||
// verify default_type matches what llama_ftype_get_default_type returns
|
||||
ggml_type computed_default = llama_ftype_get_default_type(section.ftype);
|
||||
if (computed_default != section.default_type) {
|
||||
printf(" FAIL [%s] default type mismatch: file says %s, code says %s\n", llama_ftype_to_name(section.ftype),
|
||||
ggml_type_name(section.default_type), ggml_type_name(computed_default));
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<ggml_type> result_types(mt.tensors.size());
|
||||
llama_quant_compute_types(qs, section.ftype, mt.tensors.data(), result_types.data(), mt.tensors.size());
|
||||
|
||||
std::map<std::string, ggml_type> override_map(section.overrides.begin(), section.overrides.end());
|
||||
|
||||
bool all_pass = true;
|
||||
int n_override_found = 0;
|
||||
|
||||
for (size_t i = 0; i < mt.tensors.size(); i++) {
|
||||
const char * name = ggml_get_name(mt.tensors[i]);
|
||||
ggml_type got = result_types[i];
|
||||
|
||||
ggml_type expected = section.default_type;
|
||||
auto it = override_map.find(name);
|
||||
if (it != override_map.end()) {
|
||||
expected = it->second;
|
||||
n_override_found++;
|
||||
}
|
||||
|
||||
if (got != expected) {
|
||||
printf(" FAIL %-50s %-10s expected %s, got %s\n", name, llama_ftype_to_name(section.ftype),
|
||||
ggml_type_name(expected), ggml_type_name(got));
|
||||
all_pass = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (n_override_found != (int) section.overrides.size()) {
|
||||
printf(" FAIL [%s] override count mismatch: listed %d, matched %d\n", llama_ftype_to_name(section.ftype),
|
||||
(int) section.overrides.size(), n_override_found);
|
||||
all_pass = false;
|
||||
}
|
||||
|
||||
return all_pass;
|
||||
}
|
||||
|
||||
static int run_remote_tests(const std::string & snapshot_dir, const char * argv0) {
|
||||
int total_pass = 0;
|
||||
int total_fail = 0;
|
||||
int total_skip = 0;
|
||||
|
||||
for (int m = 0; m < n_model_specs; m++) {
|
||||
const auto & spec = model_specs[m];
|
||||
std::string name = model_name_from_repo(spec.repo);
|
||||
printf("=== %s ===\n", name.c_str());
|
||||
|
||||
auto result = gguf_fetch_model_meta(spec.repo, spec.quant, "", false);
|
||||
if (!result.has_value()) {
|
||||
printf(" SKIP (could not fetch model metadata)\n\n");
|
||||
total_skip++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto & remote = result.value();
|
||||
llama_model * model = build_mock_model_from_remote(remote);
|
||||
llama_model_quantize_params qparams = llama_model_quantize_default_params();
|
||||
quantize_state_impl * qs = llama_quant_init(model, &qparams);
|
||||
auto mt = build_mock_tensors(qs, remote);
|
||||
|
||||
std::string snapshot_path = snapshot_dir + "/" + snapshot_file_from_name(name) + ".schema";
|
||||
std::vector<snapshot_section> sections;
|
||||
if (!parse_snapshot_file(snapshot_path, sections)) {
|
||||
printf(" SKIP (could not read snapshot file: %s)\n\n", snapshot_path.c_str());
|
||||
llama_quant_free(qs);
|
||||
llama_model_free(model);
|
||||
total_skip++;
|
||||
continue;
|
||||
}
|
||||
|
||||
int model_pass = 0;
|
||||
int model_fail = 0;
|
||||
|
||||
for (const auto & section : sections) {
|
||||
bool pass = run_test_section(qs, mt, section);
|
||||
if (pass) {
|
||||
model_pass++;
|
||||
} else {
|
||||
model_fail++;
|
||||
}
|
||||
}
|
||||
|
||||
printf(" %s %s: %d/%d ftype sections passed (%d tensors)\n", model_fail == 0 ? "PASS" : "FAIL", name.c_str(),
|
||||
model_pass, model_pass + model_fail, (int) mt.tensors.size());
|
||||
printf("\n");
|
||||
|
||||
if (model_fail == 0) {
|
||||
total_pass++;
|
||||
} else {
|
||||
total_fail++;
|
||||
}
|
||||
|
||||
llama_quant_free(qs);
|
||||
llama_model_free(model);
|
||||
}
|
||||
|
||||
printf("%d/%d models passed", total_pass, total_pass + total_fail);
|
||||
if (total_skip > 0) {
|
||||
printf(", %d skipped", total_skip);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
if (total_fail > 0) {
|
||||
printf("\nIf these changes are intentional, regenerate snapshot files with:\n");
|
||||
printf(" %s --generate\n", argv0);
|
||||
}
|
||||
|
||||
return total_fail > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
std::string snapshot_dir = SNAPSHOT_DIR;
|
||||
bool generate = false;
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (strcmp(argv[i], "--generate") == 0) {
|
||||
generate = true;
|
||||
} else if (strcmp(argv[i], "--snapshot-dir") == 0 && i + 1 < argc) {
|
||||
snapshot_dir = argv[++i];
|
||||
}
|
||||
}
|
||||
|
||||
if (generate) {
|
||||
return run_generate(snapshot_dir);
|
||||
}
|
||||
|
||||
// suppress llama log warnings during test (e.g. tensor type fallback messages)
|
||||
llama_log_set([](enum ggml_log_level, const char *, void *) {}, nullptr);
|
||||
|
||||
return run_remote_tests(snapshot_dir, argv[0]);
|
||||
}
|
||||
Executable
+178
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [ $# -lt 2 ]; then
|
||||
printf "Usage: %s <llama-quantize> <llama-perplexity> [<scratch-dir>] [--no-check]\n" "$0"
|
||||
printf " <scratch-dir> dir for the model and outputs, default /tmp/test-quantize-be\n"
|
||||
printf " --no-check print the perplexity of each output without checking it\n"
|
||||
printf "Example: %s ./build/bin/llama-quantize ./build/bin/llama-perplexity\n" "$0"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
quantize=$1
|
||||
perplexity=$2
|
||||
shift 2
|
||||
|
||||
scratch_dir=/tmp/test-quantize-be
|
||||
no_check=
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--no-check) no_check=--no-check ;;
|
||||
*) scratch_dir=$arg ;;
|
||||
esac
|
||||
done
|
||||
|
||||
for bin in "$quantize" "$perplexity"; do
|
||||
if [ ! -x "$bin" ]; then
|
||||
echo "error: $bin not found, build it first" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$(uname -m)" != s390x ]; then
|
||||
echo "error: this test needs a big-endian s390x host" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
MODEL_URL=https://huggingface.co/ggml-org/models/resolve/499bc8821c6b12b4e53c5bffcb21ec206f212d81/tinyllamas/stories15M.gguf
|
||||
MODEL_SHA256=61b50d457809a5194818fd22e6724b456cd7bb9a6264c52c8110684c53f3704a
|
||||
F32_SHA256=55174d7a7d46f29fc6be163ccb040330c357aa727b22ec13e2c5d713e5ba5b76
|
||||
|
||||
# perplexity ceiling of each big-endian output on the text below, about 10% above the measured value
|
||||
# a wrong byte order gives nan or a perplexity in the hundreds
|
||||
# F32 is the converted input, all other types come from llama-quantize --pure
|
||||
declare -a outputs=(
|
||||
"F32 3.60"
|
||||
"F16 3.60"
|
||||
"BF16 3.59"
|
||||
"Q4_0 4.14"
|
||||
"Q4_1 4.01"
|
||||
"Q5_0 3.82"
|
||||
"Q5_1 3.65"
|
||||
"Q8_0 3.60"
|
||||
"Q2_K 5.24"
|
||||
"Q3_K 4.25"
|
||||
"Q4_K 3.83"
|
||||
"Q5_K 3.67"
|
||||
"Q6_K 3.60"
|
||||
"IQ4_NL 3.86"
|
||||
"IQ4_XS 3.85"
|
||||
"MXFP4_MOE 4.27"
|
||||
)
|
||||
|
||||
check_sha256() {
|
||||
local name=$1
|
||||
local file=$2
|
||||
local expected=$3
|
||||
|
||||
local actual
|
||||
actual=$(sha256sum "$file" | cut -d ' ' -f 1)
|
||||
|
||||
if [ "$actual" != "$expected" ]; then
|
||||
echo "error: sha256 mismatch for $name: expected '$expected', got '$actual'" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# keep the report readable, only show a command's output when it fails
|
||||
run_logged() {
|
||||
local log=$1
|
||||
shift
|
||||
|
||||
if ! "$@" > "$log" 2>&1; then
|
||||
cat "$log" >&2
|
||||
echo "error: command failed: $*" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# quantize one type and print its perplexity, print nothing if a step fails
|
||||
measure_ppl() {
|
||||
local type=$1
|
||||
local out=$2
|
||||
local log=$scratch_dir/stories15M-be-$type-ppl.log
|
||||
|
||||
if [ "$type" != F32 ]; then
|
||||
run_logged "$scratch_dir/stories15M-be-$type.log" "$quantize" --pure "$f32" "$out" "$type" || return 1
|
||||
fi
|
||||
run_logged "$log" "$perplexity" -m "$out" -f "$text" -c 128 || return 1
|
||||
|
||||
# llama-perplexity prints no final estimate when the result is nan
|
||||
sed -n 's/.*Final estimate: PPL = \([0-9.]*\).*/\1/p' "$log"
|
||||
}
|
||||
|
||||
mkdir -p "$scratch_dir"
|
||||
|
||||
model=$scratch_dir/stories15M.gguf
|
||||
if [ ! -f "$model" ]; then
|
||||
wget -q -O "$model" "$MODEL_URL"
|
||||
fi
|
||||
|
||||
check_sha256 stories15M.gguf "$model" "$MODEL_SHA256"
|
||||
|
||||
f32=$scratch_dir/stories15M-be-F32.gguf
|
||||
cp "$model" "$f32"
|
||||
echo YES | run_logged "$scratch_dir/stories15M-be-F32.log" python3 "$(dirname "$0")/../gguf-py/gguf/scripts/gguf_convert_endian.py" "$f32" big || exit 1
|
||||
check_sha256 F32 "$f32" "$F32_SHA256"
|
||||
|
||||
# 588 tokens, enough for 4 chunks of 128
|
||||
text=$scratch_dir/stories.txt
|
||||
cat > "$text" << 'EOF'
|
||||
Once upon a time, there was a little girl named Lily. She lived in a small house with her mom and dad. Lily loved to play outside in the garden. One sunny day, she saw a big red ball under a tree. She ran to the ball and picked it up. It was soft and round.
|
||||
Lily wanted to play with her friend Tom. She went to his house and knocked on the door. Tom opened the door and smiled. "Hi, Lily! Do you want to play?" he asked. "Yes! Look at my new ball," said Lily. They went to the park together.
|
||||
At the park, Lily and Tom threw the ball to each other. They laughed and ran on the green grass. Then the ball went too high. It flew over a fence and into a pond. Lily was sad. "Oh no, my ball is in the water," she said.
|
||||
Tom had an idea. He found a long stick near the pond. He held the stick out over the water and pushed the ball to the side. Lily helped him. Soon the ball came close and Lily picked it up. It was wet, but it was not broken.
|
||||
"Thank you, Tom," said Lily. "You are a good friend." Tom smiled and said, "You are welcome. Let's play again, but not near the pond." They both laughed and went back to the grass.
|
||||
When the sun went down, Lily's mom called her home. Lily said goodbye to Tom and walked home with her ball. She told her mom about the pond and the stick. Her mom gave her a big hug. "I am happy you and Tom helped each other," she said.
|
||||
That night, Lily put the ball next to her bed. She thought about the fun day at the park. She closed her eyes and smiled. Lily knew that sharing and helping made every day better.
|
||||
The next morning, Lily woke up early. She saw a little bird outside her window. The bird was singing a happy song. Lily wanted to give the bird some food. She asked her dad for some bread. Her dad gave her a small piece, and Lily put it on the window.
|
||||
The bird hopped over and ate the bread. Then it sang again, louder than before. Lily clapped her hands. "Thank you for the song, little bird," she said. The bird flew away, but it came back every morning after that.
|
||||
Lily and the bird became good friends. Every day she gave it bread, and every day it sang for her. Lily learned that being kind to others, even small birds, can make the world a happy place.
|
||||
EOF
|
||||
|
||||
rows=()
|
||||
n_ok=0
|
||||
n_bad=0
|
||||
for entry in "${outputs[@]}"; do
|
||||
read -r type max <<< "$entry"
|
||||
out=$scratch_dir/stories15M-be-$type.gguf
|
||||
|
||||
if ! ppl=$(measure_ppl "$type" "$out"); then
|
||||
ppl=-
|
||||
status=FAIL
|
||||
elif [ "$no_check" = --no-check ]; then
|
||||
status=-
|
||||
elif [[ $ppl =~ ^[0-9.]+$ ]] && awk -v p="$ppl" -v m="$max" 'BEGIN { exit !(p <= m) }'; then
|
||||
status=OK
|
||||
else
|
||||
status=FAIL
|
||||
fi
|
||||
|
||||
case $status in
|
||||
OK) n_ok=$((n_ok + 1)) ;;
|
||||
FAIL) n_bad=$((n_bad + 1)) ;;
|
||||
esac
|
||||
|
||||
rows+=("$type ${ppl:-nan} $max $status")
|
||||
printf "%s: %-10s (%s) done\n" "$(basename "$0")" "$type" "$out"
|
||||
done
|
||||
|
||||
printf "\n%-10s %10s %10s %s\n" type ppl max status
|
||||
for row in "${rows[@]}"; do
|
||||
read -r type ppl max status <<< "$row"
|
||||
case $status in
|
||||
OK) status="\033[32mOK\033[0m" ;;
|
||||
FAIL) status="\033[31mFAIL\033[0m" ;;
|
||||
esac
|
||||
printf "%-10s %10s %10s %b\n" "$type" "$ppl" "$max" "$status"
|
||||
done
|
||||
|
||||
if [ "$no_check" != --no-check ]; then
|
||||
printf "summary: %d ok, %d failed\n" "$n_ok" "$n_bad"
|
||||
fi
|
||||
|
||||
if [ "$n_bad" -ne 0 ]; then
|
||||
printf "\nif the perplexity is expected to change, run with --no-check and update the max values in %s:\n\n" "$0"
|
||||
printf "%s %s %s %s --no-check\n" "$0" "$quantize" "$perplexity" "$scratch_dir"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,310 @@
|
||||
// Unit tests for quantization specific functions - quantize, dequantize and dot product
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#undef NDEBUG
|
||||
#include <assert.h>
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
constexpr float MAX_QUANTIZATION_REFERENCE_ERROR = 0.0001f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR = 0.002f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_BINARY = 0.025f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_TERNARY = 0.01f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_2BITS = 0.0075f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS = 0.0040f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS = 0.0050f;
|
||||
constexpr float MAX_QUANTIZATION_TOTAL_ERROR_FP4 = 0.0030f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR = 0.02f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR_LOWBIT = 0.04f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR_FP4 = 0.03f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR_BINARY = 0.40f;
|
||||
constexpr float MAX_DOT_PRODUCT_ERROR_TERNARY = 0.15f;
|
||||
|
||||
static const char* RESULT_STR[] = {"ok", "FAILED"};
|
||||
|
||||
|
||||
// Generate synthetic data
|
||||
static void generate_data(float offset, size_t n, float * dst, float amplitude = 2.0f) {
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
dst[i] = 0.1 + amplitude*cosf(i + offset);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate RMSE between two float arrays
|
||||
static float array_rmse(const float * a1, const float * a2, size_t n) {
|
||||
double sum = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
double diff = a1[i] - a2[i];
|
||||
sum += diff * diff;
|
||||
}
|
||||
return sqrtf(sum) / n;
|
||||
}
|
||||
|
||||
// Total quantization error on test data
|
||||
static float total_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {
|
||||
std::vector<uint8_t> tmp_q(2*test_size);
|
||||
std::vector<float> tmp_out(test_size);
|
||||
|
||||
qfns_cpu->from_float(test_data, tmp_q.data(), test_size);
|
||||
qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);
|
||||
return array_rmse(test_data, tmp_out.data(), test_size);
|
||||
}
|
||||
|
||||
// Total quantization error on test data
|
||||
static float reference_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {
|
||||
std::vector<uint8_t> tmp_q(2*test_size);
|
||||
std::vector<float> tmp_out(test_size);
|
||||
std::vector<float> tmp_out_ref(test_size);
|
||||
|
||||
// FIXME: why is done twice?
|
||||
qfns_cpu->from_float(test_data, tmp_q.data(), test_size);
|
||||
qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);
|
||||
|
||||
qfns->from_float_ref(test_data, tmp_q.data(), test_size);
|
||||
qfns->to_float(tmp_q.data(), tmp_out_ref.data(), test_size);
|
||||
|
||||
return array_rmse(tmp_out.data(), tmp_out_ref.data(), test_size);
|
||||
}
|
||||
|
||||
static float dot_product(const float * a1, const float * a2, size_t test_size) {
|
||||
double sum = 0;
|
||||
for (size_t i = 0; i < test_size; i++) {
|
||||
sum += a1[i] * a2[i];
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// Total dot product error
|
||||
static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type src0_type, size_t test_size,
|
||||
const float * test_data1, const float * test_data2,
|
||||
const float * test_data3, const float * test_data4,
|
||||
const int nrc) {
|
||||
const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type);
|
||||
const size_t pad = 64;
|
||||
const size_t bx = ggml_row_size(src0_type, test_size) + pad;
|
||||
const size_t by = ggml_row_size(qfns_cpu->vec_dot_type, test_size) + pad;
|
||||
|
||||
std::vector<uint8_t> tmp_q1(bx * nrc);
|
||||
std::vector<uint8_t> tmp_q2(by * nrc);
|
||||
|
||||
qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size);
|
||||
vdot->from_float(test_data2, tmp_q2.data(), test_size);
|
||||
|
||||
if (nrc == 1) {
|
||||
float result = INFINITY;
|
||||
qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1);
|
||||
|
||||
const float dot_ref = dot_product(test_data1, test_data2, test_size);
|
||||
return fabsf(result - dot_ref) / test_size;
|
||||
}
|
||||
|
||||
// nrc == 2: kernel computes a 2x2 dot product matrix
|
||||
// Output layout: s[0]=dot(vx0,vy0), s[1]=dot(vx1,vy0), s[bs]=dot(vx0,vy1), s[bs+1]=dot(vx1,vy1)
|
||||
// row and output strides are padded, same as in the mul_mat path
|
||||
qfns_cpu->from_float(test_data3, tmp_q1.data() + bx, test_size);
|
||||
vdot->from_float(test_data4, tmp_q2.data() + by, test_size);
|
||||
|
||||
const size_t bs = 16;
|
||||
std::vector<float> result(bs + 2, INFINITY);
|
||||
qfns_cpu->vec_dot(test_size, result.data(), bs, tmp_q1.data(), bx, tmp_q2.data(), by, 2);
|
||||
|
||||
const float ref00 = dot_product(test_data1, test_data2, test_size);
|
||||
const float ref10 = dot_product(test_data3, test_data2, test_size);
|
||||
const float ref01 = dot_product(test_data1, test_data4, test_size);
|
||||
const float ref11 = dot_product(test_data3, test_data4, test_size);
|
||||
|
||||
const auto err = [test_size](float val, float ref) {
|
||||
const float e = fabsf(val - ref) / test_size;
|
||||
return std::isfinite(e) ? e : INFINITY;
|
||||
};
|
||||
|
||||
return std::max({err(result[0], ref00), err(result[1], ref10), err(result[bs], ref01), err(result[bs + 1], ref11)});
|
||||
}
|
||||
|
||||
static int test_vec_dot_f32(bool verbose) {
|
||||
const auto * f32 = ggml_get_type_traits_cpu(GGML_TYPE_F32);
|
||||
int num_failed = 0;
|
||||
for (int n : {1, 2, 3, 5, 7, 8, 15, 16, 17, 31, 33, 63, 67, 127, 129, 193, 255, 1023}) {
|
||||
std::vector<float> a(n);
|
||||
std::vector<float> b(n);
|
||||
generate_data(0.0, n, a.data());
|
||||
generate_data(1.0, n, b.data());
|
||||
|
||||
float result = 0.0f;
|
||||
f32->vec_dot(n, &result, 0, a.data(), 0, b.data(), 0, 1);
|
||||
const float ref = dot_product(a.data(), b.data(), n);
|
||||
const float error = fabsf(result - ref) / n;
|
||||
|
||||
const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR);
|
||||
num_failed += failed;
|
||||
if (failed || verbose) {
|
||||
printf(" f32 vec_dot n=%4d: %s (ref=%f got=%f err=%f)\n",
|
||||
n, RESULT_STR[failed], ref, result, error);
|
||||
}
|
||||
}
|
||||
return num_failed;
|
||||
}
|
||||
|
||||
static int test_vec_dot_q(bool verbose) {
|
||||
int num_failed = 0;
|
||||
|
||||
const size_t test_size = 32 * 128;
|
||||
|
||||
std::vector<float> test_data(test_size);
|
||||
std::vector<float> test_data2(test_size);
|
||||
std::vector<float> test_data3(test_size);
|
||||
std::vector<float> test_data4(test_size);
|
||||
|
||||
generate_data(0.0, test_data.size(), test_data.data());
|
||||
generate_data(1.0, test_data2.size(), test_data2.data());
|
||||
generate_data(3.0, test_data3.size(), test_data3.data(), 1.0f);
|
||||
generate_data(4.0, test_data4.size(), test_data4.data(), 1.5f);
|
||||
|
||||
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
|
||||
ggml_type type = (ggml_type) i;
|
||||
const auto * qfns = ggml_get_type_traits(type);
|
||||
const auto * qfns_cpu = ggml_get_type_traits_cpu(type);
|
||||
|
||||
// deprecated - skip
|
||||
if (qfns->blck_size == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const ggml_type ei = (ggml_type)i;
|
||||
|
||||
printf("Testing %s\n", ggml_type_name((ggml_type) i));
|
||||
ggml_quantize_init(ei);
|
||||
|
||||
if (qfns_cpu->from_float && qfns->to_float) {
|
||||
const float total_error = total_quantization_error(qfns, qfns_cpu, test_size, test_data.data());
|
||||
const float max_quantization_error =
|
||||
type == GGML_TYPE_Q1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY :
|
||||
type == GGML_TYPE_TQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
|
||||
type == GGML_TYPE_TQ2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
|
||||
type == GGML_TYPE_Q2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :
|
||||
type == GGML_TYPE_Q2_K ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :
|
||||
type == GGML_TYPE_IQ2_S ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :
|
||||
type == GGML_TYPE_Q3_K ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :
|
||||
type == GGML_TYPE_IQ3_S ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :
|
||||
type == GGML_TYPE_IQ3_XXS ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS :
|
||||
type == GGML_TYPE_NVFP4 ? MAX_QUANTIZATION_TOTAL_ERROR_FP4 : MAX_QUANTIZATION_TOTAL_ERROR;
|
||||
bool failed = !(total_error < max_quantization_error);
|
||||
num_failed += failed;
|
||||
if (failed || verbose) {
|
||||
printf("%5s absolute quantization error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error);
|
||||
}
|
||||
|
||||
const float reference_error = reference_quantization_error(qfns, qfns_cpu, test_size, test_data.data());
|
||||
failed = !(reference_error < MAX_QUANTIZATION_REFERENCE_ERROR);
|
||||
num_failed += failed;
|
||||
if (failed || verbose) {
|
||||
printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error);
|
||||
}
|
||||
|
||||
const float vec_dot_error = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), nullptr, nullptr, 1);
|
||||
const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS ||
|
||||
type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S
|
||||
? MAX_DOT_PRODUCT_ERROR_LOWBIT
|
||||
: type == GGML_TYPE_Q1_0
|
||||
? MAX_DOT_PRODUCT_ERROR_BINARY
|
||||
: type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0
|
||||
? MAX_DOT_PRODUCT_ERROR_TERNARY
|
||||
: type == GGML_TYPE_NVFP4
|
||||
? MAX_DOT_PRODUCT_ERROR_FP4
|
||||
: MAX_DOT_PRODUCT_ERROR;
|
||||
failed = !(vec_dot_error < max_allowed_error);
|
||||
num_failed += failed;
|
||||
if (failed || verbose) {
|
||||
printf("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error);
|
||||
}
|
||||
|
||||
// Test nrc=2 path for types that support it
|
||||
if (qfns_cpu->nrows == 2) {
|
||||
const float vec_dot_error_nrc2 = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), test_data3.data(), test_data4.data(), 2);
|
||||
failed = !(vec_dot_error_nrc2 < max_allowed_error);
|
||||
num_failed += failed;
|
||||
if (failed || verbose) {
|
||||
printf("%5s dot product error (nrc=2): %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return num_failed;
|
||||
}
|
||||
|
||||
// In every group, all values with importance are equal, and the max (0) has none.
|
||||
// The scale search in make_qkx3_quants then fits min == max and passes inf/nan to nearest_int (#29804).
|
||||
static int test_quantize_imatrix_degenerate(bool verbose) {
|
||||
const int64_t n = 256;
|
||||
std::vector<float> x(n);
|
||||
std::vector<float> imatrix(n);
|
||||
std::vector<float> out(n);
|
||||
int num_failed = 0;
|
||||
|
||||
printf("Testing degenerate imatrix:\n");
|
||||
for (ggml_type type : {GGML_TYPE_Q2_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1}) {
|
||||
const int64_t group = type == GGML_TYPE_Q2_K ? 16 : 32;
|
||||
for (int64_t i = 0; i < n; ++i) {
|
||||
const int64_t g = i / group;
|
||||
const int64_t p = i % group;
|
||||
const bool important = p % 3 == 1;
|
||||
x[i] = important ? -0.02f*(g + 1) : (p % 2 ? -1.0f : 0.0f);
|
||||
imatrix[i] = important ? 1.0f : 0.0f;
|
||||
}
|
||||
printf(" - %s\n", ggml_type_name(type));
|
||||
|
||||
std::vector<uint8_t> q(ggml_row_size(type, n));
|
||||
ggml_quantize_init(type);
|
||||
ggml_quantize_chunk(type, x.data(), q.data(), 0, 1, n, imatrix.data());
|
||||
ggml_get_type_traits(type)->to_float(q.data(), out.data(), n);
|
||||
|
||||
const bool failed = !std::all_of(out.begin(), out.end(), [](float v) { return std::isfinite(v); });
|
||||
num_failed += failed;
|
||||
if (failed || verbose) {
|
||||
printf("%5s imatrix degenerate groups: %s\n", ggml_type_name(type), RESULT_STR[failed]);
|
||||
}
|
||||
}
|
||||
|
||||
return num_failed;
|
||||
}
|
||||
|
||||
int main(int argc, char * argv[]) {
|
||||
bool verbose = false;
|
||||
|
||||
std::string arg;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
arg = argv[i];
|
||||
|
||||
if (arg == "-v") {
|
||||
verbose = true;
|
||||
} else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
ggml_cpu_init();
|
||||
|
||||
int num_failed = 0;
|
||||
|
||||
num_failed += test_vec_dot_f32(verbose);
|
||||
num_failed += test_vec_dot_q(verbose);
|
||||
num_failed += test_quantize_imatrix_degenerate(verbose);
|
||||
|
||||
if (num_failed || verbose) {
|
||||
printf("%d tests failed\n", num_failed);
|
||||
}
|
||||
|
||||
return num_failed > 0;
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
// Benchmark quantization specific functions on synthetic data
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#undef NDEBUG
|
||||
#include <algorithm>
|
||||
#include <assert.h>
|
||||
#include <functional>
|
||||
#include <math.h>
|
||||
#include <memory>
|
||||
#include <stdio.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
#define MAX_ALIGNMENT 64
|
||||
#define QK 32
|
||||
#define WARMUP 5
|
||||
#define ITERATIONS 10
|
||||
#define MAX_ITERATIONS 100000000
|
||||
|
||||
#define L1_SIZE 32*128
|
||||
#define L2_SIZE 32*2048
|
||||
#define L3_SIZE 32*20480
|
||||
#define MEM_SIZE 32*2048000
|
||||
|
||||
struct quantize_perf_params {
|
||||
std::vector<std::string> include_types;
|
||||
std::vector<size_t> test_sizes;
|
||||
size_t alignment_offset = 0;
|
||||
bool op_quantize_row_q_reference = false;
|
||||
bool op_quantize_row_q = false;
|
||||
bool op_dequantize_row_q = false;
|
||||
bool op_quantize_row_q_dot = false;
|
||||
bool op_vec_dot_q = false;
|
||||
int64_t iterations = ITERATIONS;
|
||||
};
|
||||
|
||||
#if defined(__x86_64__) || defined(__i386__)
|
||||
|
||||
#include <x86intrin.h>
|
||||
inline int64_t cpu_cycles() {
|
||||
// Rough way to detect new-ish CPUs
|
||||
#ifdef __POPCNT__
|
||||
unsigned int dummy;
|
||||
return __rdtscp(&dummy);
|
||||
#else
|
||||
return __rdtsc();
|
||||
#endif
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#define cpu_cycles() 0
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
// Generate synthetic data
|
||||
static void generate_data(float offset, size_t n, float * dst) {
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
dst[i] = 0.1 + 2*cosf(i + offset);
|
||||
}
|
||||
}
|
||||
|
||||
static float gigabytes_per_second(size_t bytes, int64_t usecs) {
|
||||
return bytes / (float) usecs * 1000000 / (1024*1024*1024);
|
||||
}
|
||||
|
||||
static void * align_with_offset(void * ptr, int offset) {
|
||||
size_t dummy_size = MAX_ALIGNMENT * 4;
|
||||
return (char *) std::align(MAX_ALIGNMENT, MAX_ALIGNMENT, ptr, dummy_size) + offset;
|
||||
}
|
||||
|
||||
static void benchmark_function(size_t size, size_t q_size, int64_t iterations, const std::function<float(void)> & func) {
|
||||
int64_t min_time_us = INT64_MAX;
|
||||
int64_t total_time_us = 0;
|
||||
int64_t min_time_cycles = INT64_MAX;
|
||||
int64_t total_time_cycles = 0;
|
||||
|
||||
for (int i = 0; i < WARMUP; i++) {
|
||||
func();
|
||||
}
|
||||
|
||||
for (int i = 0; i < iterations; i++) {
|
||||
const int64_t start_time = ggml_time_us();
|
||||
const int64_t start_cycles = cpu_cycles();
|
||||
|
||||
func();
|
||||
|
||||
const int64_t end_cycles = cpu_cycles();
|
||||
const int64_t end_time = ggml_time_us();
|
||||
|
||||
total_time_cycles += end_cycles - start_cycles;
|
||||
min_time_cycles = std::min(min_time_cycles, end_cycles - start_cycles);
|
||||
total_time_us += end_time - start_time;
|
||||
min_time_us = std::min(min_time_us, end_time - start_time);
|
||||
}
|
||||
|
||||
printf(" min cycles/%d vals : %9.2f\n", QK, QK * min_time_cycles / (float) size);
|
||||
printf(" avg cycles/%d vals : %9.2f\n", QK, QK * total_time_cycles / (float) (size * iterations));
|
||||
printf(" float32 throughput : %9.2f GB/s\n", gigabytes_per_second(4 * size * iterations, total_time_us));
|
||||
printf(" quantized throughput : %9.2f GB/s\n", gigabytes_per_second(q_size * iterations, total_time_us));
|
||||
}
|
||||
|
||||
static void usage(char * argv[]) {
|
||||
printf("Benchmark quantization specific functions on synthetic data\n");
|
||||
printf("\n");
|
||||
printf("usage: %s [options]\n", argv[0]);
|
||||
printf("\n");
|
||||
printf("options: (default)\n");
|
||||
printf(" -h, --help show this help message and exit\n");
|
||||
printf(" --size SIZE set test size, divisible by 32 (L1_SIZE:%d)\n", L1_SIZE);
|
||||
printf(" -3 use size as L1, L2, L3 sizes (L1:%d L2:%d L3:%d)\n", L1_SIZE, L2_SIZE, L3_SIZE);
|
||||
printf(" -4 use size as L1, L2, L3, MEM sizes (L1:%d L2:%d L3:%d MEM:%d)\n", L1_SIZE, L2_SIZE, L3_SIZE, MEM_SIZE);
|
||||
printf(" --op OP set test operation as quantize_row_q_reference, quantize_row_q, dequantize_row_q,\n");
|
||||
printf(" quantize_row_q_dot, vec_dot_q (all)\n");
|
||||
printf(" --type TYPE set test type as");
|
||||
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
|
||||
ggml_type type = (ggml_type) i;
|
||||
const auto * qfns = ggml_get_type_traits(type);
|
||||
const auto * qfns_cpu = ggml_get_type_traits_cpu(type);
|
||||
if (ggml_type_name(type) != NULL) {
|
||||
if (qfns_cpu->from_float && qfns->to_float) {
|
||||
printf(" %s", ggml_type_name(type));
|
||||
}
|
||||
}
|
||||
}
|
||||
printf(" (all)\n");
|
||||
printf(" --alignment-offset OFFSET\n");
|
||||
printf(" set alignment offset as OFFSET (0)\n");
|
||||
printf(" -i NUM, --iterations NUM\n");
|
||||
printf(" set test iteration number (%d)\n", ITERATIONS);
|
||||
}
|
||||
|
||||
int main(int argc, char * argv[]) {
|
||||
quantize_perf_params params {};
|
||||
|
||||
// read command line
|
||||
|
||||
bool invalid_param = false;
|
||||
std::string arg;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
arg = argv[i];
|
||||
|
||||
if (arg == "--size") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
size_t size = std::stoi(argv[i]);
|
||||
if (size % 32 != 0) {
|
||||
fprintf(stderr, "error: size %zu not divisible by 32\n", size);
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.test_sizes.push_back(size);
|
||||
} else if (arg == "-3") {
|
||||
// quick select sizes that probably fit in CPU caches
|
||||
params.test_sizes.push_back(L1_SIZE);
|
||||
params.test_sizes.push_back(L2_SIZE);
|
||||
params.test_sizes.push_back(L3_SIZE);
|
||||
} else if (arg == "-4") {
|
||||
// quick select cache sizes + memory
|
||||
params.test_sizes.push_back(L1_SIZE);
|
||||
params.test_sizes.push_back(L2_SIZE);
|
||||
params.test_sizes.push_back(L3_SIZE);
|
||||
params.test_sizes.push_back(MEM_SIZE);
|
||||
} else if (arg == "--op") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
std::string op {argv[i]};
|
||||
if (op == "quantize_row_q_reference") {
|
||||
params.op_quantize_row_q_reference = true;
|
||||
} else if (op == "quantize_row_q") {
|
||||
params.op_quantize_row_q = true;
|
||||
} else if (op == "dequantize_row_q") {
|
||||
params.op_dequantize_row_q = true;
|
||||
} else if (op == "quantize_row_q_dot") {
|
||||
params.op_quantize_row_q_dot = true;
|
||||
} else if (op == "vec_dot_q") {
|
||||
params.op_vec_dot_q = true;
|
||||
} else {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
} else if (arg == "--type") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.include_types.push_back(argv[i]);
|
||||
} else if (arg == "--alignment-offset") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
int alignment = std::stoi(argv[i]);
|
||||
if (alignment < 0 || alignment > MAX_ALIGNMENT) {
|
||||
fprintf(stderr, "error: alignment-offset must be less than %d\n", MAX_ALIGNMENT);
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.alignment_offset = alignment;
|
||||
} else if ((arg == "-i") || (arg == "--iterations")) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
int number = std::stoi(argv[i]);
|
||||
if (number < 0 || number > MAX_ITERATIONS) {
|
||||
fprintf(stderr, "error: iterations must be less than %d\n", MAX_ITERATIONS);
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.iterations = number;
|
||||
} else if ((arg == "-h") || (arg == "--help")) {
|
||||
usage(argv);
|
||||
return 1;
|
||||
} else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (invalid_param) {
|
||||
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.test_sizes.empty()) {
|
||||
params.test_sizes.push_back(L1_SIZE);
|
||||
}
|
||||
if (!(params.op_quantize_row_q_reference || params.op_quantize_row_q || params.op_dequantize_row_q || params.op_quantize_row_q_dot || params.op_vec_dot_q)) {
|
||||
params.op_quantize_row_q_reference = params.op_quantize_row_q = params.op_dequantize_row_q = params.op_quantize_row_q_dot = params.op_vec_dot_q = true;
|
||||
}
|
||||
|
||||
std::sort(params.test_sizes.begin(), params.test_sizes.end());
|
||||
size_t largest = params.test_sizes.back();
|
||||
|
||||
std::vector<uint8_t> test_data1_v(largest*4 + MAX_ALIGNMENT*2);
|
||||
std::vector<uint8_t> test_data2_v(largest*4 + MAX_ALIGNMENT*2);
|
||||
std::vector<uint8_t> test_q1_v (largest*4 + MAX_ALIGNMENT*2);
|
||||
std::vector<uint8_t> test_q2_v (largest*4 + MAX_ALIGNMENT*2);
|
||||
std::vector<uint8_t> test_out_v (largest*4 + MAX_ALIGNMENT*2);
|
||||
|
||||
float * test_data1 = (float *) align_with_offset(test_data1_v.data(), params.alignment_offset);
|
||||
float * test_data2 = (float *) align_with_offset(test_data2_v.data(), params.alignment_offset);
|
||||
float * test_q1 = (float *) align_with_offset(test_q1_v.data(), params.alignment_offset);
|
||||
float * test_q2 = (float *) align_with_offset(test_q2_v.data(), params.alignment_offset);
|
||||
float * test_out = (float *) align_with_offset(test_out_v.data(), params.alignment_offset);
|
||||
|
||||
generate_data(0, largest, test_data1);
|
||||
generate_data(1, largest, test_data2);
|
||||
|
||||
int64_t iterations = params.iterations;
|
||||
|
||||
ggml_cpu_init();
|
||||
|
||||
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
|
||||
ggml_type type = (ggml_type) i;
|
||||
const auto * qfns = ggml_get_type_traits(type);
|
||||
const auto * qfns_cpu = ggml_get_type_traits_cpu(type);
|
||||
if (!params.include_types.empty() && ggml_type_name(type) && std::find(params.include_types.begin(), params.include_types.end(), ggml_type_name(type)) == params.include_types.end()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (qfns_cpu->from_float && qfns->to_float) {
|
||||
printf("%s\n", ggml_type_name(type));
|
||||
|
||||
ggml_quantize_init(type);
|
||||
|
||||
if (params.op_quantize_row_q_reference) {
|
||||
printf(" quantize_row_q_reference\n");
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
qfns->from_float_ref(test_data1, test_q1, size);
|
||||
return test_q1[0];
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
if (params.op_quantize_row_q) {
|
||||
printf(" quantize_row_q\n");
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
qfns_cpu->from_float(test_data1, test_q1, size);
|
||||
return test_q1[0];
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
if (params.op_dequantize_row_q) {
|
||||
printf(" dequantize_row_q\n");
|
||||
qfns_cpu->from_float(test_data1, test_q1, largest);
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
qfns->to_float(test_q1, test_out, size);
|
||||
return test_out[0];
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
if (params.op_quantize_row_q_dot) {
|
||||
printf(" quantize_row_q_dot\n");
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type);
|
||||
vdot->from_float(test_data1, test_q1, size);
|
||||
return test_q1[0];
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
if (params.op_vec_dot_q) {
|
||||
printf(" vec_dot_q\n");
|
||||
qfns_cpu->from_float(test_data1, test_q1, largest);
|
||||
qfns_cpu->from_float(test_data2, test_q2, largest);
|
||||
for (size_t size : params.test_sizes) {
|
||||
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
|
||||
auto quantize_fn = [&](void) -> float {
|
||||
float result;
|
||||
qfns_cpu->vec_dot(size, &result, 0, test_q1, 0, test_q2, 0, 1);
|
||||
return result;
|
||||
};
|
||||
size_t quantized_size = ggml_row_size(type, size);
|
||||
benchmark_function(size, quantized_size, iterations, quantize_fn);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
#include "llama.h"
|
||||
|
||||
#include "build-info.h"
|
||||
#include "common.h"
|
||||
|
||||
#include "../src/llama-model.h"
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <cinttypes>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <numeric>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <thread>
|
||||
#include <mutex>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
struct quantize_stats_params {
|
||||
std::string model = "models/7B/ggml-model-f16.gguf";
|
||||
bool verbose = false;
|
||||
bool per_layer_stats = false;
|
||||
bool print_histogram = false;
|
||||
bool reference = false;
|
||||
std::vector<std::string> include_layers;
|
||||
std::vector<std::string> exclude_layers;
|
||||
std::vector<enum ggml_type> include_types;
|
||||
};
|
||||
|
||||
constexpr size_t HISTOGRAM_BUCKETS = 150;
|
||||
constexpr double HISTOGRAM_RANGE = 0.03;
|
||||
|
||||
struct error_stats {
|
||||
size_t num_samples;
|
||||
double total_error;
|
||||
double max_error;
|
||||
uint64_t error_histogram[HISTOGRAM_BUCKETS];
|
||||
};
|
||||
|
||||
static void quantize_stats_print_usage(int /*argc*/, char ** argv) {
|
||||
quantize_stats_params params;
|
||||
fprintf(stderr, "usage: %s [options]\n", argv[0]);
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "options:\n");
|
||||
fprintf(stderr, " -h, --help show this help message and exit\n");
|
||||
fprintf(stderr, " -m FNAME, --model FNAME\n");
|
||||
fprintf(stderr, " model path (default: %s)\n", params.model.c_str());
|
||||
fprintf(stderr, " -r, --reference\n");
|
||||
fprintf(stderr, " use reference implementation (default: false)\n");
|
||||
fprintf(stderr, " -v, --verbose\n");
|
||||
fprintf(stderr, " verbose output (default: false)\n");
|
||||
fprintf(stderr, " -p, --per-layer-stats\n");
|
||||
fprintf(stderr, " print stats per layer (default: false)\n");
|
||||
fprintf(stderr, " --histogram\n");
|
||||
fprintf(stderr, " print error histogram (default: false)\n");
|
||||
fprintf(stderr, " -l LAYER, --include-layer LAYER\n");
|
||||
fprintf(stderr, " only test layers matching pattern\n");
|
||||
fprintf(stderr, " -L LAYER, --exclude-layer LAYER\n");
|
||||
fprintf(stderr, " exclude layers matching pattern\n");
|
||||
fprintf(stderr, " -t TYPE, --type TYPE\n");
|
||||
fprintf(stderr, " only test given type (q4_0, q4_1)\n");
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
|
||||
// Check if a layer is included/excluded by command line
|
||||
static bool layer_included(const quantize_stats_params & params, const std::string & layer) {
|
||||
for (const auto& excluded : params.exclude_layers) {
|
||||
if (std::regex_search(layer, std::regex(excluded))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (const auto& included : params.include_layers) {
|
||||
if (std::regex_search(layer, std::regex(included))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return params.include_layers.empty();
|
||||
}
|
||||
|
||||
// Update error statistics given vectors with the before/after result of quantization
|
||||
static void update_error_stats(int64_t nelements, const float * input, const float * output, error_stats & stats) {
|
||||
for (int64_t i = 0; i < nelements; i++) {
|
||||
double diff = input[i] - output[i];
|
||||
stats.total_error += diff * diff;
|
||||
stats.max_error = fmax(fabs(diff), stats.max_error);
|
||||
stats.error_histogram[std::max(std::min((size_t) floor(fabs(diff) / HISTOGRAM_RANGE * HISTOGRAM_BUCKETS), HISTOGRAM_BUCKETS-1), (size_t) 0)]++;
|
||||
}
|
||||
stats.num_samples += nelements;
|
||||
}
|
||||
|
||||
static void combine_error_stats(error_stats & into, const error_stats & from) {
|
||||
into.num_samples += from.num_samples;
|
||||
into.total_error += from.total_error;
|
||||
if (from.max_error > into.max_error) into.max_error = from.max_error;
|
||||
for (size_t i=0; i<HISTOGRAM_BUCKETS; ++i) into.error_histogram[i] += from.error_histogram[i];
|
||||
}
|
||||
|
||||
static double find_quantile(const error_stats & stats, double quantile) {
|
||||
double sum = std::accumulate(std::begin(stats.error_histogram), std::end(stats.error_histogram), 0.0);
|
||||
|
||||
double accum = 0;
|
||||
for (size_t i = 0; i < HISTOGRAM_BUCKETS; i++) {
|
||||
accum += stats.error_histogram[i];
|
||||
if (accum >= sum*quantile) {
|
||||
return (i+1) * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS;
|
||||
}
|
||||
}
|
||||
return INFINITY;
|
||||
}
|
||||
|
||||
static void print_error_stats(const std::string & name, const error_stats & stats, bool print_histogram) {
|
||||
double rmse = sqrt(stats.total_error / (double) stats.num_samples);
|
||||
double median = find_quantile(stats, .5);
|
||||
double pct95 = find_quantile(stats, .95);
|
||||
printf("%-50s: rmse %.8f, maxerr %.8f, 95pct<%.4f, median<%.4f\n", name.c_str(), rmse, stats.max_error, pct95, median);
|
||||
if (print_histogram) {
|
||||
printf("Error distribution:\n");
|
||||
for (size_t i = 0; i < HISTOGRAM_BUCKETS; i++) {
|
||||
double lower = i * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS;
|
||||
double upper = (i+1) * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS;
|
||||
if (i == HISTOGRAM_BUCKETS -1) upper = INFINITY;
|
||||
printf("[%3.4f, %3.4f): %11" PRIu64 "\n", lower, upper, stats.error_histogram[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// copied from ggml.h - verify that we can access this as a flat array
|
||||
static bool tensor_is_contiguous(const struct ggml_tensor * tensor) {
|
||||
static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");
|
||||
|
||||
return
|
||||
tensor->nb[0] == ggml_type_size(tensor->type) &&
|
||||
tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) &&
|
||||
tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&
|
||||
tensor->nb[3] == tensor->nb[2]*tensor->ne[2];
|
||||
}
|
||||
|
||||
static void test_roundtrip_on_chunk(
|
||||
const ggml_tensor * layer, int64_t offset, int64_t chunk_size, const ggml_type_traits & qfns, const ggml_type_traits_cpu & qfns_cpu, bool use_reference,
|
||||
float * input_scratch, char * quantized_scratch, float * output_scratch, error_stats & stats
|
||||
) {
|
||||
if (layer->type == GGML_TYPE_F16) {
|
||||
for (int i = 0; i < chunk_size; i++) {
|
||||
input_scratch[i] = ggml_get_f32_1d(layer, i + offset);
|
||||
}
|
||||
} else {
|
||||
input_scratch = ggml_get_data_f32(layer) + offset;
|
||||
}
|
||||
|
||||
if (use_reference) {
|
||||
qfns.from_float_ref(input_scratch, quantized_scratch, chunk_size);
|
||||
} else {
|
||||
qfns_cpu.from_float(input_scratch, quantized_scratch, chunk_size);
|
||||
}
|
||||
qfns.to_float(quantized_scratch, output_scratch, chunk_size);
|
||||
|
||||
update_error_stats(chunk_size, input_scratch, output_scratch, stats);
|
||||
}
|
||||
|
||||
|
||||
// Run quantization function for a single layer and update error stats
|
||||
static void test_roundtrip_on_layer(
|
||||
std::string & name, bool print_layer_stats, const ggml_type_traits & qfns, const ggml_type_traits_cpu & qfns_cpu, bool use_reference,
|
||||
const ggml_tensor * layer, std::vector<float> & input_scratch, std::vector<char> & quantized_scratch,
|
||||
std::vector<float> & output_scratch, error_stats & total_error, int max_thread = 0
|
||||
) {
|
||||
assert(tensor_is_contiguous(layer));
|
||||
error_stats layer_error {};
|
||||
uint64_t nelements = ggml_nelements(layer);
|
||||
|
||||
float* input_scratch_ptr = nullptr;
|
||||
if (layer->type == GGML_TYPE_F16) {
|
||||
if (input_scratch.size() < nelements) input_scratch.resize(nelements);
|
||||
input_scratch_ptr = input_scratch.data();
|
||||
}
|
||||
if (quantized_scratch.size() < 4*nelements) quantized_scratch.resize(4*nelements);
|
||||
if (output_scratch.size() < nelements) output_scratch.resize(nelements);
|
||||
|
||||
if (max_thread < 1) max_thread = std::thread::hardware_concurrency();
|
||||
int chunk_size = 32*512;
|
||||
int num_chunks = (nelements + chunk_size - 1)/chunk_size;
|
||||
|
||||
if (num_chunks < 2 || max_thread < 2) {
|
||||
test_roundtrip_on_chunk(layer, 0, nelements, qfns, qfns_cpu, use_reference, input_scratch_ptr, quantized_scratch.data(),
|
||||
output_scratch.data(), print_layer_stats ? layer_error : total_error);
|
||||
} else {
|
||||
auto & stats = print_layer_stats ? layer_error : total_error;
|
||||
std::mutex mutex;
|
||||
uint64_t counter = 0;
|
||||
auto compute = [&mutex, &counter, &stats, &qfns, &qfns_cpu, nelements, layer, use_reference, input_scratch_ptr,
|
||||
&quantized_scratch, &output_scratch, chunk_size] () {
|
||||
error_stats local_stats {};
|
||||
while (true) {
|
||||
std::unique_lock<std::mutex> lock(mutex);
|
||||
uint64_t offset = counter; counter += chunk_size;
|
||||
if (offset >= nelements) {
|
||||
combine_error_stats(stats, local_stats);
|
||||
break;
|
||||
}
|
||||
lock.unlock();
|
||||
uint64_t chunk = offset + chunk_size < nelements ? chunk_size : nelements - offset;
|
||||
test_roundtrip_on_chunk(layer, offset, chunk, qfns, qfns_cpu, use_reference, input_scratch_ptr + offset,
|
||||
quantized_scratch.data() + 4*offset, output_scratch.data() + offset, local_stats);
|
||||
}
|
||||
};
|
||||
int nthread = std::min(num_chunks, max_thread);
|
||||
std::vector<std::thread> workers(nthread-1);
|
||||
for (auto& w : workers) w = std::thread(compute);
|
||||
compute();
|
||||
for (auto& w : workers) w.join();
|
||||
}
|
||||
|
||||
if (print_layer_stats) {
|
||||
print_error_stats(name, layer_error, false);
|
||||
combine_error_stats(total_error, layer_error);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
ggml_time_init();
|
||||
|
||||
quantize_stats_params params;
|
||||
|
||||
// read command line
|
||||
|
||||
int max_thread = 0;
|
||||
bool invalid_param = false;
|
||||
std::string arg;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
arg = argv[i];
|
||||
|
||||
if (arg == "-h" || arg == "--help") {
|
||||
quantize_stats_print_usage(argc, argv);
|
||||
exit(0);
|
||||
} else if (arg == "-r" || arg == "--reference") {
|
||||
params.reference = true;
|
||||
} else if (arg == "-v") {
|
||||
params.verbose = true;
|
||||
} else if (arg == "-p" || arg == "--per-layer-stats") {
|
||||
params.per_layer_stats = true;
|
||||
} else if (arg == "--histogram") {
|
||||
params.print_histogram = true;
|
||||
} else if (arg == "-m" || arg == "--model") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.model = argv[i];
|
||||
} else if (arg == "-l" || arg == "--include-layer") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.include_layers.emplace_back(argv[i]);
|
||||
} else if (arg == "-L" || arg == "--exclude-layer") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.exclude_layers.emplace_back(argv[i]);
|
||||
} else if (arg == "-t" || arg == "--type") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
int j;
|
||||
for (j = 0; j < GGML_TYPE_COUNT; ++j) {
|
||||
const auto * name = ggml_type_name((ggml_type) j);
|
||||
if (name && strcmp(argv[i], name) == 0) break;
|
||||
}
|
||||
if (j < GGML_TYPE_COUNT) {
|
||||
params.include_types.push_back((ggml_type) j);
|
||||
} else {
|
||||
fprintf(stderr, "error: %s not in list of types\n", argv[i]);
|
||||
invalid_param = true;
|
||||
}
|
||||
} else if (arg == "-n" || arg == "--num-threads") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
max_thread = atoi(argv[i]);
|
||||
} else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
||||
quantize_stats_print_usage(argc, argv);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (invalid_param) {
|
||||
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
|
||||
quantize_stats_print_usage(argc, argv);
|
||||
return 1;
|
||||
}
|
||||
|
||||
llama_print_build_info(llama_version());
|
||||
|
||||
// load the model
|
||||
fprintf(stderr, "Loading model\n");
|
||||
|
||||
const int64_t t_main_start_us = ggml_time_us();
|
||||
llama_model * model;
|
||||
llama_context * ctx;
|
||||
|
||||
{
|
||||
auto mparams = llama_model_default_params();
|
||||
mparams.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
|
||||
model = llama_model_load_from_file(params.model.c_str(), mparams);
|
||||
|
||||
if (model == NULL) {
|
||||
fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
auto cparams = llama_context_default_params();
|
||||
cparams.n_ctx = 256;
|
||||
|
||||
ctx = llama_init_from_model(model, cparams);
|
||||
|
||||
if (ctx == NULL) {
|
||||
fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, params.model.c_str());
|
||||
llama_model_free(model);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
const auto & tensors = llama_internal_get_tensor_map(model);
|
||||
|
||||
// check layer tensors
|
||||
int included_layers = 0;
|
||||
int64_t max_nelements = 0;
|
||||
bool is_f16 = false;
|
||||
for (const auto & kv_tensor : tensors) {
|
||||
if (!layer_included(params, kv_tensor.first)) {
|
||||
continue;
|
||||
}
|
||||
if (params.verbose) {
|
||||
printf("%s: type %s, size %" PRId64 "\n", kv_tensor.first.c_str(), ggml_type_name(kv_tensor.second->type), ggml_nelements(kv_tensor.second));
|
||||
}
|
||||
if (kv_tensor.second->type == GGML_TYPE_F16) {
|
||||
is_f16 = true;
|
||||
} else if (kv_tensor.second->type != GGML_TYPE_F32) {
|
||||
fprintf(stderr, "%s: error: Quantization should be tested with a float model, "
|
||||
"this model contains already quantized layers (%s is type %d)\n", __func__, kv_tensor.first.c_str(), kv_tensor.second->type);
|
||||
llama_free(ctx);
|
||||
llama_model_free(model);
|
||||
return 1;
|
||||
}
|
||||
included_layers++;
|
||||
max_nelements = std::max(max_nelements, ggml_nelements(kv_tensor.second));
|
||||
}
|
||||
|
||||
if (is_f16) {
|
||||
printf("note: source model is f16\n");
|
||||
}
|
||||
printf("testing %d layers with max size %" PRId64 "\n", included_layers, max_nelements);
|
||||
// allocate scratch space
|
||||
std::vector<float> input_scratch;
|
||||
std::vector<char> quantized_scratch;
|
||||
std::vector<float> output_scratch;
|
||||
|
||||
// loop throught quantization types
|
||||
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
|
||||
const ggml_type type = (ggml_type) i;
|
||||
if (!params.include_types.empty() && std::find(params.include_types.begin(), params.include_types.end(), i) == params.include_types.end()) {
|
||||
continue;
|
||||
}
|
||||
const auto * qfns = ggml_get_type_traits(type);
|
||||
const auto * qfns_cpu = ggml_get_type_traits_cpu(type);
|
||||
if (qfns_cpu->from_float && qfns->to_float) {
|
||||
if (params.verbose) {
|
||||
printf("testing %s ...\n", ggml_type_name(type));
|
||||
}
|
||||
|
||||
ggml_quantize_init(type);
|
||||
|
||||
error_stats global_stats {};
|
||||
|
||||
for (const auto & kv_tensor : tensors) {
|
||||
if (!layer_included(params, kv_tensor.first)) {
|
||||
continue;
|
||||
}
|
||||
if (params.verbose) {
|
||||
printf(" %s ...\n", kv_tensor.first.c_str());
|
||||
}
|
||||
std::string layer_name { ggml_type_name(type) };
|
||||
layer_name += "::" + kv_tensor.first;
|
||||
test_roundtrip_on_layer(
|
||||
layer_name,
|
||||
params.per_layer_stats,
|
||||
*qfns, *qfns_cpu,
|
||||
params.reference,
|
||||
kv_tensor.second,
|
||||
input_scratch,
|
||||
quantized_scratch,
|
||||
output_scratch,
|
||||
global_stats,
|
||||
max_thread
|
||||
);
|
||||
}
|
||||
|
||||
print_error_stats(ggml_type_name(type), global_stats, params.print_histogram);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
llama_free(ctx);
|
||||
llama_model_free(model);
|
||||
// report timing
|
||||
{
|
||||
const int64_t t_main_end_us = ggml_time_us();
|
||||
|
||||
printf("\n");
|
||||
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,505 @@
|
||||
#include "reasoning-budget.h"
|
||||
#include "unicode.h"
|
||||
|
||||
#include "llama.h"
|
||||
#include "ggml.h"
|
||||
|
||||
#ifdef NDEBUG
|
||||
#undef NDEBUG
|
||||
#endif
|
||||
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// Reasoning budget sampler test helper
|
||||
// These tests use nullptr vocab which safely falls back to treating all tokens as complete
|
||||
// (The UTF-8 boundary detection logic is tested separately in test_utf8_boundary_detection)
|
||||
static void test_reasoning_budget(
|
||||
const char * test_name,
|
||||
const std::vector<llama_token> & sequence,
|
||||
const std::vector<llama_tokens> & start_seqs,
|
||||
const std::vector<llama_tokens> & end_seqs,
|
||||
const std::vector<llama_token> & forced_tokens,
|
||||
int32_t budget,
|
||||
common_reasoning_budget_state initial_state,
|
||||
size_t expected_force_start, // token index where forcing should start (SIZE_MAX = never)
|
||||
size_t expected_force_end // token index where forcing should end (after this, no more forcing)
|
||||
) {
|
||||
// Find the maximum token ID to ensure our vocab covers all tokens
|
||||
llama_token max_token = 0;
|
||||
for (auto t : sequence) max_token = std::max(max_token, t);
|
||||
for (const auto & seq : start_seqs) {
|
||||
for (auto t : seq) max_token = std::max(max_token, t);
|
||||
}
|
||||
for (const auto & seq : end_seqs) {
|
||||
for (auto t : seq) max_token = std::max(max_token, t);
|
||||
}
|
||||
for (auto t : forced_tokens) max_token = std::max(max_token, t);
|
||||
|
||||
// Create a minimal sampler with mock vocabulary
|
||||
// For this test, we use nullptr as vocab since we're testing state transitions
|
||||
// The UTF-8 boundary check will treat all tokens as complete (safe fallback)
|
||||
auto * sampler = common_reasoning_budget_init(
|
||||
nullptr, // vocab - not used for basic state machine tests
|
||||
start_seqs,
|
||||
end_seqs,
|
||||
forced_tokens,
|
||||
budget,
|
||||
initial_state
|
||||
);
|
||||
|
||||
// Create a test token data array for checking forcing behavior
|
||||
// Vocab size must be large enough to include all tokens (start, end, forced, sequence)
|
||||
std::vector<llama_token_data> cur;
|
||||
const size_t n_vocab = (size_t)max_token + 1;
|
||||
for (size_t i = 0; i < n_vocab; i++) {
|
||||
cur.emplace_back(llama_token_data{(llama_token)i, logf((float)(i+1)), 0.0f});
|
||||
}
|
||||
llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
|
||||
|
||||
size_t actual_force_start = SIZE_MAX;
|
||||
size_t actual_force_end = SIZE_MAX;
|
||||
|
||||
// Feed the sequence and track when forcing occurs
|
||||
for (size_t i = 0; i < sequence.size(); i++) {
|
||||
// Check if we're in forcing state by applying and seeing if logits are modified
|
||||
cur_p.selected = -1;
|
||||
for (size_t j = 0; j < cur.size(); j++) {
|
||||
cur[j].logit = logf((float)(j+1)); // reset logits
|
||||
}
|
||||
|
||||
llama_sampler_apply(sampler, &cur_p);
|
||||
|
||||
// Check if forcing is active (all logits except one should be -INFINITY)
|
||||
size_t finite_count = 0;
|
||||
llama_token finite_token = -1;
|
||||
for (size_t j = 0; j < cur.size(); j++) {
|
||||
if (std::isfinite(cur[j].logit)) {
|
||||
finite_count++;
|
||||
finite_token = cur[j].id;
|
||||
}
|
||||
}
|
||||
|
||||
llama_sampler_accept(sampler, sequence[i]);
|
||||
|
||||
fprintf(stderr, " i=%zu: token=%d, finite_count=%zu, finite_token=%d\n", i, (int)sequence[i], finite_count, (int)finite_token);
|
||||
|
||||
if (finite_count == 1) {
|
||||
if (actual_force_start == SIZE_MAX) {
|
||||
actual_force_start = i;
|
||||
}
|
||||
actual_force_end = i;
|
||||
} else if (actual_force_start != SIZE_MAX && actual_force_end != SIZE_MAX) {
|
||||
// Forcing stopped
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
llama_sampler_free(sampler);
|
||||
|
||||
// Verify forcing occurred at expected positions
|
||||
if (expected_force_start == SIZE_MAX) {
|
||||
if (actual_force_start != SIZE_MAX) {
|
||||
fprintf(stderr, "Test '%s' FAILED: Expected no forcing, but forcing occurred at %zu\n", test_name, actual_force_start);
|
||||
GGML_ASSERT(false && "Expected no forcing, but forcing occurred");
|
||||
}
|
||||
} else {
|
||||
if (actual_force_start == SIZE_MAX) {
|
||||
fprintf(stderr, "Test '%s' FAILED: Expected forcing but none occurred\n", test_name);
|
||||
GGML_ASSERT(false && "Expected forcing but none occurred");
|
||||
}
|
||||
if (actual_force_start != expected_force_start) {
|
||||
fprintf(stderr, "Test '%s' FAILED: Forcing started at %zu, expected %zu\n", test_name, actual_force_start, expected_force_start);
|
||||
GGML_ASSERT(false && "Forcing started at wrong position");
|
||||
}
|
||||
}
|
||||
|
||||
if (expected_force_end != SIZE_MAX) {
|
||||
if (actual_force_end < expected_force_end) {
|
||||
fprintf(stderr, "Test '%s' FAILED: Forcing ended at %zu, expected >= %zu\n", test_name, actual_force_end, expected_force_end);
|
||||
GGML_ASSERT(false && "Forcing ended too early");
|
||||
}
|
||||
}
|
||||
|
||||
fprintf(stderr, " Test '%s' passed (force_start=%zu, force_end=%zu)\n", test_name, actual_force_start, actual_force_end);
|
||||
(void)sequence;
|
||||
}
|
||||
|
||||
static llama_token get_forced_token(struct llama_sampler * sampler, llama_token max_token) {
|
||||
std::vector<llama_token_data> cur;
|
||||
const size_t n_vocab = (size_t) max_token + 1;
|
||||
for (size_t i = 0; i < n_vocab; i++) {
|
||||
cur.emplace_back(llama_token_data{(llama_token) i, logf((float) (i + 1)), 0.0f});
|
||||
}
|
||||
|
||||
llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
|
||||
llama_sampler_apply(sampler, &cur_p);
|
||||
|
||||
size_t finite_count = 0;
|
||||
llama_token finite_token = LLAMA_TOKEN_NULL;
|
||||
for (size_t i = 0; i < cur.size(); i++) {
|
||||
if (std::isfinite(cur[i].logit)) {
|
||||
finite_count++;
|
||||
finite_token = cur[i].id;
|
||||
}
|
||||
}
|
||||
|
||||
GGML_ASSERT(finite_count == 1 && "sampler is not forcing exactly one token");
|
||||
return finite_token;
|
||||
}
|
||||
|
||||
static void test_reasoning_budget_clone_mid_counting() {
|
||||
const std::vector<llama_token> start = {100};
|
||||
const std::vector<llama_token> end = {101};
|
||||
const std::vector<llama_token> forced = {102, 101};
|
||||
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, {start}, {end}, forced, 2, REASONING_BUDGET_IDLE);
|
||||
|
||||
llama_sampler_accept(sampler, 100); // COUNTING, remaining=2
|
||||
llama_sampler_accept(sampler, 50); // COUNTING, remaining=1
|
||||
|
||||
auto * clone = llama_sampler_clone(sampler);
|
||||
llama_sampler_accept(clone, 51); // should exhaust the cloned remaining budget
|
||||
|
||||
GGML_ASSERT(get_forced_token(clone, 102) == 102 && "cloned counting state lost remaining budget");
|
||||
|
||||
llama_sampler_free(clone);
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
static void test_reasoning_budget_clone_mid_forcing() {
|
||||
const std::vector<llama_token> start = {100};
|
||||
const std::vector<llama_token> end = {101};
|
||||
const std::vector<llama_token> forced = {102, 101};
|
||||
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, {start}, {end}, forced, 0, REASONING_BUDGET_FORCING);
|
||||
|
||||
GGML_ASSERT(get_forced_token(sampler, 102) == 102);
|
||||
llama_sampler_accept(sampler, 102); // advance to the second forced token
|
||||
|
||||
auto * clone = llama_sampler_clone(sampler);
|
||||
|
||||
GGML_ASSERT(get_forced_token(clone, 102) == 101 && "cloned forcing state lost force position");
|
||||
|
||||
llama_sampler_free(clone);
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
static void test_reasoning_budget_force_manual() {
|
||||
const std::vector<llama_token> start = {100};
|
||||
const std::vector<llama_token> end = {101};
|
||||
const std::vector<llama_token> forced = {102, 101};
|
||||
|
||||
// if COUNTING, force() succeeds and begins forcing the end sequence from the start
|
||||
{
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, {start}, {end}, forced, 5, REASONING_BUDGET_IDLE);
|
||||
|
||||
llama_sampler_accept(sampler, 100); // COUNTING, remaining=5
|
||||
llama_sampler_accept(sampler, 50); // COUNTING, remaining=4
|
||||
GGML_ASSERT(common_reasoning_budget_get_state(sampler) == REASONING_BUDGET_COUNTING);
|
||||
|
||||
GGML_ASSERT(common_reasoning_budget_force(sampler) && "force() should succeed from COUNTING");
|
||||
GGML_ASSERT(common_reasoning_budget_get_state(sampler) == REASONING_BUDGET_FORCING);
|
||||
|
||||
// forces the configured sequence from force_pos=0, then transitions to DONE
|
||||
GGML_ASSERT(get_forced_token(sampler, 102) == 102);
|
||||
llama_sampler_accept(sampler, 102);
|
||||
GGML_ASSERT(get_forced_token(sampler, 102) == 101);
|
||||
llama_sampler_accept(sampler, 101);
|
||||
GGML_ASSERT(common_reasoning_budget_get_state(sampler) == REASONING_BUDGET_DONE);
|
||||
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
// if IDLE, force() is a no-op
|
||||
{
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, {start}, {end}, forced, 5, REASONING_BUDGET_IDLE);
|
||||
|
||||
GGML_ASSERT(!common_reasoning_budget_force(sampler) && "force() must not transition from IDLE");
|
||||
GGML_ASSERT(common_reasoning_budget_get_state(sampler) == REASONING_BUDGET_IDLE);
|
||||
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
// if DONE, force() is a no-op
|
||||
{
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, {start}, {end}, forced, 5, REASONING_BUDGET_IDLE);
|
||||
|
||||
llama_sampler_accept(sampler, 100); // COUNTING
|
||||
llama_sampler_accept(sampler, 101); // natural end -> DONE
|
||||
GGML_ASSERT(common_reasoning_budget_get_state(sampler) == REASONING_BUDGET_DONE);
|
||||
|
||||
GGML_ASSERT(!common_reasoning_budget_force(sampler) && "force() must not transition from DONE");
|
||||
GGML_ASSERT(common_reasoning_budget_get_state(sampler) == REASONING_BUDGET_DONE);
|
||||
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
// if FORCING, force() is a no-op and must not rewind the force position
|
||||
{
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, {start}, {end}, forced, 0, REASONING_BUDGET_FORCING);
|
||||
|
||||
GGML_ASSERT(get_forced_token(sampler, 102) == 102);
|
||||
llama_sampler_accept(sampler, 102); // advance to the second forced token (force_pos=1)
|
||||
|
||||
GGML_ASSERT(!common_reasoning_budget_force(sampler) && "force() must not transition from FORCING");
|
||||
GGML_ASSERT(common_reasoning_budget_get_state(sampler) == REASONING_BUDGET_FORCING);
|
||||
GGML_ASSERT(get_forced_token(sampler, 102) == 101 && "force() must not rewind the force position");
|
||||
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
// a null sampler is safely ignored
|
||||
GGML_ASSERT(!common_reasoning_budget_force(nullptr));
|
||||
|
||||
fprintf(stderr, " Test 'manual force transition' passed\n");
|
||||
}
|
||||
|
||||
static void test_reasoning_budget_end_match() {
|
||||
const std::vector<llama_tokens> start = {{100}};
|
||||
const std::vector<llama_tokens> end = {{101}, {103, 104}};
|
||||
|
||||
// natural end records the sequence that matched; re-arming clears it
|
||||
{
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, start, end, {102, 101}, 5, REASONING_BUDGET_IDLE);
|
||||
|
||||
GGML_ASSERT(common_reasoning_budget_get_end_match(sampler) == nullptr);
|
||||
|
||||
llama_sampler_accept(sampler, 100); // COUNTING
|
||||
llama_sampler_accept(sampler, 50);
|
||||
llama_sampler_accept(sampler, 103);
|
||||
llama_sampler_accept(sampler, 104); // end matched via {103, 104}, DONE
|
||||
|
||||
const llama_tokens * matched = common_reasoning_budget_get_end_match(sampler);
|
||||
GGML_ASSERT(matched != nullptr);
|
||||
GGML_ASSERT(*matched == llama_tokens({103, 104}));
|
||||
|
||||
llama_sampler_accept(sampler, 100); // re-arm, COUNTING
|
||||
GGML_ASSERT(common_reasoning_budget_get_end_match(sampler) == nullptr);
|
||||
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
// overlapping end sequences: the longest one ending at the position wins
|
||||
{
|
||||
const std::vector<llama_tokens> end_overlap = {{104}, {103, 104}};
|
||||
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, start, end_overlap, {102, 104}, 5, REASONING_BUDGET_IDLE);
|
||||
|
||||
llama_sampler_accept(sampler, 100); // COUNTING
|
||||
llama_sampler_accept(sampler, 103);
|
||||
llama_sampler_accept(sampler, 104); // both {104} and {103, 104} end here
|
||||
|
||||
const llama_tokens * matched = common_reasoning_budget_get_end_match(sampler);
|
||||
GGML_ASSERT(matched != nullptr);
|
||||
GGML_ASSERT(*matched == llama_tokens({103, 104}));
|
||||
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
// forcing records the end sequence terminating forced_tokens
|
||||
{
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, start, end, {102, 103, 104}, 0, REASONING_BUDGET_FORCING);
|
||||
|
||||
llama_sampler_accept(sampler, 102);
|
||||
llama_sampler_accept(sampler, 103);
|
||||
GGML_ASSERT(common_reasoning_budget_get_end_match(sampler) == nullptr);
|
||||
llama_sampler_accept(sampler, 104); // forced sequence complete, DONE
|
||||
|
||||
const llama_tokens * matched = common_reasoning_budget_get_end_match(sampler);
|
||||
GGML_ASSERT(matched != nullptr);
|
||||
GGML_ASSERT(*matched == llama_tokens({103, 104}));
|
||||
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
// forced_tokens not ending with a known end sequence records nothing
|
||||
{
|
||||
auto * sampler = common_reasoning_budget_init(nullptr, start, end, {102}, 0, REASONING_BUDGET_FORCING);
|
||||
|
||||
llama_sampler_accept(sampler, 102); // forced sequence complete, DONE
|
||||
GGML_ASSERT(common_reasoning_budget_get_state(sampler) == REASONING_BUDGET_DONE);
|
||||
GGML_ASSERT(common_reasoning_budget_get_end_match(sampler) == nullptr);
|
||||
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
// a null sampler is safely ignored
|
||||
GGML_ASSERT(common_reasoning_budget_get_end_match(nullptr) == nullptr);
|
||||
|
||||
fprintf(stderr, " Test 'matched end sequence' passed\n");
|
||||
}
|
||||
|
||||
// UTF-8 boundary detection unit test
|
||||
// Tests common_utf8_is_complete() from reasoning-budget.h
|
||||
static void test_utf8_boundary_detection() {
|
||||
// Complete sequences
|
||||
GGML_ASSERT(common_utf8_is_complete("hello"));
|
||||
GGML_ASSERT(common_utf8_is_complete(""));
|
||||
GGML_ASSERT(common_utf8_is_complete("\xC2\xA0")); // complete 2-byte UTF-8 (U+00A0)
|
||||
GGML_ASSERT(common_utf8_is_complete("\xE2\x80\x9C")); // complete 3-byte UTF-8 (left double quote)
|
||||
GGML_ASSERT(common_utf8_is_complete("\xF0\x9F\x98\x80")); // complete 4-byte UTF-8 (emoji)
|
||||
GGML_ASSERT(common_utf8_is_complete("abc\xC3\xA9")); // ASCII + complete 2-byte
|
||||
|
||||
// Incomplete sequences
|
||||
GGML_ASSERT(!common_utf8_is_complete(std::string("\xC2", 1))); // 2-byte start, missing continuation
|
||||
GGML_ASSERT(!common_utf8_is_complete(std::string("\xE2\x80", 2))); // 3-byte start + 1 cont, missing 1
|
||||
GGML_ASSERT(!common_utf8_is_complete(std::string("\xE2", 1))); // 3-byte start, missing 2
|
||||
GGML_ASSERT(!common_utf8_is_complete(std::string("\xF0\x9F\x98", 3))); // 4-byte start + 2 cont, missing 1
|
||||
GGML_ASSERT(!common_utf8_is_complete(std::string("\xF0\x9F", 2))); // 4-byte start + 1 cont, missing 2
|
||||
GGML_ASSERT(!common_utf8_is_complete(std::string("\xF0", 1))); // 4-byte start, missing 3
|
||||
GGML_ASSERT(!common_utf8_is_complete(std::string("\x80", 1))); // orphan continuation byte
|
||||
|
||||
// Mixed: ASCII followed by start of multi-byte
|
||||
GGML_ASSERT(!common_utf8_is_complete(std::string("hello\xC3", 6))); // ASCII + incomplete 2-byte
|
||||
GGML_ASSERT(common_utf8_is_complete(std::string("hello\xC3\xA9", 7))); // ASCII + complete 2-byte
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
// Reasoning budget sampler tests
|
||||
printf("Testing reasoning budget sampler... ");
|
||||
|
||||
// Test 1: Basic budget with start/end tokens - no forcing (natural end before budget exhausted)
|
||||
{
|
||||
const std::vector<llama_token> start = {100}; // start token
|
||||
const std::vector<llama_token> end = {101}; // end token
|
||||
const std::vector<llama_token> forced = {102}; // forced token (not used in this test)
|
||||
const std::vector<llama_token> sequence = {100, 50, 51, 101, 52}; // start, two tokens, end, one more
|
||||
|
||||
test_reasoning_budget("natural end before budget exhausted", sequence, {start}, {end}, forced,
|
||||
5, // budget of 5 tokens
|
||||
REASONING_BUDGET_IDLE,
|
||||
SIZE_MAX, SIZE_MAX); // no forcing expected (natural end)
|
||||
}
|
||||
|
||||
// Test 2: Budget exhausted, forcing should occur
|
||||
// Flow: i=0 apply()->passthrough, accept(100)->COUNTING; i=1 accept(50)->remaining=1
|
||||
// i=2 accept(51)->remaining=0->FORCING; i=3 apply() forces token[0]; i=4 apply() forces token[1]
|
||||
// At i=4, accept() advances force_pos to 2 which equals forced_tokens.size(), so state becomes DONE
|
||||
{
|
||||
const std::vector<llama_token> start = {100};
|
||||
const std::vector<llama_token> end = {101};
|
||||
const std::vector<llama_token> forced = {102, 101}; // forced message + end
|
||||
const std::vector<llama_token> sequence = {100, 50, 51, 52, 53}; // start + 4 tokens (budget=2)
|
||||
|
||||
test_reasoning_budget("budget exhausted forcing", sequence, {start}, {end}, forced,
|
||||
2, // budget of 2 tokens
|
||||
REASONING_BUDGET_IDLE,
|
||||
3, // forcing starts at i=3 (accept at i=2 depletes budget, apply at i=3 forces)
|
||||
4); // forcing continues through i=4 (accept at i=4 transitions to DONE)
|
||||
}
|
||||
|
||||
// Test 3: Activate immediately with budget=0, forcing should start right away
|
||||
// Flow: init promotes COUNTING+budget=0 to FORCING, so apply() sees FORCING at i=0
|
||||
{
|
||||
const std::vector<llama_token> start = {100};
|
||||
const std::vector<llama_token> end = {101};
|
||||
const std::vector<llama_token> forced = {102, 101};
|
||||
const std::vector<llama_token> sequence = {100, 50, 51, 52}; // start token first, then 3 tokens
|
||||
|
||||
test_reasoning_budget("activate immediately budget=0", sequence, {start}, {end}, forced,
|
||||
0, // budget of 0 tokens
|
||||
REASONING_BUDGET_COUNTING, // starts counting, promoted to FORCING since budget=0
|
||||
0, // forcing starts at i=0 (initialized in FORCING, apply forces immediately)
|
||||
1); // forcing continues through i=1 (accept at i=1 transitions to DONE)
|
||||
}
|
||||
|
||||
// Test 4: No start/end tokens configured - passthrough (no forcing)
|
||||
{
|
||||
const std::vector<llama_token> start = {};
|
||||
const std::vector<llama_token> end = {};
|
||||
const std::vector<llama_token> forced = {102};
|
||||
const std::vector<llama_token> sequence = {50, 51, 52, 53};
|
||||
|
||||
test_reasoning_budget("no start/end configured", sequence, {start}, {end}, forced,
|
||||
2, // budget
|
||||
REASONING_BUDGET_IDLE,
|
||||
SIZE_MAX, SIZE_MAX); // no forcing (no start/end configured)
|
||||
}
|
||||
|
||||
// Test 5: Activate immediately with budget > 0, count down then force
|
||||
// Flow: i=0 accept(50)->remaining=1, i=1 accept(51)->remaining=0->FORCING
|
||||
// Forcing starts at i=2 (apply sees FORCING after accept at i=1 transitioned)
|
||||
{
|
||||
const std::vector<llama_token> start = {100};
|
||||
const std::vector<llama_token> end = {101};
|
||||
const std::vector<llama_token> forced = {102, 101};
|
||||
const std::vector<llama_token> sequence = {50, 51, 52, 53};
|
||||
|
||||
test_reasoning_budget("activate immediately with budget", sequence, {start}, {end}, forced,
|
||||
2, // budget of 2 tokens
|
||||
REASONING_BUDGET_COUNTING,
|
||||
2, // forcing starts at i=2 (after 2 accepts deplete budget, apply at i=2 forces)
|
||||
3); // forcing continues through i=3
|
||||
}
|
||||
|
||||
// Test 6: Multi-block thinking. First block ends naturally at i=2, second
|
||||
// start tag at i=3 re-arms the budget, which then exhausts at i=5.
|
||||
// Regression: before this fix, DONE absorbed all subsequent tokens and a
|
||||
// second <think> block ran unbudgeted.
|
||||
// Flow: i=0 accept(100)->COUNTING rem=2; i=1 accept(50)->rem=1;
|
||||
// i=2 accept(101)->end_matcher matches, DONE;
|
||||
// i=3 accept(100)->re-arm, COUNTING rem=2;
|
||||
// i=4 accept(60)->rem=1; i=5 accept(61)->rem=0->FORCING;
|
||||
// i=6 apply()->forces token[0]=102, accept(62)->force_pos=1, stay FORCING;
|
||||
// i=7 apply()->forces token[1]=101, accept(63)->force_pos=2->DONE.
|
||||
{
|
||||
const std::vector<llama_token> start = {100};
|
||||
const std::vector<llama_token> end = {101};
|
||||
const std::vector<llama_token> forced = {102, 101};
|
||||
const std::vector<llama_token> sequence = {100, 50, 101, 100, 60, 61, 62, 63};
|
||||
|
||||
test_reasoning_budget("multi-block re-arms budget after DONE", sequence, {start}, {end}, forced,
|
||||
2, // budget of 2 tokens (per block)
|
||||
REASONING_BUDGET_IDLE,
|
||||
6, // forcing starts at i=6 (after second block exhausts at i=5)
|
||||
7); // forcing continues through i=7
|
||||
}
|
||||
|
||||
// Test 7: Multiple start sequences - the second sequence activates counting
|
||||
// Flow: i=0 accept(110), i=1 accept(111)->COUNTING rem=2; i=2 accept(50)->rem=1;
|
||||
// i=3 accept(51)->rem=0->FORCING; i=4..5 apply() forces the end sequence
|
||||
{
|
||||
const std::vector<llama_tokens> start = {{100}, {110, 111}};
|
||||
const std::vector<llama_tokens> end = {{101}};
|
||||
const std::vector<llama_token> forced = {102, 101};
|
||||
const std::vector<llama_token> sequence = {110, 111, 50, 51, 52, 53};
|
||||
|
||||
test_reasoning_budget("multiple start sequences", sequence, start, end, forced,
|
||||
2, // budget of 2 tokens
|
||||
REASONING_BUDGET_IDLE,
|
||||
4, // forcing starts at i=4 (accept at i=3 depletes budget)
|
||||
5); // forcing continues through i=5
|
||||
}
|
||||
|
||||
// Test 8: Multiple end sequences - natural end via the second sequence
|
||||
// Flow: i=0 accept(100)->COUNTING rem=5; i=1 accept(50)->rem=4;
|
||||
// i=2 accept(103)->partial end, rem=3; i=3 accept(104)->end matched, DONE
|
||||
{
|
||||
const std::vector<llama_tokens> start = {{100}};
|
||||
const std::vector<llama_tokens> end = {{101}, {103, 104}};
|
||||
const std::vector<llama_token> forced = {102, 101};
|
||||
const std::vector<llama_token> sequence = {100, 50, 103, 104, 52};
|
||||
|
||||
test_reasoning_budget("multiple end sequences", sequence, start, end, forced,
|
||||
5, // budget of 5 tokens
|
||||
REASONING_BUDGET_IDLE,
|
||||
SIZE_MAX, SIZE_MAX); // no forcing expected (natural end)
|
||||
}
|
||||
|
||||
test_reasoning_budget_clone_mid_counting();
|
||||
test_reasoning_budget_clone_mid_forcing();
|
||||
test_reasoning_budget_force_manual();
|
||||
test_reasoning_budget_end_match();
|
||||
|
||||
printf("OK (12 tests passed)\n");
|
||||
|
||||
printf("Testing UTF-8 boundary detection... ");
|
||||
test_utf8_boundary_detection();
|
||||
printf("OK\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cassert>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
#if defined(__GNUC__)
|
||||
#pragma GCC diagnostic ignored "-Wdouble-promotion"
|
||||
#endif
|
||||
|
||||
#define MAX_NARGS 3
|
||||
|
||||
#undef MIN
|
||||
#undef MAX
|
||||
#define MIN(a, b) ((a) < (b) ? (a) : (b))
|
||||
#define MAX(a, b) ((a) > (b) ? (a) : (b))
|
||||
|
||||
#define GGML_SILU_FP16
|
||||
|
||||
//
|
||||
// logging
|
||||
//
|
||||
|
||||
#if (GGML_DEBUG >= 1)
|
||||
#define GGML_PRINT_DEBUG(...) printf(__VA_ARGS__)
|
||||
#else
|
||||
#define GGML_PRINT_DEBUG(...)
|
||||
#endif
|
||||
|
||||
#if (GGML_DEBUG >= 5)
|
||||
#define GGML_PRINT_DEBUG_5(...) printf(__VA_ARGS__)
|
||||
#else
|
||||
#define GGML_PRINT_DEBUG_5(...)
|
||||
#endif
|
||||
|
||||
#if (GGML_DEBUG >= 10)
|
||||
#define GGML_PRINT_DEBUG_10(...) printf(__VA_ARGS__)
|
||||
#else
|
||||
#define GGML_PRINT_DEBUG_10(...)
|
||||
#endif
|
||||
|
||||
#define GGML_PRINT(...) printf(__VA_ARGS__)
|
||||
|
||||
static float frand(void) {
|
||||
return (float)rand()/(float)RAND_MAX;
|
||||
}
|
||||
|
||||
static int irand(int n) {
|
||||
if (n == 0) return 0;
|
||||
return rand()%n;
|
||||
}
|
||||
|
||||
static void get_random_dims(int64_t * dims, int ndims) {
|
||||
dims[0] = dims[1] = dims[2] = dims[3] = 1;
|
||||
|
||||
for (int i = 0; i < ndims; i++) {
|
||||
dims[i] = 1 + irand(4);
|
||||
}
|
||||
}
|
||||
|
||||
static struct ggml_tensor * get_random_tensor_f32(
|
||||
struct ggml_context * ctx0,
|
||||
int ndims,
|
||||
const int64_t ne[],
|
||||
float fmin,
|
||||
float fmax) {
|
||||
struct ggml_tensor * result = ggml_new_tensor(ctx0, GGML_TYPE_F32, ndims, ne);
|
||||
|
||||
switch (ndims) {
|
||||
case 1:
|
||||
for (int i0 = 0; i0 < ne[0]; i0++) {
|
||||
((float *)result->data)[i0] = frand()*(fmax - fmin) + fmin;
|
||||
}
|
||||
break;
|
||||
case 2:
|
||||
for (int i1 = 0; i1 < ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < ne[0]; i0++) {
|
||||
((float *)result->data)[i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 3:
|
||||
for (int i2 = 0; i2 < ne[2]; i2++) {
|
||||
for (int i1 = 0; i1 < ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < ne[0]; i0++) {
|
||||
((float *)result->data)[i2*ne[1]*ne[0] + i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 4:
|
||||
for (int i3 = 0; i3 < ne[3]; i3++) {
|
||||
for (int i2 = 0; i2 < ne[2]; i2++) {
|
||||
for (int i1 = 0; i1 < ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < ne[0]; i0++) {
|
||||
((float *)result->data)[i3*ne[2]*ne[1]*ne[0] + i2*ne[1]*ne[0] + i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
};
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static void ggml_graph_compute_helper(std::vector<uint8_t> & buf, ggml_cgraph * graph, int n_threads) {
|
||||
struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, nullptr);
|
||||
|
||||
if (plan.work_size > 0) {
|
||||
buf.resize(plan.work_size);
|
||||
plan.work_data = buf.data();
|
||||
}
|
||||
|
||||
ggml_graph_compute(graph, &plan);
|
||||
}
|
||||
|
||||
int main(int /*argc*/, const char ** /*argv*/) {
|
||||
struct ggml_init_params params = {
|
||||
/* .mem_size = */ 128*1024*1024,
|
||||
/* .mem_buffer = */ NULL,
|
||||
/* .no_alloc = */ false,
|
||||
};
|
||||
|
||||
std::vector<uint8_t> work_buffer;
|
||||
|
||||
struct ggml_context * ctx0 = ggml_init(params);
|
||||
|
||||
struct ggml_tensor * x;
|
||||
|
||||
// rope f32
|
||||
for (int m = 0; m < 5; ++m) {
|
||||
const int ndims = 4;
|
||||
|
||||
const int64_t n_rot = 128;
|
||||
const int64_t ne[4] = { 2*n_rot, 32, 73, 1 };
|
||||
|
||||
const int n_past_0 = 100;
|
||||
const int n_past_2 = 33;
|
||||
|
||||
struct ggml_tensor * r0;
|
||||
struct ggml_tensor * r1;
|
||||
struct ggml_tensor * r2;
|
||||
x = get_random_tensor_f32(ctx0, ndims, ne, -1.0f, 1.0f);
|
||||
int mode = -1;
|
||||
|
||||
if (m < 2) {
|
||||
struct ggml_tensor * p0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2]);
|
||||
struct ggml_tensor * p1 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2]);
|
||||
struct ggml_tensor * p2 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2]);
|
||||
|
||||
for (int i = 0; i < ne[2]; ++i) {
|
||||
((int32_t *) p0->data)[i] = n_past_0 + i;
|
||||
((int32_t *) p1->data)[i] = n_past_2 - n_past_0;
|
||||
((int32_t *) p2->data)[i] = n_past_2 + i;
|
||||
}
|
||||
// test mode 0, 2 (standard, GPT-NeoX)
|
||||
mode = m == 0 ? GGML_ROPE_TYPE_NORMAL : GGML_ROPE_TYPE_NEOX;
|
||||
|
||||
// 100, 101, 102, ..., 172
|
||||
r0 = ggml_rope(ctx0, x, p0, n_rot, mode);
|
||||
// -67, -67, -67, ..., -67
|
||||
r1 = ggml_rope(ctx0, r0, p1, n_rot, mode); // "context swap", i.e. forget n_past_0 - n_past_2 tokens
|
||||
|
||||
// 33, 34, 35, ..., 105
|
||||
r2 = ggml_rope(ctx0, x, p2, n_rot, mode);
|
||||
} else {
|
||||
// testing multi-dimension rope position embedding mode
|
||||
struct ggml_tensor * p0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2] * 4);
|
||||
struct ggml_tensor * p1 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2] * 4);
|
||||
struct ggml_tensor * p2 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2] * 4);
|
||||
|
||||
int sections[4] = {16, 24, 24, 0};
|
||||
|
||||
mode = (m == 2) ? GGML_ROPE_TYPE_MROPE : (m == 3) ? GGML_ROPE_TYPE_VISION : GGML_ROPE_TYPE_IMROPE;
|
||||
|
||||
for (int i = 0; i < ne[2]; ++i) {
|
||||
for (int j = 0; j < 4; ++j) {
|
||||
((int32_t *) p0->data)[i + ne[2] * j] = n_past_0 + i + j;
|
||||
((int32_t *) p1->data)[i + ne[2] * j] = n_past_2 - n_past_0;
|
||||
((int32_t *) p2->data)[i + ne[2] * j] = n_past_2 + i + j;
|
||||
}
|
||||
}
|
||||
|
||||
// [[100, 101, 102, ..., 172],
|
||||
// [101, 102, 103, ..., 173],
|
||||
// [102, 103, 104, ..., 174]]
|
||||
r0 = ggml_rope_multi(
|
||||
ctx0, x, p0, nullptr,
|
||||
n_rot, sections, mode, 32768, 1000000, 1, 0, 1, 32, 1);
|
||||
// [[-67, -67, -67, ..., -67]
|
||||
// [-67, -67, -67, ..., -67]
|
||||
// [-67, -67, -67, ..., -67]]
|
||||
r1 = ggml_rope_multi(
|
||||
ctx0, r0, p1, nullptr,
|
||||
n_rot, sections, mode, 32768, 1000000, 1, 0, 1, 32, 1);
|
||||
|
||||
// [[33, 34, 35, ..., 105]
|
||||
// [34, 35, 36, ..., 106]
|
||||
// [35, 36, 37, ..., 107]]
|
||||
r2 = ggml_rope_multi(
|
||||
ctx0, x, p2, nullptr,
|
||||
n_rot, sections, mode, 32768, 1000000, 1, 0, 1, 32, 1);
|
||||
}
|
||||
|
||||
ggml_cgraph * gf = ggml_new_graph(ctx0);
|
||||
|
||||
ggml_build_forward_expand(gf, r0);
|
||||
ggml_build_forward_expand(gf, r1);
|
||||
ggml_build_forward_expand(gf, r2);
|
||||
|
||||
ggml_graph_compute_helper(work_buffer, gf, 4);
|
||||
|
||||
// check that r1 and r2 are the same
|
||||
{
|
||||
double sum0 = 0.0f;
|
||||
double sum1 = 0.0f;
|
||||
double diff = 0.0f;
|
||||
|
||||
const float * r1_data = (float *) r1->data;
|
||||
const float * r2_data = (float *) r2->data;
|
||||
|
||||
const int n_elements = ggml_nelements(r1);
|
||||
|
||||
for (int i = 0; i < n_elements; ++i) {
|
||||
sum0 += fabs(r1_data[i]);
|
||||
sum1 += fabs(r2_data[i]);
|
||||
diff += fabs(r1_data[i] - r2_data[i]);
|
||||
//if (fabs(r1_data[i] - r2_data[i]) > 0.0001f) {
|
||||
// printf("%d: %f %f\n", i, r1_data[i], r2_data[i]);
|
||||
// printf("diff: %f\n", fabs(r1_data[i] - r2_data[i]));
|
||||
//}
|
||||
}
|
||||
|
||||
//for (int i = 4096; i < 4096 + 128; ++i) {
|
||||
// printf("%f %f\n", r1_data[i], r2_data[i]);
|
||||
//}
|
||||
|
||||
printf("mode: %d\n", mode);
|
||||
printf("sum0: %f\n", sum0);
|
||||
printf("sum1: %f\n", sum1);
|
||||
printf("diff: %f\n", diff);
|
||||
printf("rel err: %f\n", diff / sum0);
|
||||
printf("rel err: %f\n", diff / sum1);
|
||||
|
||||
GGML_ASSERT(diff / sum0 < 0.0001f);
|
||||
GGML_ASSERT(diff / sum1 < 0.0001f);
|
||||
}
|
||||
}
|
||||
|
||||
ggml_free(ctx0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml-impl.h"
|
||||
#include "ggml-rpc.h"
|
||||
#include "ggml.h"
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
GGML_ASSERT(argc == 3);
|
||||
ggml_backend_load_all();
|
||||
|
||||
const char * endpoint_a = argv[1];
|
||||
const char * endpoint_b = argv[2];
|
||||
|
||||
ggml_backend_t backend_a = ggml_backend_rpc_init(endpoint_a, 0);
|
||||
ggml_backend_t backend_b = ggml_backend_rpc_init(endpoint_b, 0);
|
||||
GGML_ASSERT(backend_a != nullptr);
|
||||
GGML_ASSERT(backend_b != nullptr);
|
||||
|
||||
ggml_init_params params = {
|
||||
/* .mem_size = */ 3*ggml_tensor_overhead() + ggml_graph_overhead_custom(1, false),
|
||||
/* .mem_buffer = */ nullptr,
|
||||
/* .no_alloc = */ true,
|
||||
};
|
||||
ggml_context * ctx = ggml_init(params);
|
||||
GGML_ASSERT(ctx != nullptr);
|
||||
|
||||
ggml_tensor * tensor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
|
||||
ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend_a);
|
||||
GGML_ASSERT(buffer != nullptr);
|
||||
|
||||
// A remote pointer allocated by server A is not meaningful to server B.
|
||||
ggml_cgraph * graph = ggml_new_graph_custom(ctx, 1, false);
|
||||
graph->nodes[0] = tensor;
|
||||
graph->n_nodes = 1;
|
||||
|
||||
GGML_ASSERT(ggml_backend_graph_compute(backend_b, graph) == GGML_STATUS_SUCCESS);
|
||||
// Wait for server B to finish the graph before the script checks its log.
|
||||
size_t free_mem;
|
||||
size_t total_mem;
|
||||
ggml_backend_rpc_get_device_memory(endpoint_b, 0, &free_mem, &total_mem);
|
||||
GGML_ASSERT(total_mem > 0);
|
||||
ggml_backend_buffer_free(buffer);
|
||||
|
||||
// Two tensors with the same ne[] but different nb[] must not share a cached alloc size.
|
||||
// ref: https://github.com/ggml-org/llama.cpp/issues/28360
|
||||
ggml_backend_buffer_type_t buft = ggml_backend_rpc_buffer_type(endpoint_a, 0);
|
||||
GGML_ASSERT(buft != nullptr);
|
||||
|
||||
// MUL_MAT may need extra memory, so the size is read from the server [TAG_ALLOC_SIZE_EXPAND]
|
||||
ggml_tensor * packed = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 64, 64);
|
||||
packed->op = GGML_OP_MUL_MAT;
|
||||
|
||||
// same ne[], twice the row stride
|
||||
ggml_tensor * strided = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 64, 64);
|
||||
strided->op = GGML_OP_MUL_MAT;
|
||||
strided->nb[1] = 2*strided->nb[1];
|
||||
strided->nb[2] = strided->ne[1]*strided->nb[1];
|
||||
strided->nb[3] = strided->nb[2];
|
||||
GGML_ASSERT(ggml_nbytes(strided) > ggml_nbytes(packed));
|
||||
|
||||
// ask for the packed tensor first, so a cache keyed without nb[] holds the smaller size
|
||||
GGML_ASSERT(ggml_backend_buft_get_alloc_size(buft, packed) >= ggml_nbytes(packed));
|
||||
GGML_ASSERT(ggml_backend_buft_get_alloc_size(buft, strided) >= ggml_nbytes(strided));
|
||||
|
||||
ggml_free(ctx);
|
||||
ggml_backend_free(backend_b);
|
||||
ggml_backend_free(backend_a);
|
||||
return 0;
|
||||
}
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
server=$1
|
||||
client=$2
|
||||
port_a=$((40000 + $$ % 10000))
|
||||
port_b=$((port_a + 1))
|
||||
endpoint_a="127.0.0.1:${port_a}"
|
||||
endpoint_b="127.0.0.1:${port_b}"
|
||||
test_dir=$(mktemp -d)
|
||||
|
||||
cleanup() {
|
||||
kill "${pid_a:-}" "${pid_b:-}" 2>/dev/null || true
|
||||
rm -rf "$test_dir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
wait_for_port() {
|
||||
local port=$1
|
||||
for _ in {1..600}; do
|
||||
if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then
|
||||
exec 3>&-
|
||||
exec 3<&-
|
||||
return 0
|
||||
fi
|
||||
sleep 0.05
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
"$server" --device CPU --host 127.0.0.1 --port "$port_a" >"$test_dir/server-a.log" 2>&1 &
|
||||
pid_a=$!
|
||||
"$server" --device CPU --host 127.0.0.1 --port "$port_b" >"$test_dir/server-b.log" 2>&1 &
|
||||
pid_b=$!
|
||||
wait_for_port "$port_a"
|
||||
wait_for_port "$port_b"
|
||||
|
||||
"$client" "$endpoint_a" "$endpoint_b"
|
||||
|
||||
if grep -q "invalid data ptr" "$test_dir/server-b.log"; then
|
||||
cat "$test_dir/server-b.log"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,53 @@
|
||||
// ref: https://github.com/ggml-org/llama.cpp/issues/25937
|
||||
// only works reliably when run with a large model that occupies 3GB+ of wired memory
|
||||
// thus, this test is not run by default
|
||||
// example model to run with: google/gemma-4-E4B-it-qat-q4_0-gguf
|
||||
|
||||
#include "llama.h"
|
||||
#include "common.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <mach/mach.h>
|
||||
#include <mach/mach_host.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static uint64_t wired_memory() {
|
||||
vm_statistics64_data_t vmstat;
|
||||
mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
|
||||
if (host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vmstat, &count) != KERN_SUCCESS) {
|
||||
return UINT64_MAX;
|
||||
}
|
||||
return static_cast<uint64_t>(vmstat.wire_count) * vm_kernel_page_size;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
auto * model_path = common_get_model_or_exit(argc, argv);
|
||||
|
||||
llama_backend_init();
|
||||
|
||||
const uint64_t wired_initial = wired_memory();
|
||||
|
||||
llama_model_params params = llama_model_default_params();
|
||||
params.load_mode = LLAMA_LOAD_MODE_NONE;
|
||||
struct llama_model* model = llama_model_load_from_file(model_path, params);
|
||||
|
||||
const uint64_t wired_loaded = wired_memory();
|
||||
const uint64_t wired_delta = wired_loaded - wired_initial;
|
||||
// system memory fluctuates, so we need to allocate enough to reliably detect the release
|
||||
GGML_ASSERT(wired_delta > 2'000'000'000); // 2GB
|
||||
|
||||
llama_model_free(model);
|
||||
|
||||
const uint64_t t_start_ms = ggml_time_ms();
|
||||
|
||||
// expect most of the allocated memory to be released within 10 seconds
|
||||
// we allow for some tolerance due to system-wide memory fluctuations
|
||||
while (wired_memory() > wired_loaded - 0.75 * wired_delta) {
|
||||
GGML_ASSERT(ggml_time_ms() - t_start_ms < 10'000);
|
||||
usleep(100'000); // 100ms
|
||||
}
|
||||
|
||||
llama_backend_free();
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
#include "ggml.h"
|
||||
#include "llama.h"
|
||||
|
||||
#ifdef NDEBUG
|
||||
#undef NDEBUG
|
||||
#endif
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
extern struct llama_sampler * llama_sampler_init_dry_testing(float dry_multiplier, float dry_base, int32_t dry_allowed_length, int32_t dry_penalty_last_n, const std::vector<std::vector<llama_token>>& seq_breakers);
|
||||
|
||||
static void dump(const llama_token_data_array * cur_p) {
|
||||
for (size_t i = 0; i < cur_p->size; i++) {
|
||||
printf("%d: %f (%f)\n", cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
|
||||
}
|
||||
}
|
||||
|
||||
#define DUMP(__cur_p) do { printf("%s:%d (%s)\n", __FILE__, __LINE__, __func__); dump((__cur_p)); printf("-\n"); } while(0)
|
||||
|
||||
struct sampler_tester {
|
||||
sampler_tester(size_t n_vocab) {
|
||||
cur.reserve(n_vocab);
|
||||
for (llama_token token_id = 0; token_id < (llama_token)n_vocab; token_id++) {
|
||||
const float logit = logf(token_id);
|
||||
cur.emplace_back(llama_token_data{token_id, logit, 0.0f});
|
||||
}
|
||||
|
||||
cur_p = llama_token_data_array { cur.data(), cur.size(), -1, false };
|
||||
}
|
||||
|
||||
sampler_tester(const std::vector<float> & probs, const std::vector<float> & probs_expected) : probs_expected(probs_expected) {
|
||||
cur.reserve(probs.size());
|
||||
for (llama_token token_id = 0; token_id < (llama_token)probs.size(); token_id++) {
|
||||
const float logit = logf(probs[token_id]);
|
||||
cur.emplace_back(llama_token_data{token_id, logit, probs[token_id]});
|
||||
}
|
||||
|
||||
cur_p = llama_token_data_array { cur.data(), cur.size(), -1, false };
|
||||
}
|
||||
|
||||
void apply(llama_sampler * sampler) {
|
||||
llama_sampler_apply(sampler, &cur_p);
|
||||
llama_sampler_free(sampler);
|
||||
}
|
||||
|
||||
void check() {
|
||||
GGML_ASSERT(cur_p.size == probs_expected.size());
|
||||
for (size_t i = 0; i < cur_p.size; i++) {
|
||||
GGML_ASSERT(fabs(cur_p.data[i].p - probs_expected[i]) < 1e-5);
|
||||
}
|
||||
}
|
||||
|
||||
llama_token_data_array cur_p;
|
||||
|
||||
private:
|
||||
const std::vector<float> probs_expected;
|
||||
|
||||
std::vector<llama_token_data> cur;
|
||||
};
|
||||
|
||||
static llama_token sample_dist(llama_sampler * sampler, const std::vector<float> & logits) {
|
||||
std::vector<llama_token_data> cur;
|
||||
for (llama_token token_id = 0; token_id < (llama_token) logits.size(); ++token_id) {
|
||||
cur.push_back({ token_id, logits[token_id], 0.0f });
|
||||
}
|
||||
|
||||
llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
|
||||
llama_sampler_apply(sampler, &cur_p);
|
||||
GGML_ASSERT(cur_p.selected >= 0);
|
||||
GGML_ASSERT((size_t) cur_p.selected < cur_p.size);
|
||||
return cur_p.data[cur_p.selected].id;
|
||||
}
|
||||
|
||||
static void test_dist_singleton_rng() {
|
||||
llama_sampler * singleton = llama_sampler_init_dist(4242);
|
||||
llama_sampler * control = llama_sampler_init_dist(4242);
|
||||
|
||||
sample_dist(singleton, { 0.0f });
|
||||
sample_dist(control, { 0.0f, 0.0f });
|
||||
|
||||
const std::vector<float> logits(256, 0.0f);
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
GGML_ASSERT(sample_dist(singleton, logits) == sample_dist(control, logits));
|
||||
}
|
||||
|
||||
llama_sampler_free(singleton);
|
||||
llama_sampler_free(control);
|
||||
}
|
||||
|
||||
static void test_temp(const std::vector<float> & probs, const std::vector<float> & probs_expected, float temp) {
|
||||
sampler_tester tester(probs, probs_expected);
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(llama_sampler_init_temp(temp));
|
||||
tester.apply(llama_sampler_init_dist(0));
|
||||
DUMP(&tester.cur_p);
|
||||
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_temp_ext(const std::vector<float> & probs, const std::vector<float> & probs_expected, float temp, float delta, float exponent) {
|
||||
sampler_tester tester(probs, probs_expected);
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(llama_sampler_init_temp_ext(temp, delta, exponent));
|
||||
tester.apply(llama_sampler_init_dist (0));
|
||||
DUMP(&tester.cur_p);
|
||||
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_top_k(const std::vector<float> & probs, const std::vector<float> & probs_expected, int k) {
|
||||
sampler_tester tester(probs, probs_expected);
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(llama_sampler_init_top_k(k));
|
||||
tester.apply(llama_sampler_init_dist (0));
|
||||
DUMP(&tester.cur_p);
|
||||
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_top_p(const std::vector<float> & probs, const std::vector<float> & probs_expected, float p) {
|
||||
sampler_tester tester(probs, probs_expected);
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(llama_sampler_init_top_p(p, 0));
|
||||
tester.apply(llama_sampler_init_dist (0));
|
||||
DUMP(&tester.cur_p);
|
||||
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_min_p(const std::vector<float> & probs, const std::vector<float> & probs_expected, float p) {
|
||||
sampler_tester tester(probs, probs_expected);
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(llama_sampler_init_min_p(p, 0));
|
||||
tester.apply(llama_sampler_init_dist (0));
|
||||
DUMP(&tester.cur_p);
|
||||
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_xtc(const std::vector<float> & probs, const std::vector<float> & probs_expected, float p, float t) {
|
||||
sampler_tester tester(probs, probs_expected);
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(llama_sampler_init_xtc(p, t, 0, 0));
|
||||
DUMP(&tester.cur_p);
|
||||
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_typical(const std::vector<float> & probs, const std::vector<float> & probs_expected, float p) {
|
||||
sampler_tester tester(probs, probs_expected);
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(llama_sampler_init_typical(p, 0));
|
||||
DUMP(&tester.cur_p);
|
||||
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_penalties(
|
||||
const std::vector<float> & probs, const std::vector<llama_token> & last_tokens,
|
||||
const std::vector<float> & probs_expected, float repeat_penalty, float alpha_frequency, float alpha_presence
|
||||
) {
|
||||
GGML_ASSERT(probs.size() == probs_expected.size());
|
||||
|
||||
sampler_tester tester(probs, probs_expected);
|
||||
|
||||
auto * sampler = llama_sampler_init_penalties((int32_t) probs.size(), (int32_t) last_tokens.size(), repeat_penalty, alpha_frequency, alpha_presence);
|
||||
|
||||
for (size_t i = 0; i < last_tokens.size(); i++) {
|
||||
llama_sampler_accept(sampler, last_tokens[i]);
|
||||
}
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(sampler);
|
||||
tester.apply(llama_sampler_init_dist(0));
|
||||
DUMP(&tester.cur_p);
|
||||
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_dry(
|
||||
const std::vector<float> & probs, const std::vector<llama_token> & last_tokens,
|
||||
const std::vector<float> & expected_probs, float dry_multiplier, float dry_base,
|
||||
int dry_allowed_length, int dry_penalty_last_n,
|
||||
const std::vector<std::vector<llama_token>> & seq_breakers
|
||||
) {
|
||||
GGML_ASSERT(probs.size() == expected_probs.size());
|
||||
|
||||
sampler_tester tester(probs, expected_probs);
|
||||
|
||||
auto * sampler = llama_sampler_init_dry_testing(dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n, seq_breakers);
|
||||
|
||||
for (size_t i = 0; i < last_tokens.size(); i++) {
|
||||
llama_sampler_accept(sampler, last_tokens[i]);
|
||||
}
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(sampler);
|
||||
tester.apply(llama_sampler_init_dist(0));
|
||||
DUMP(&tester.cur_p);
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_top_n_sigma(const std::vector<float> & probs, const std::vector<float> & probs_expected, int n) {
|
||||
sampler_tester tester(probs, probs_expected);
|
||||
|
||||
DUMP(&tester.cur_p);
|
||||
tester.apply(llama_sampler_init_top_n_sigma(n));
|
||||
tester.apply(llama_sampler_init_dist (0));
|
||||
DUMP(&tester.cur_p);
|
||||
|
||||
tester.check();
|
||||
}
|
||||
|
||||
static void test_sampler_queue(const size_t n_vocab, const std::string & samplers_sequence, const int top_k, const float top_p, const float min_p
|
||||
) {
|
||||
sampler_tester tester(n_vocab);
|
||||
|
||||
llama_token min_token_id = 0;
|
||||
const llama_token max_token_id = n_vocab - 1;
|
||||
|
||||
for (auto s : samplers_sequence) {
|
||||
switch (s) {
|
||||
case 'k': tester.apply(llama_sampler_init_top_k(top_k)); break;
|
||||
case 'y': GGML_ABORT("typical test not implemented");
|
||||
case 'p': tester.apply(llama_sampler_init_top_p(top_p, 1)); break;
|
||||
case 'm': tester.apply(llama_sampler_init_min_p(min_p, 1)); break;
|
||||
case 't': GGML_ABORT("temperature test not implemented");
|
||||
default : GGML_ABORT("Unknown sampler");
|
||||
}
|
||||
|
||||
tester.apply(llama_sampler_init_dist(0));
|
||||
|
||||
auto & cur_p = tester.cur_p;
|
||||
|
||||
const int size = cur_p.size;
|
||||
|
||||
if (s == 'k') {
|
||||
const int expected_size = std::min(size, top_k);
|
||||
min_token_id = std::max(min_token_id, (llama_token)(n_vocab - top_k));
|
||||
|
||||
GGML_ASSERT(size == expected_size);
|
||||
GGML_ASSERT(cur_p.data[0].id == max_token_id);
|
||||
GGML_ASSERT(cur_p.data[expected_size-1].id == min_token_id);
|
||||
} else if (s == 'p') {
|
||||
const int softmax_divisor = n_vocab * (n_vocab-1) / 2 - min_token_id * (min_token_id-1) / 2;
|
||||
const int softmax_numerator_target = ceilf(top_p * softmax_divisor);
|
||||
|
||||
min_token_id = n_vocab;
|
||||
int expected_size = 0;
|
||||
int cumsum = 0;
|
||||
do { // do-while because always at least one token is sampled
|
||||
min_token_id--;
|
||||
expected_size++;
|
||||
|
||||
cumsum += min_token_id;
|
||||
} while (cumsum < softmax_numerator_target);
|
||||
|
||||
// token 0 has p == 0, need special consideration for cumsum because top_p immediately returns
|
||||
if (min_token_id == 1) {
|
||||
min_token_id--;
|
||||
expected_size += 1;
|
||||
}
|
||||
|
||||
GGML_ASSERT(size == expected_size);
|
||||
GGML_ASSERT(!cur_p.sorted || cur_p.data[0].id == max_token_id);
|
||||
GGML_ASSERT(!cur_p.sorted || cur_p.data[expected_size-1].id == min_token_id);
|
||||
} else if (s == 'm') {
|
||||
int expected_size = ceilf((1.0f - min_p) * n_vocab);
|
||||
expected_size = std::max(expected_size, 1);
|
||||
expected_size = std::min(expected_size, size);
|
||||
|
||||
min_token_id = floorf(min_p * n_vocab);
|
||||
min_token_id = std::max(min_token_id, 1);
|
||||
min_token_id = std::max(min_token_id, (llama_token)(n_vocab - size));
|
||||
min_token_id = std::min(min_token_id, (llama_token)(n_vocab - 1));
|
||||
|
||||
GGML_ASSERT(size == expected_size);
|
||||
GGML_ASSERT(!cur_p.sorted || cur_p.data[0].id == max_token_id);
|
||||
GGML_ASSERT(!cur_p.sorted || cur_p.data[expected_size-1].id == min_token_id);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
}
|
||||
|
||||
printf("Sampler queue %3s OK with n_vocab=%05zu top_k=%5d top_p=%f min_p=%f\n",
|
||||
samplers_sequence.c_str(), n_vocab, top_k, top_p, min_p);
|
||||
}
|
||||
|
||||
static void bench(llama_sampler * cnstr, const char * cnstr_name, const std::vector<llama_token_data> & data, int n_iter) {
|
||||
std::vector<llama_token_data> cur(data.size());
|
||||
std::copy(data.begin(), data.end(), cur.begin());
|
||||
llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
|
||||
llama_sampler_apply(cnstr, &cur_p);
|
||||
llama_sampler_reset(cnstr);
|
||||
const int64_t t_start = ggml_time_us();
|
||||
for (int i = 0; i < n_iter; i++) {
|
||||
std::copy(data.begin(), data.end(), cur.begin());
|
||||
llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
|
||||
llama_sampler_apply(cnstr, &cur_p);
|
||||
llama_sampler_reset(cnstr);
|
||||
}
|
||||
const int64_t t_end = ggml_time_us();
|
||||
llama_sampler_free(cnstr);
|
||||
printf("%-43s: %8.3f us/iter\n", cnstr_name, (t_end - t_start) / (float)n_iter);
|
||||
}
|
||||
|
||||
#define BENCH(__cnstr, __data, __n_iter) bench((__cnstr), #__cnstr, (__data), (__n_iter))
|
||||
|
||||
static void test_perf() {
|
||||
const int n_vocab = 1 << 17;
|
||||
|
||||
std::vector<llama_token_data> data;
|
||||
|
||||
data.reserve(n_vocab);
|
||||
for (int i = 0; i < n_vocab; i++) {
|
||||
const float logit = 2.0f*((double)(rand())/RAND_MAX - 0.5);
|
||||
data.emplace_back(llama_token_data{i, logit, 0.0f});
|
||||
}
|
||||
|
||||
BENCH(llama_sampler_init_top_k (40), data, 32);
|
||||
BENCH(llama_sampler_init_top_p (0.8f, 1), data, 32);
|
||||
BENCH(llama_sampler_init_min_p (0.2f, 1), data, 32);
|
||||
BENCH(llama_sampler_init_typical(0.5f, 1), data, 32);
|
||||
BENCH(llama_sampler_init_xtc (1.0f, 0.1f, 1, 1), data, 32);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
ggml_time_init();
|
||||
|
||||
test_dist_singleton_rng();
|
||||
|
||||
test_temp({0.1f, 0.2f, 0.3f, 0.4f}, {0.1f, 0.2f, 0.3f, 0.4f}, 1.0f);
|
||||
test_temp({0.1f, 0.2f, 0.3f, 0.4f}, {0.0f, 0.0f, 0.0f, 1.0f}, 0.0f);
|
||||
|
||||
test_temp_ext({0.1f, 0.2f, 0.3f, 0.4f}, {0.1f, 0.2f, 0.3f, 0.4f}, 1.0f, 0.0f, 1.0f);
|
||||
test_temp_ext({0.1f, 0.2f, 0.3f, 0.4f}, {0.0f, 0.0f, 0.0f, 1.0f}, 0.0f, 0.0f, 1.0f);
|
||||
|
||||
test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {1.0f}, 1);
|
||||
test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.44444f, 0.33333f, 0.22222f}, 3);
|
||||
test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f, 0.3f, 0.2f, 0.1f}, 4);
|
||||
test_top_k({0.1f, 0.2f, 0.3f, 0.4f}, {0.1f, 0.2f, 0.3f, 0.4f}, 0);
|
||||
|
||||
test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {1.0f}, 0);
|
||||
test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.571429f, 0.428571f}, 0.7f);
|
||||
test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.44444f, 0.33333f, 0.22222f}, 0.8f);
|
||||
test_top_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.1f, 0.2f, 0.3f, 0.4f}, 1.0f);
|
||||
|
||||
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.1f/1.0f, 0.2f/1.0f, 0.3f/1.0f, 0.4f/1.0f}, 0.00f);
|
||||
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.1f/1.0f, 0.2f/1.0f, 0.3f/1.0f, 0.4f/1.0f}, 0.24f);
|
||||
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.2f/0.9f, 0.3f/0.9f, 0.4f/0.9f}, 0.26f);
|
||||
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.2f/0.9f, 0.3f/0.9f, 0.4f/0.9f}, 0.49f);
|
||||
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.3f/0.7f, 0.4f/0.7f}, 0.51f);
|
||||
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.3f/0.7f, 0.4f/0.7f}, 0.74f);
|
||||
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.4f}, 0.76f);
|
||||
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.4f}, 1.00f);
|
||||
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.4f}, 1.05f);
|
||||
|
||||
printf("XTC should:\n");
|
||||
test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.1f}, 0.99f, 0.09f);
|
||||
test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.2f, 0.1f}, 0.99f, 0.19f);
|
||||
test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.3f, 0.2f, 0.1f}, 0.99f, 0.29f);
|
||||
|
||||
printf("XTC should not:\n");
|
||||
test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.4f, 0.3f, 0.2f, 0.1f}, 0.99f, 0.39f);
|
||||
|
||||
test_typical({0.97f, 0.01f, 0.01f, 0.01f}, {0.97f}, 0.5f);
|
||||
test_typical({0.4f, 0.2f, 0.2f, 0.2f}, {0.2f, 0.2f, 0.2f}, 0.5f);
|
||||
|
||||
test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0}, {0, 0.25f, 0.25f, 0.25f, 0.25f}, 50.0f, 0.0f, 0.0f);
|
||||
test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2}, {0, 0, 0, 0.5f, 0.5f}, 50.0f, 0.0f, 0.0f);
|
||||
test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2, 0, 0}, {0, 0, 0, 0.5f, 0.5f}, 50.0f, 0.0f, 0.0f);
|
||||
|
||||
test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0}, {0.000011f, 0.249997f, 0.249997f, 0.249997f, 0.249997f}, 1.0f, 5.0f, 5.0f);
|
||||
test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2}, {0.000023f, 0.000023f, 0.000023f, 0.499966f, 0.499966f}, 1.0f, 5.0f, 5.0f);
|
||||
test_penalties({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2, 0, 0}, {0.000000f, 0.000023f, 0.000023f, 0.499977f, 0.499977f}, 1.0f, 5.0f, 5.0f);
|
||||
|
||||
|
||||
test_dry({0.25f, 0.25f, 0.25f, 0.25f}, {0, 1}, {0.25f, 0.25f, 0.25f, 0.25f}, 1.0f, 1.1f, 2, 4, {});
|
||||
test_dry({0.25f, 0.25f, 0.25f, 0.25f}, {0, 1, 2, 0, 1}, {0.296923f, 0.296923f, 0.109232f, 0.296923f}, 1.0f, 1.1f, 2, 5, {});
|
||||
test_dry({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 3, 4, 0, 1}, {0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, 1.0f, 1.1f, 2, 6, {{3}});
|
||||
test_dry({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2, 0, 1}, {0.241818f, 0.241818f, 0.032727f, 0.241818f, 0.241818f}, 2.0f, 1.1f, 2, 5, {});
|
||||
test_dry({0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, {0, 1, 2, 3, 4, 0, 1}, {0.2f, 0.2f, 0.2f, 0.2f, 0.2f}, 1.0f, 1.1f, 4, 7, {});
|
||||
|
||||
test_top_n_sigma({0.1f, 0.2f, 0.3f, 0.4f}, {0.0f, 0.0f, 0.428571f, 0.571429f}, 1.00f);
|
||||
test_top_n_sigma({0.1f, 0.2f, 0.3f, 0.4f}, {0.1f, 0.2f, 0.3f, 0.4f}, 0.00f); // top_n_sigma == 0 now represents a no-op rather than greedy decoding as of PR#13345
|
||||
test_top_n_sigma({0.1f, 0.2f, 0.3f, 0.4f}, {0.1f, 0.2f, 0.3f, 0.4f}, 3.00f);
|
||||
|
||||
test_sampler_queue(10000, "k", 10000, 1.0f, 1.0f);
|
||||
test_sampler_queue(10000, "k", 1, 1.0f, 1.0f);
|
||||
test_sampler_queue(10000, "p", 10000, 1.0f, 1.0f);
|
||||
test_sampler_queue(10000, "p", 10000, 0.0f, 1.0f);
|
||||
test_sampler_queue(10000, "m", 10000, 1.0f, 1.0f);
|
||||
test_sampler_queue(10000, "m", 10000, 1.0f, 1e-12);
|
||||
|
||||
test_sampler_queue(10000, "k", 100, 1.0000f, 1.0f);
|
||||
test_sampler_queue(10000, "p", 10000, 0.0003f, 1.0f);
|
||||
test_sampler_queue(10000, "p", 10000, 0.8000f, 1.0f);
|
||||
test_sampler_queue(10000, "m", 10000, 1.0000f, 9997.9f/9999.0f);
|
||||
test_sampler_queue(10000, "m", 10000, 1.0000f, 0.1f);
|
||||
|
||||
test_sampler_queue(10000, "kp", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "km", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "pk", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "pm", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "mk", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "mp", 100, 0.8f, 9997.9f/9999.0f);
|
||||
test_sampler_queue(10000, "mp", 100, 0.8f, 0.1f);
|
||||
|
||||
test_sampler_queue(10000, "kpm", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "kmp", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "pkm", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "pmk", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "mkp", 100, 0.8f, 0.1f);
|
||||
test_sampler_queue(10000, "mpk", 100, 0.8f, 0.1f);
|
||||
|
||||
printf("OK\n");
|
||||
|
||||
test_perf();
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,581 @@
|
||||
#include "testing.h"
|
||||
|
||||
#include "../tools/server/server-common.h"
|
||||
#include "../tools/server/server-decision.h"
|
||||
#include "../tools/server/server-schema.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
// unit tests for server internal components, no model required
|
||||
// NOTE: end-to-end tests live in tools/server/tests
|
||||
// see skills/code-review/SKILL.md for more
|
||||
|
||||
//
|
||||
// server_tokens (text only, has_mtmd = false)
|
||||
//
|
||||
|
||||
static void test_server_tokens_basic(testing & t) {
|
||||
server_tokens st(llama_tokens{1, 2, 3}, false);
|
||||
t.assert_equal("size", (size_t) 3, st.size());
|
||||
t.assert_true("not empty", !st.empty());
|
||||
t.assert_equal("operator[]", 2, st[1]);
|
||||
|
||||
st.push_back(4);
|
||||
t.assert_equal("push_back", (size_t) 4, st.size());
|
||||
t.assert_true("get_tokens", st.get_tokens() == llama_tokens({1, 2, 3, 4}));
|
||||
|
||||
st.insert({5, 6});
|
||||
t.assert_true("insert appends", st.get_tokens() == llama_tokens({1, 2, 3, 4, 5, 6}));
|
||||
|
||||
st.set_token(0, 9);
|
||||
t.assert_equal("set_token", 9, st[0]);
|
||||
|
||||
bool thrown = false;
|
||||
try {
|
||||
st.push_back(LLAMA_TOKEN_NULL);
|
||||
} catch (const std::exception &) {
|
||||
thrown = true;
|
||||
}
|
||||
t.assert_true("push_back(LLAMA_TOKEN_NULL) throws", thrown);
|
||||
t.assert_equal("size unchanged after throw", (size_t) 6, st.size());
|
||||
|
||||
st.clear();
|
||||
t.assert_true("clear", st.empty());
|
||||
}
|
||||
|
||||
static void test_server_tokens_pos(testing & t) {
|
||||
server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
|
||||
t.assert_equal("pos_next()", (llama_pos) 5, st.pos_next());
|
||||
t.assert_equal("pos_next(0)", (llama_pos) 0, st.pos_next(0));
|
||||
t.assert_equal("pos_next(3)", (llama_pos) 3, st.pos_next(3));
|
||||
|
||||
t.assert_equal("size_up_to_pos(0)", (size_t) 0, st.size_up_to_pos(0));
|
||||
t.assert_equal("size_up_to_pos(3)", (size_t) 3, st.size_up_to_pos(3));
|
||||
t.assert_equal("size_up_to_pos clamps", (size_t) 5, st.size_up_to_pos(100));
|
||||
}
|
||||
|
||||
static void test_server_tokens_keep_first(testing & t) {
|
||||
server_tokens st(llama_tokens{1, 2, 3, 4, 5}, false);
|
||||
st.keep_first(5);
|
||||
t.assert_equal("keep all", (size_t) 5, st.size());
|
||||
st.keep_first(2);
|
||||
t.assert_true("keep 2", st.get_tokens() == llama_tokens({1, 2}));
|
||||
st.keep_first(0);
|
||||
t.assert_true("keep 0", st.empty());
|
||||
}
|
||||
|
||||
static void test_server_tokens_common_prefix(testing & t) {
|
||||
server_tokens a(llama_tokens{1, 2, 3, 4}, false);
|
||||
server_tokens empty(llama_tokens{}, false);
|
||||
|
||||
t.assert_equal("both empty", (size_t) 0, empty.get_common_prefix(empty));
|
||||
t.assert_equal("one empty", (size_t) 0, a.get_common_prefix(empty));
|
||||
t.assert_equal("identical", (size_t) 4, a.get_common_prefix(a));
|
||||
t.assert_equal("diverge at 2", (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 9, 4}, false)));
|
||||
t.assert_equal("diverge at 0", (size_t) 0, a.get_common_prefix(server_tokens(llama_tokens{9, 2, 3, 4}, false)));
|
||||
t.assert_equal("b is prefix", (size_t) 2, a.get_common_prefix(server_tokens(llama_tokens{1, 2}, false)));
|
||||
t.assert_equal("a is prefix", (size_t) 4, a.get_common_prefix(server_tokens(llama_tokens{1, 2, 3, 4, 5, 6}, false)));
|
||||
}
|
||||
|
||||
static void test_server_tokens_serialize(testing & t) {
|
||||
// same packing as slot save/restore in server-context.cpp
|
||||
auto pack = [](const std::vector<char> & data) {
|
||||
llama_tokens packed(data.size() / sizeof(llama_token));
|
||||
memcpy(packed.data(), data.data(), data.size());
|
||||
return packed;
|
||||
};
|
||||
|
||||
t.test("roundtrip", [&](testing & t) {
|
||||
server_tokens st(llama_tokens{10, 20, 30}, false);
|
||||
std::vector<char> data = st.serialize();
|
||||
t.assert_equal("aligned to llama_token", (size_t) 0, data.size() % sizeof(llama_token));
|
||||
|
||||
server_tokens restored = server_tokens::deserialize(pack(data), false);
|
||||
t.assert_true("tokens", restored.get_tokens() == st.get_tokens());
|
||||
});
|
||||
|
||||
t.test("roundtrip empty", [&](testing & t) {
|
||||
server_tokens st(llama_tokens{}, false);
|
||||
server_tokens restored = server_tokens::deserialize(pack(st.serialize()), false);
|
||||
t.assert_true("empty", restored.empty());
|
||||
});
|
||||
|
||||
t.test("legacy plain token list", [&](testing & t) {
|
||||
server_tokens restored = server_tokens::deserialize(llama_tokens{7, 8, 9}, false);
|
||||
t.assert_true("tokens", restored.get_tokens() == llama_tokens({7, 8, 9}));
|
||||
});
|
||||
|
||||
auto expect_throw = [](testing & t, const std::string & msg, const llama_tokens & packed) {
|
||||
bool thrown = false;
|
||||
try {
|
||||
server_tokens::deserialize(packed, false);
|
||||
} catch (const std::exception &) {
|
||||
thrown = true;
|
||||
}
|
||||
t.assert_true(msg, thrown);
|
||||
};
|
||||
|
||||
t.test("corrupted", [&](testing & t) {
|
||||
llama_tokens good = pack(server_tokens(llama_tokens{10, 20, 30}, false).serialize());
|
||||
|
||||
llama_tokens bad_version = good;
|
||||
bad_version[1] = 999;
|
||||
expect_throw(t, "bad version", bad_version);
|
||||
|
||||
llama_tokens truncated = good;
|
||||
truncated.pop_back();
|
||||
expect_throw(t, "truncated", truncated);
|
||||
|
||||
llama_tokens trailing = good;
|
||||
trailing.push_back(42);
|
||||
expect_throw(t, "trailing data", trailing);
|
||||
|
||||
llama_tokens huge_count = good;
|
||||
huge_count[2] = 0x7fffffff;
|
||||
expect_throw(t, "huge token count", huge_count);
|
||||
|
||||
expect_throw(t, "marker only", llama_tokens{LLAMA_TOKEN_NULL});
|
||||
});
|
||||
}
|
||||
|
||||
static void test_server_tokens_clone(testing & t) {
|
||||
server_tokens st(llama_tokens{1, 2, 3}, false);
|
||||
server_tokens cl = st.clone();
|
||||
cl.push_back(4);
|
||||
t.assert_equal("original untouched", (size_t) 3, st.size());
|
||||
t.assert_equal("clone grew", (size_t) 4, cl.size());
|
||||
t.assert_true("get_text_tokens", cl.get_text_tokens() == llama_tokens({1, 2, 3, 4}));
|
||||
}
|
||||
|
||||
static void test_server_tokens(testing & t) {
|
||||
t.test("basic", test_server_tokens_basic);
|
||||
t.test("pos", test_server_tokens_pos);
|
||||
t.test("keep_first", test_server_tokens_keep_first);
|
||||
t.test("common_prefix", test_server_tokens_common_prefix);
|
||||
t.test("serialize", test_server_tokens_serialize);
|
||||
t.test("clone", test_server_tokens_clone);
|
||||
}
|
||||
|
||||
//
|
||||
// string / json helpers
|
||||
//
|
||||
|
||||
static void test_utf8(testing & t) {
|
||||
t.test("validate_utf8", [](testing & t) {
|
||||
t.assert_equal("empty", (size_t) 0, validate_utf8(""));
|
||||
t.assert_equal("ascii", (size_t) 3, validate_utf8("abc"));
|
||||
t.assert_equal("full 2-byte", (size_t) 3, validate_utf8("a\xC3\xA9"));
|
||||
t.assert_equal("full 3-byte", (size_t) 4, validate_utf8("a\xE2\x82\xAC"));
|
||||
t.assert_equal("full 4-byte", (size_t) 5, validate_utf8("a\xF0\x9F\x98\x80"));
|
||||
t.assert_equal("cut 2-byte (1/2)", (size_t) 1, validate_utf8("a\xC3"));
|
||||
t.assert_equal("cut 3-byte (1/3)", (size_t) 1, validate_utf8("a\xE2"));
|
||||
t.assert_equal("cut 3-byte (2/3)", (size_t) 1, validate_utf8("a\xE2\x82"));
|
||||
t.assert_equal("cut 4-byte (1/4)", (size_t) 1, validate_utf8("a\xF0"));
|
||||
t.assert_equal("cut 4-byte (2/4)", (size_t) 1, validate_utf8("a\xF0\x9F"));
|
||||
t.assert_equal("cut 4-byte (3/4)", (size_t) 1, validate_utf8("a\xF0\x9F\x98"));
|
||||
t.assert_equal("cut only", (size_t) 0, validate_utf8("\xF0\x9F\x98"));
|
||||
t.assert_equal("full then cut", (size_t) 4, validate_utf8("\xF0\x9F\x98\x80\xC3"));
|
||||
});
|
||||
|
||||
t.test("is_valid_utf8", [](testing & t) {
|
||||
t.assert_true("empty", is_valid_utf8(""));
|
||||
t.assert_true("ascii", is_valid_utf8("hello"));
|
||||
t.assert_true("multi-byte", is_valid_utf8("\xC3\xA9\xE2\x82\xAC\xF0\x9F\x98\x80"));
|
||||
t.assert_true("truncated", !is_valid_utf8("\xE2\x82"));
|
||||
t.assert_true("bad cont. byte", !is_valid_utf8("\xC3\x41"));
|
||||
t.assert_true("lone cont. byte", !is_valid_utf8("\x80"));
|
||||
t.assert_true("invalid lead", !is_valid_utf8("\xFF"));
|
||||
});
|
||||
}
|
||||
|
||||
static void test_sse(testing & t) {
|
||||
t.test("oai", [](testing & t) {
|
||||
t.assert_equal("single", std::string("data: {\"a\":1}\n\n"), format_oai_sse(json{{"a", 1}}));
|
||||
t.assert_equal("array", std::string("data: {\"a\":1}\n\ndata: {\"b\":2}\n\n"),
|
||||
format_oai_sse(json::array({json{{"a", 1}}, json{{"b", 2}}})));
|
||||
t.assert_equal("invalid utf8 replaced", std::string("data: \"x\xEF\xBF\xBD\"\n\n"), format_oai_sse(json("x\xC3")));
|
||||
});
|
||||
|
||||
t.test("oai_resp", [](testing & t) {
|
||||
json ev = {{"event", "response.created"}, {"data", {{"id", "r1"}}}};
|
||||
t.assert_equal("single", std::string("event: response.created\ndata: {\"id\":\"r1\"}\n\n"), format_oai_resp_sse(ev));
|
||||
t.assert_equal("array", std::string("event: response.created\ndata: {\"id\":\"r1\"}\n\nevent: response.created\ndata: {\"id\":\"r1\"}\n\n"),
|
||||
format_oai_resp_sse(json::array({ev, ev})));
|
||||
});
|
||||
|
||||
t.test("anthropic", [](testing & t) {
|
||||
json ev = {{"event", "message_start"}, {"data", {{"type", "message_start"}}}};
|
||||
t.assert_equal("with event", std::string("event: message_start\ndata: {\"type\":\"message_start\"}\n\n"), format_anthropic_sse(ev));
|
||||
t.assert_equal("data only", std::string("data: {\"type\":\"ping\"}\n\n"), format_anthropic_sse(json{{"type", "ping"}}));
|
||||
});
|
||||
}
|
||||
|
||||
static void test_json_helpers(testing & t) {
|
||||
t.test("json_is_array_of_numbers", [](testing & t) {
|
||||
t.assert_true("ints", json_is_array_of_numbers(json::array({1, 2, 3})));
|
||||
t.assert_true("empty array", json_is_array_of_numbers(json::array()));
|
||||
t.assert_true("float", !json_is_array_of_numbers(json::array({1, 2.5})));
|
||||
t.assert_true("string", !json_is_array_of_numbers(json::array({1, "a"})));
|
||||
t.assert_true("not an array", !json_is_array_of_numbers(json(1)));
|
||||
});
|
||||
|
||||
t.test("json_is_array_of_mixed_numbers_strings", [](testing & t) {
|
||||
t.assert_true("mixed", json_is_array_of_mixed_numbers_strings(json::array({"a", 1})));
|
||||
t.assert_true("numbers only", !json_is_array_of_mixed_numbers_strings(json::array({1, 2})));
|
||||
t.assert_true("strings only", !json_is_array_of_mixed_numbers_strings(json::array({"a", "b"})));
|
||||
t.assert_true("empty", !json_is_array_of_mixed_numbers_strings(json::array()));
|
||||
});
|
||||
|
||||
t.test("json_is_array_and_contains_numbers", [](testing & t) {
|
||||
t.assert_true("contains", json_is_array_and_contains_numbers(json::array({"a", 1})));
|
||||
t.assert_true("strings only", !json_is_array_and_contains_numbers(json::array({"a"})));
|
||||
t.assert_true("not an array", !json_is_array_and_contains_numbers(json(1)));
|
||||
});
|
||||
|
||||
t.test("json_get_nested_values", [](testing & t) {
|
||||
json js = {{"a", {{"b", {{"c", 1}}}}}, {"x", 2}};
|
||||
json res = json_get_nested_values({"a/b/c", "x", "a/b", "a/missing", "x/y"}, js);
|
||||
t.assert_equal("nested", 1, res.at("a/b/c").get<int>());
|
||||
t.assert_equal("top", 2, res.at("x").get<int>());
|
||||
t.assert_true("object", res.at("a/b") == json{{"c", 1}});
|
||||
t.assert_true("missing", !res.contains("a/missing"));
|
||||
t.assert_true("not obj", !res.contains("x/y"));
|
||||
t.assert_equal("count", (size_t) 3, res.size());
|
||||
});
|
||||
}
|
||||
|
||||
static void test_lora_helpers(testing & t) {
|
||||
// only pointer identity is used, never dereferenced
|
||||
auto * p0 = reinterpret_cast<llama_adapter_lora *>(0x10);
|
||||
auto * p1 = reinterpret_cast<llama_adapter_lora *>(0x20);
|
||||
|
||||
auto make = [](llama_adapter_lora * ptr, float scale) {
|
||||
common_adapter_lora_info info;
|
||||
info.scale = scale;
|
||||
info.ptr = ptr;
|
||||
return info;
|
||||
};
|
||||
|
||||
t.test("parse_lora_request", [](testing & t) {
|
||||
auto res = parse_lora_request(json::array({
|
||||
json{{"id", 0}, {"scale", 0.5}},
|
||||
json{{"id", 2}, {"scale", 1.0}},
|
||||
json{{"id", 0}, {"scale", 0.25}}, // last one wins
|
||||
}));
|
||||
t.assert_equal("count", (size_t) 2, res.size());
|
||||
t.assert_equal("id 0", 0.25f, res.at(0));
|
||||
t.assert_equal("id 2", 1.0f, res.at(2));
|
||||
|
||||
auto defaults = parse_lora_request(json::array({json::object()}));
|
||||
t.assert_equal("default id/scale", 0.0f, defaults.at(-1));
|
||||
});
|
||||
|
||||
t.test("are_lora_equal", [&](testing & t) {
|
||||
t.assert_true("both empty", are_lora_equal({}, {}));
|
||||
t.assert_true("same", are_lora_equal({make(p0, 1.0f)}, {make(p0, 1.0f)}));
|
||||
t.assert_true("diff scale", !are_lora_equal({make(p0, 1.0f)}, {make(p0, 0.5f)}));
|
||||
t.assert_true("diff ptr", !are_lora_equal({make(p0, 1.0f)}, {make(p1, 1.0f)}));
|
||||
t.assert_true("diff size", !are_lora_equal({make(p0, 1.0f)}, {}));
|
||||
});
|
||||
|
||||
t.test("lora_get_enabled_ids", [&](testing & t) {
|
||||
auto ids = lora_get_enabled_ids({make(p0, 0.0f), make(p1, 0.5f), make(p0, -1.0f), make(p1, 2.0f)});
|
||||
t.assert_true("ids", ids == std::vector<size_t>({1, 3}));
|
||||
});
|
||||
}
|
||||
|
||||
//
|
||||
// server_pipe
|
||||
//
|
||||
|
||||
static void test_server_pipe(testing & t) {
|
||||
auto stop_now = []() { return true; };
|
||||
|
||||
t.test("drain then eof", [](testing & t) {
|
||||
server_pipe<int> p;
|
||||
t.assert_true("write 1", p.write(1));
|
||||
t.assert_true("write 2", p.write(2));
|
||||
p.close_write();
|
||||
|
||||
int v = 0;
|
||||
t.assert_true("read 1", p.read(v, nullptr) && v == 1);
|
||||
t.assert_true("read 2", p.read(v, nullptr) && v == 2);
|
||||
t.assert_true("eof", !p.read(v, nullptr));
|
||||
});
|
||||
|
||||
t.test("broken pipe", [](testing & t) {
|
||||
server_pipe<int> p;
|
||||
p.close_read();
|
||||
t.assert_true("write fails", !p.write(1));
|
||||
});
|
||||
|
||||
t.test("max_size drops oldest", [](testing & t) {
|
||||
server_pipe<int> p;
|
||||
p.max_size = 2;
|
||||
p.write(1);
|
||||
p.write(2);
|
||||
p.write(3);
|
||||
p.close_write();
|
||||
|
||||
int v = 0;
|
||||
t.assert_true("read 2", p.read(v, nullptr) && v == 2);
|
||||
t.assert_true("read 3", p.read(v, nullptr) && v == 3);
|
||||
t.assert_true("eof", !p.read(v, nullptr));
|
||||
});
|
||||
|
||||
t.test("queued data wins over should_stop", [&](testing & t) {
|
||||
server_pipe<int> p;
|
||||
p.write(1);
|
||||
int v = 0;
|
||||
t.assert_true("read", p.read(v, stop_now) && v == 1);
|
||||
});
|
||||
|
||||
t.test("should_stop closes the pipe", [&](testing & t) {
|
||||
server_pipe<int> p;
|
||||
int v = 0;
|
||||
t.assert_true("read stopped", !p.read(v, stop_now));
|
||||
t.assert_true("write fails", !p.write(1));
|
||||
});
|
||||
|
||||
t.test("should_stop as deadline keeps the pipe open", [&](testing & t) {
|
||||
server_pipe<int> p;
|
||||
int v = 0;
|
||||
t.assert_true("read stopped", !p.read(v, stop_now, /* close_on_stop */ false));
|
||||
t.assert_true("write ok", p.write(1));
|
||||
t.assert_true("read again", p.read(v, nullptr) && v == 1);
|
||||
});
|
||||
|
||||
t.test("cross thread", [](testing & t) {
|
||||
constexpr int n = 1000;
|
||||
server_pipe<int> p;
|
||||
std::thread producer([&]() {
|
||||
for (int i = 0; i < n; i++) {
|
||||
p.write(int(i));
|
||||
}
|
||||
p.close_write();
|
||||
});
|
||||
|
||||
int v = 0;
|
||||
int count = 0;
|
||||
bool in_order = true;
|
||||
while (p.read(v, nullptr)) {
|
||||
in_order &= v == count;
|
||||
count++;
|
||||
}
|
||||
producer.join();
|
||||
|
||||
t.assert_equal("count", n, count);
|
||||
t.assert_true("in order", in_order);
|
||||
});
|
||||
}
|
||||
|
||||
//
|
||||
// server_decision_escape_special_tokens
|
||||
//
|
||||
|
||||
// "<|name|>" with U+00A6 in place of "|"
|
||||
static std::string escaped(const std::string & name) {
|
||||
return "<\xC2\xA6" + name + "\xC2\xA6>";
|
||||
}
|
||||
|
||||
static void test_decision_escape(testing & t) {
|
||||
t.test("cases", [](testing & t) {
|
||||
const std::vector<std::pair<std::string, std::string>> cases = {
|
||||
{ "", "" },
|
||||
{ "no tokens", "no tokens" },
|
||||
{ "<|im_start|>user", escaped("im_start") + "user" },
|
||||
{ "<|a_1|><|B|>", escaped("a_1") + escaped("B") },
|
||||
{ "<|<|a|>", "<|" + escaped("a") },
|
||||
{ "<|a|>|>", escaped("a") + "|>" },
|
||||
{ "<||>", "<||>" },
|
||||
{ "<|a b|>", "<|a b|>" },
|
||||
{ "<|a|b|>", "<|a|b|>" },
|
||||
{ "<|caf\xC3\xA9|>", "<|caf\xC3\xA9|>" },
|
||||
{ "<|abc", "<|abc" },
|
||||
};
|
||||
for (const auto & [text, expected] : cases) {
|
||||
t.assert_equal(text, expected, server_decision_escape_special_tokens(text));
|
||||
}
|
||||
});
|
||||
|
||||
// every string of up to 6 chars from "<|>a-" gives the text of the regex
|
||||
t.test("same as std::regex", [](testing & t) {
|
||||
static const std::regex re_special("<\\|([A-Za-z0-9_]+)\\|>");
|
||||
const std::string chars = "<|>a-";
|
||||
size_t n_strings = 0;
|
||||
size_t n_diff = 0;
|
||||
for (size_t len = 0; len <= 6; len++) {
|
||||
size_t n = 1;
|
||||
for (size_t k = 0; k < len; k++) {
|
||||
n *= chars.size();
|
||||
}
|
||||
for (size_t idx = 0; idx < n; idx++) {
|
||||
std::string text;
|
||||
for (size_t k = 0, v = idx; k < len; k++, v /= chars.size()) {
|
||||
text += chars[v % chars.size()];
|
||||
}
|
||||
if (server_decision_escape_special_tokens(text) != std::regex_replace(text, re_special, "<\xC2\xA6$1\xC2\xA6>")) {
|
||||
n_diff++;
|
||||
}
|
||||
n_strings++;
|
||||
}
|
||||
}
|
||||
t.assert_equal("strings that differ, of " + std::to_string(n_strings), (size_t) 0, n_diff);
|
||||
});
|
||||
|
||||
// std::regex overflowed the stack on these
|
||||
t.test("long input", [](testing & t) {
|
||||
const std::string name(1000000, 'a');
|
||||
t.assert_true("not closed", server_decision_escape_special_tokens("<|" + name) == "<|" + name);
|
||||
t.assert_true("closed", server_decision_escape_special_tokens("<|" + name + "|>") == escaped(name));
|
||||
});
|
||||
}
|
||||
|
||||
static void test_rerank_template(testing & t) {
|
||||
struct test_case {
|
||||
const char * name;
|
||||
std::string tmpl;
|
||||
std::string query;
|
||||
std::string doc;
|
||||
std::string expected;
|
||||
};
|
||||
const test_case cases[] = {
|
||||
{"plain inputs", "Q: {query}\nD: {document}", "query", "doc", "Q: query\nD: doc"},
|
||||
{"document placeholder in query", "Q: {query}\nD: {document}", "Explain {document}.", "doc", "Q: Explain {document}.\nD: doc"},
|
||||
{"query placeholder in query", "{query}|{document}", "{query}", "doc", "{query}|doc"},
|
||||
{"placeholders in both inputs", "{query}|{document}", "{query} {document}", "{query} {document}", "{query} {document}|{query} {document}"},
|
||||
{"repeated placeholders", "{query}|{document}|{query}|{document}", "{document}", "doc", "{document}|doc|{document}|doc"},
|
||||
{"adjacent reversed placeholders", "{document}{query}", "query", "doc", "docquery"},
|
||||
{"query only", "{query}", "{document}", "doc", "{document}"},
|
||||
{"document only", "{document}", "query", "{query}", "{query}"},
|
||||
{"no placeholders", "fixed", "{document}", "{query}", "fixed"},
|
||||
{"unknown and incomplete placeholders", "{unknown}|{query}|{document}|{query", "query", "doc", "{unknown}|query|doc|{query"},
|
||||
{"nested braces", "{{query}}|{{document}}", "query", "doc", "{query}|{doc}"},
|
||||
{"placeholder formed at boundary", "{query}ument}", "{doc", "doc", "{document}"},
|
||||
{"empty inputs", "Q: {query}\nD: {document}", "", "", "Q: \nD: "},
|
||||
{"empty template", "", "query", "doc", ""},
|
||||
};
|
||||
for (const auto & test : cases) {
|
||||
t.assert_equal(test.name, test.expected, format_rerank_template(test.tmpl, test.query, test.doc));
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// server_decision_context: the recursive functions stop at RECURSIVE_LIMIT
|
||||
//
|
||||
|
||||
static void test_decision_depth(testing & t) {
|
||||
const auto prepare = [](size_t depth) {
|
||||
server_decision_context ctx;
|
||||
ctx.type = COMMON_DECISION_TYPE_H2O_LIGHTNING; // the state goes through decision_py_dump
|
||||
std::string error;
|
||||
try {
|
||||
ctx.prepare_state(json::parse(std::string(depth, '[') + std::string(depth, ']')));
|
||||
} catch (const std::invalid_argument & e) {
|
||||
error = e.what();
|
||||
}
|
||||
return error;
|
||||
};
|
||||
t.test("state at the limit", [&](testing & t) {
|
||||
const std::string error = prepare(128);
|
||||
t.assert_true("accepted: " + error, error.empty());
|
||||
});
|
||||
t.test("deep state", [&](testing & t) {
|
||||
const std::string error = prepare(1000);
|
||||
t.assert_true("rejected as nested: " + error, error.find("nested more than 128 levels deep") != std::string::npos);
|
||||
});
|
||||
}
|
||||
|
||||
// DRY sequence breakers
|
||||
//
|
||||
|
||||
static void test_dry_sequence_breakers(testing & t) {
|
||||
common_params params_base;
|
||||
|
||||
const auto eval = [&](const json & data) {
|
||||
return server_schema::eval_llama_cmpl_schema(nullptr, params_base, {}, data);
|
||||
};
|
||||
|
||||
{
|
||||
json data;
|
||||
data["dry_sequence_breakers"] = json::array({"ab", "ab", "cd", "ab"});
|
||||
const auto params = eval(data);
|
||||
const std::vector<std::string> expected = {"ab", "cd"};
|
||||
t.assert_true("duplicates removed, order kept", params.sampling.dry_sequence_breakers == expected);
|
||||
}
|
||||
|
||||
{
|
||||
std::vector<std::string> many;
|
||||
for (int i = 0; i <= LLAMA_DRY_MAX_SEQ_BREAKERS; ++i) {
|
||||
many.push_back("b" + std::to_string(i));
|
||||
}
|
||||
json data;
|
||||
data["dry_sequence_breakers"] = many;
|
||||
|
||||
bool thrown = false;
|
||||
try {
|
||||
eval(data);
|
||||
} catch (const std::exception &) {
|
||||
thrown = true;
|
||||
}
|
||||
t.assert_true("more than the cap is rejected", thrown);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// glob_match
|
||||
//
|
||||
|
||||
static void test_glob_match(testing & t) {
|
||||
t.test("patterns", [](testing & t) {
|
||||
t.assert_true("literal", glob_match("src/main.cpp", "src/main.cpp"));
|
||||
t.assert_true("star", glob_match("src/*.cpp", "src/main.cpp"));
|
||||
t.assert_true("star stops at /", !glob_match("*.cpp", "src/main.cpp"));
|
||||
t.assert_true("double star", glob_match("**/*.cpp", "a/b/c/main.cpp"));
|
||||
t.assert_true("double star at end", glob_match("src/**", "src/a/b"));
|
||||
t.assert_true("question mark", glob_match("ma?n.cpp", "main.cpp"));
|
||||
t.assert_true("question mark not /", !glob_match("a?b", "a/b"));
|
||||
t.assert_true("class", glob_match("[a-c]x", "bx"));
|
||||
t.assert_true("negated class", !glob_match("[!a-c]x", "bx"));
|
||||
t.assert_true("unclosed class", glob_match("[ab", "[ab"));
|
||||
t.assert_true("mismatch", !glob_match("main.cpp", "main.c"));
|
||||
});
|
||||
t.test("long input", [](testing & t) {
|
||||
const std::string path(100000, 'a');
|
||||
t.assert_true("long literal", glob_match(path, path));
|
||||
t.assert_true("long path", glob_match("**", path));
|
||||
t.assert_true("too many wildcards", !glob_match(std::string(1000, '*'), "a"));
|
||||
});
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
testing t;
|
||||
|
||||
const char * verbose = getenv("LLAMA_TEST_VERBOSE");
|
||||
if (verbose) {
|
||||
t.verbose = std::string(verbose) == "1";
|
||||
}
|
||||
|
||||
if (argc > 1) {
|
||||
t.set_filter(argv[1]);
|
||||
}
|
||||
|
||||
t.test("server_tokens", test_server_tokens);
|
||||
t.test("utf8", test_utf8);
|
||||
t.test("sse", test_sse);
|
||||
t.test("json_helpers", test_json_helpers);
|
||||
t.test("lora_helpers", test_lora_helpers);
|
||||
t.test("server_pipe", test_server_pipe);
|
||||
t.test("decision_escape", test_decision_escape);
|
||||
t.test("rerank_template", test_rerank_template);
|
||||
t.test("decision_depth", test_decision_depth);
|
||||
t.test("glob_match", test_glob_match);
|
||||
t.test("dry_sequence_breakers", test_dry_sequence_breakers);
|
||||
|
||||
return t.summary();
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
#include "sampling.h"
|
||||
#include "llama.h"
|
||||
#include "common.h"
|
||||
#include "arg.h"
|
||||
#include "log.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <optional>
|
||||
#include <random>
|
||||
#include <unordered_map>
|
||||
|
||||
// Tests that common_sampler_sample_and_accept_n_rejection preserves the target
|
||||
// distribution regardless of what the draft distribution Q looks like.
|
||||
|
||||
static int test_speculative_rejection_sampling(common_params & params, int n_trials) {
|
||||
params.sampling.backend_sampling = false;
|
||||
auto llama_init = common_init_from_params(params);
|
||||
|
||||
auto * model_tgt = llama_init->model();
|
||||
auto * ctx_tgt = llama_init->context();
|
||||
if (!model_tgt || !ctx_tgt) {
|
||||
LOG_ERR("failed to initialize target model and context\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
const llama_tokens prompt = { 1, 2, 3, 1, 2 };
|
||||
|
||||
// Process the prompt with the target model to get its true distribution for
|
||||
// the position right after the prompt.
|
||||
common_batch batch = common_batch_get_one(ctx_tgt, prompt);
|
||||
if (llama_process(ctx_tgt, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
|
||||
LOG_ERR("failed to decode the test prompt\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
common_params_sampling sampling;
|
||||
sampling.seed = 18;
|
||||
sampling.samplers = { COMMON_SAMPLER_TYPE_TEMPERATURE };
|
||||
sampling.temp = 1.0f;
|
||||
|
||||
// Initialize the sampler for the target model.
|
||||
common_sampler_ptr smpl(common_sampler_init(model_tgt, sampling));
|
||||
if (!smpl) {
|
||||
LOG_ERR("failed to initialize target sampler\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Sample once so the sampler chain runs and populates cur_p. We don't need
|
||||
// the returned token id itself so the return value is ignored.
|
||||
common_sampler_sample(smpl.get(), ctx_tgt, -1);
|
||||
|
||||
// Get the target candidates distribution in sorted order.
|
||||
const llama_token_data_array * dist_tgt = common_sampler_get_candidates(smpl.get(), true);
|
||||
|
||||
// Clone smpl before the trial loop starts. We need the dist_tgt pointer to stay valid.
|
||||
common_sampler_ptr smpl_verify(common_sampler_clone(smpl.get()));
|
||||
if (!smpl_verify) {
|
||||
LOG_ERR("failed to clone target sampler\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
const llama_vocab * vocab = llama_model_get_vocab(model_tgt);
|
||||
const int n_vocab = llama_vocab_n_tokens(vocab);
|
||||
|
||||
std::mt19937 rng(18);
|
||||
std::uniform_real_distribution<float> uni(0.0f, 1.0f);
|
||||
|
||||
std::unordered_map<llama_token, int> token_counts;
|
||||
|
||||
std::vector<llama_token_data> q(n_vocab);
|
||||
for (int i = 0; i < n_vocab; ++i) {
|
||||
q[i].id = i;
|
||||
q[i].logit = 0.0f;
|
||||
}
|
||||
|
||||
for (int trial = 0; trial < n_trials; ++trial) {
|
||||
float q_sum = 0.0f;
|
||||
|
||||
for (auto & td : q) {
|
||||
td.p = uni(rng);
|
||||
q_sum += td.p;
|
||||
}
|
||||
|
||||
const float inv_sum = 1.0f / q_sum;
|
||||
for (auto & td : q) {
|
||||
td.p *= inv_sum;
|
||||
}
|
||||
|
||||
float u = uni(rng);
|
||||
|
||||
// Fallback to the last bucket to absorb residual probability from FP precision loss
|
||||
llama_token draft_tok = q.back().id;
|
||||
for (const auto & e : q) {
|
||||
u -= e.p;
|
||||
if (u <= 0.0f) {
|
||||
draft_tok = e.id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Both indices reuse the prompt's last output logits. The second entry
|
||||
// is the bonus draw slot; it is always stale and its result is ignored.
|
||||
const llama_tokens draft = { draft_tok };
|
||||
const std::vector<std::vector<llama_token_data>> draft_q = { q };
|
||||
const std::vector<int> idxs = { -1, -1 };
|
||||
|
||||
const std::vector<llama_token> result =
|
||||
common_sampler_sample_and_accept_n_rejection(smpl_verify.get(), ctx_tgt, idxs, draft, draft_q, false);
|
||||
|
||||
token_counts[result[0]]++;
|
||||
}
|
||||
|
||||
LOG("\n");
|
||||
LOG("Empirical vs target distribution after %d trials:\n", n_trials);
|
||||
|
||||
bool ok = true;
|
||||
size_t covered = 0;
|
||||
|
||||
auto get_count_for = [&](llama_token id) -> int {
|
||||
const auto it = token_counts.find(id);
|
||||
return it != token_counts.end() ? it->second : 0;
|
||||
};
|
||||
|
||||
// Iterate over the target distribution
|
||||
for (size_t i = 0; i < dist_tgt->size; ++i) {
|
||||
const llama_token id = dist_tgt->data[i].id;
|
||||
const float p_t = dist_tgt->data[i].p;
|
||||
|
||||
const int count = get_count_for(id);
|
||||
const float p_e = (float) count / n_trials;
|
||||
const float diff = std::fabs(p_t - p_e);
|
||||
|
||||
// We only deal with ids in dist_tgt, and if each trial result was one
|
||||
// of dist_tgt's candidates then covered ends up equal to the number of
|
||||
// trials. If not this inconsistency tells us how many trials produced
|
||||
// a token outside of the target's distribution.
|
||||
covered += count;
|
||||
|
||||
const float standard_error = std::sqrt(p_t * (1.0f - p_t) / n_trials);
|
||||
const float tolerance = 6.0f * standard_error;
|
||||
|
||||
LOG("token %3d : target = %f, empirical = %f, diff = %f, tol = %f\n", id, p_t, p_e, diff, tolerance);
|
||||
|
||||
if (diff > tolerance) {
|
||||
LOG_ERR("token %d: empirical probability %f too far from target %f (diff %f > tol %f)\n",
|
||||
id, p_e, p_t, diff, tolerance);
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (covered != (size_t) n_trials) {
|
||||
LOG_ERR("%zu of %d trials produced a token outside dist_tgt's candidates\n",
|
||||
(size_t) n_trials - covered, n_trials);
|
||||
ok = false;
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct parsed_args {
|
||||
int n_trials;
|
||||
std::vector<char *> argv;
|
||||
};
|
||||
|
||||
static std::optional<parsed_args> parse_n_trials_arg(int argc, char ** argv, int default_n_trials) {
|
||||
parsed_args result;
|
||||
result.n_trials = default_n_trials;
|
||||
result.argv.push_back(argv[0]);
|
||||
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
if (strcmp(argv[i], "--n-trials") == 0) {
|
||||
if (i + 1 >= argc) {
|
||||
LOG_ERR("%s: --n-trials requires a number argument\n", __func__);
|
||||
return std::nullopt;
|
||||
}
|
||||
int value = std::atoi(argv[i + 1]);
|
||||
if (value <= 0) {
|
||||
LOG_ERR("%s: --n-trials requires a positive number argument\n", __func__);
|
||||
return std::nullopt;
|
||||
}
|
||||
result.n_trials = value;
|
||||
i++;
|
||||
} else {
|
||||
result.argv.push_back(argv[i]);
|
||||
}
|
||||
}
|
||||
result.argv.push_back(nullptr);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
|
||||
common_params params;
|
||||
params.n_ctx = 256;
|
||||
|
||||
common_init();
|
||||
|
||||
auto parsed = parse_n_trials_arg(argc, argv, 5000);
|
||||
if (!parsed) {
|
||||
return 1;
|
||||
}
|
||||
const int fargc = (int) parsed->argv.size() - 1;
|
||||
|
||||
if (!common_params_parse(fargc, parsed->argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
LOG("Using target model '%s'\n", params.model.path.c_str());
|
||||
|
||||
llama_backend_init();
|
||||
|
||||
const int result = test_speculative_rejection_sampling(params, parsed->n_trials);
|
||||
if (result == 0) {
|
||||
LOG("\nAll tests passed.\n");
|
||||
} else {
|
||||
LOG("\nSome tests failed.\n");
|
||||
}
|
||||
|
||||
llama_backend_free();
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
// thread safety test
|
||||
// - Loads a copy of the same model on each GPU, plus a copy on the CPU, all in parallel
|
||||
// - Creates n_parallel (--parallel) contexts per model
|
||||
// - Runs inference in parallel on each context
|
||||
|
||||
#include <array>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <atomic>
|
||||
#include "llama.h"
|
||||
#include "arg.h"
|
||||
#include "common.h"
|
||||
#include "log.h"
|
||||
#include "sampling.h"
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
common_params params;
|
||||
|
||||
common_init();
|
||||
|
||||
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
llama_backend_init();
|
||||
llama_numa_init(params.numa);
|
||||
|
||||
LOG_INF("%s\n", common_params_get_system_info(params).c_str());
|
||||
|
||||
//llama_log_set([](ggml_log_level level, const char * text, void * /*user_data*/) {
|
||||
// if (level == GGML_LOG_LEVEL_ERROR) {
|
||||
// common_log_add(common_log_main(), level, "%s", text);
|
||||
// }
|
||||
//}, NULL);
|
||||
|
||||
auto cparams = common_context_params_to_llama(params);
|
||||
|
||||
// each context has a single sequence
|
||||
cparams.n_seq_max = 1;
|
||||
|
||||
int dev_count = ggml_backend_dev_count();
|
||||
std::vector<std::array<ggml_backend_dev_t, 2>> gpus;
|
||||
for (int i = 0; i < dev_count; ++i) {
|
||||
auto * dev = ggml_backend_dev_get(i);
|
||||
if (dev && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_GPU) {
|
||||
gpus.push_back({dev, nullptr});
|
||||
}
|
||||
}
|
||||
const int gpu_dev_count = (int)gpus.size();
|
||||
const int num_models = gpu_dev_count + 1 + 1; // GPUs + 1 CPU model + 1 layer split
|
||||
//const int num_models = std::max(1, gpu_dev_count);
|
||||
const int num_contexts = std::max(1, params.n_parallel);
|
||||
|
||||
std::vector<llama_model_ptr> models(num_models);
|
||||
std::vector<std::thread> load_threads;
|
||||
std::vector<std::thread> threads;
|
||||
std::atomic<bool> failed = false;
|
||||
|
||||
for (int m = 0; m < num_models; ++m) {
|
||||
load_threads.emplace_back([&, m]() {
|
||||
auto mparams = common_model_params_to_llama(params);
|
||||
|
||||
if (m < gpu_dev_count) {
|
||||
mparams.split_mode = LLAMA_SPLIT_MODE_NONE;
|
||||
mparams.devices = gpus[m].data();
|
||||
} else if (m == gpu_dev_count) {
|
||||
mparams.split_mode = LLAMA_SPLIT_MODE_NONE;
|
||||
mparams.main_gpu = -1; // CPU model
|
||||
} else {
|
||||
mparams.split_mode = LLAMA_SPLIT_MODE_LAYER;
|
||||
}
|
||||
|
||||
llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
|
||||
if (model == NULL) {
|
||||
LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());
|
||||
failed.store(true);
|
||||
return;
|
||||
}
|
||||
|
||||
models[m].reset(model);
|
||||
});
|
||||
}
|
||||
|
||||
for (auto & thread : load_threads) {
|
||||
thread.join();
|
||||
}
|
||||
|
||||
if (failed) {
|
||||
LOG_ERR("One or more models failed to load.\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
for (int m = 0; m < num_models; ++m) {
|
||||
auto * model = models[m].get();
|
||||
for (int c = 0; c < num_contexts; ++c) {
|
||||
threads.emplace_back([&, m, c, model]() {
|
||||
LOG_INF("Creating context %d/%d for model %d/%d\n", c + 1, num_contexts, m + 1, num_models);
|
||||
|
||||
llama_context_ptr ctx { llama_init_from_model(model, cparams) };
|
||||
if (ctx == NULL) {
|
||||
LOG_ERR("failed to create context\n");
|
||||
failed.store(true);
|
||||
return;
|
||||
}
|
||||
|
||||
std::unique_ptr<common_sampler, decltype(&common_sampler_free)> sampler { common_sampler_init(model, params.sampling), common_sampler_free };
|
||||
if (sampler == NULL) {
|
||||
LOG_ERR("failed to create sampler\n");
|
||||
failed.store(true);
|
||||
return;
|
||||
}
|
||||
|
||||
{
|
||||
auto prompt = common_tokenize(ctx.get(), params.prompt, true);
|
||||
if (prompt.empty()) {
|
||||
LOG_ERR("failed to tokenize prompt\n");
|
||||
failed.store(true);
|
||||
return;
|
||||
}
|
||||
common_batch batch = common_batch_get_one(ctx.get(), prompt);
|
||||
if (llama_process(ctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
LOG_ERR("failed to decode prompt\n");
|
||||
failed.store(true);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const auto * vocab = llama_model_get_vocab(model);
|
||||
std::string result = params.prompt;
|
||||
|
||||
for (int i = 0; i < params.n_predict; i++) {
|
||||
llama_token token = common_sampler_sample(sampler.get(), ctx.get(), -1);
|
||||
|
||||
result += common_token_to_piece(ctx.get(), token);
|
||||
|
||||
if (llama_vocab_is_eog(vocab, token)) {
|
||||
break;
|
||||
}
|
||||
|
||||
common_batch batch = common_batch_get_one(ctx.get(), &token, 1);
|
||||
|
||||
int ret = llama_process(ctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get());
|
||||
if (ret == 1 && i > 0) {
|
||||
LOG_INF("Context full, stopping generation.\n");
|
||||
break;
|
||||
}
|
||||
|
||||
if (ret != 0) {
|
||||
LOG_ERR("Model %d/%d, Context %d/%d: failed to decode\n", m + 1, num_models, c + 1, num_contexts);
|
||||
failed.store(true);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INF("Model %d/%d, Context %d/%d: %s\n\n", m + 1, num_models, c + 1, num_contexts, result.c_str());
|
||||
|
||||
llama_synchronize(ctx.get());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (auto & thread : threads) {
|
||||
thread.join();
|
||||
}
|
||||
|
||||
if (failed) {
|
||||
LOG_ERR("One or more threads failed.\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
LOG_INF("All threads finished without errors.\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,852 @@
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml.h"
|
||||
|
||||
#include <time.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#endif
|
||||
|
||||
// failed checks; main exits non-zero on any failures
|
||||
static int n_failed = 0;
|
||||
|
||||
static float * gen_rand_f32(int64_t n) {
|
||||
float * data = (float *) malloc(n * sizeof(float));
|
||||
for (int64_t i = 0; i < n; ++i) {
|
||||
data[i] = (float)rand() / (float)RAND_MAX - 0.5f;
|
||||
data[i] *= 5.0f;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
// updates pointers to max_err and rms_err
|
||||
static void compare_f32(const float * ref, const float * out, int64_t n, float * max_err, float * rms_err) {
|
||||
*max_err = 0.0f;
|
||||
double sum_sq_err = 0.0;
|
||||
for (int64_t i = 0; i < n; ++i) {
|
||||
float err = fabsf(ref[i] - out[i]);
|
||||
if (err > *max_err) {
|
||||
*max_err = err;
|
||||
}
|
||||
sum_sq_err += (double)err * err;
|
||||
}
|
||||
*rms_err = sqrt(sum_sq_err / n);
|
||||
}
|
||||
|
||||
// Fill from a flat row-major f32 source: rows * cols floats.
|
||||
static void fill_tensor(struct ggml_tensor * t, const float * src, int64_t rows, int64_t cols, ggml_type qtype) {
|
||||
GGML_ASSERT(t->ne[0] == cols);
|
||||
GGML_ASSERT(rows * cols == ggml_nelements(t));
|
||||
if (qtype == GGML_TYPE_F32) {
|
||||
ggml_backend_tensor_set(t, src, 0, ggml_nbytes(t));
|
||||
return;
|
||||
}
|
||||
void * q = malloc(ggml_nbytes(t));
|
||||
// some iq types (iq2_xxs, iq2_xs, iq1_s) steer their code choice with an imatrix; the
|
||||
// quantized bytes are shared by the reference and tiled paths, so a dummy imatrix is enough
|
||||
const float * imatrix = NULL;
|
||||
float * im = NULL;
|
||||
if (ggml_quantize_requires_imatrix(qtype)) {
|
||||
im = (float *) malloc(rows * cols * sizeof(float));
|
||||
for (int64_t i = 0; i < rows * cols; i++) im[i] = 1.0f;
|
||||
imatrix = im;
|
||||
}
|
||||
ggml_quantize_chunk(qtype, src, q, 0, rows, cols, imatrix);
|
||||
ggml_backend_tensor_set(t, q, 0, ggml_nbytes(t));
|
||||
free(q);
|
||||
free(im);
|
||||
}
|
||||
|
||||
// The CPU-specific control API is resolved through the backend registry: with
|
||||
// GGML_BACKEND_DL the CPU backend is a runtime-loaded module and its symbols
|
||||
// are not available at link time
|
||||
static void cpu_set_use_ref(ggml_backend_t backend, bool use_ref) {
|
||||
ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
|
||||
void (* set_use_ref)(ggml_backend_t, bool) =
|
||||
(void (*)(ggml_backend_t, bool)) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_use_ref");
|
||||
if (!set_use_ref) {
|
||||
fprintf(stderr, "ggml_backend_cpu_set_use_ref not available\n");
|
||||
exit(1);
|
||||
}
|
||||
set_use_ref(backend, use_ref);
|
||||
}
|
||||
|
||||
static void cpu_set_n_threads(ggml_backend_t backend, int n_threads) {
|
||||
ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));
|
||||
void (* set_n_threads)(ggml_backend_t, int) =
|
||||
(void (*)(ggml_backend_t, int)) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
|
||||
if (!set_n_threads) {
|
||||
fprintf(stderr, "ggml_backend_set_n_threads not available\n");
|
||||
exit(1);
|
||||
}
|
||||
set_n_threads(backend, n_threads);
|
||||
}
|
||||
|
||||
static void test_matmul(ggml_backend_t backend, int64_t M, int64_t N, int64_t K, ggml_type quant_type) {
|
||||
srand(0xBEEF);
|
||||
|
||||
float * src1_ref = gen_rand_f32(M * N);
|
||||
float * src0_ref = gen_rand_f32(N * K);
|
||||
float * dst_out = (float *) malloc(M * K * sizeof(float));
|
||||
float * dst_tiled = (float *) malloc(M * K * sizeof(float));
|
||||
|
||||
struct ggml_init_params ip = { 1024*1024*1024, nullptr, true };
|
||||
struct ggml_context * ctx = ggml_init(ip);
|
||||
|
||||
// ggml_mul_mat(t0, t1) computes t1 * t0^T: t0 (src0, quant) is N wide x
|
||||
// K high, t1 (src1, f32) is N wide x M high, dst = (K, M).
|
||||
struct ggml_tensor * src1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, N, M);
|
||||
struct ggml_tensor * src0 = ggml_new_tensor_2d(ctx, quant_type, N, K);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
struct ggml_tensor * dst = ggml_mul_mat(ctx, src0, src1);
|
||||
ggml_build_forward_expand(gf, dst);
|
||||
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
|
||||
|
||||
fill_tensor(src1, src1_ref, M, N, GGML_TYPE_F32);
|
||||
fill_tensor(src0, src0_ref, K, N, quant_type);
|
||||
|
||||
cpu_set_use_ref(backend, true);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, dst_out, 0, ggml_nbytes(dst));
|
||||
cpu_set_use_ref(backend, false);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, dst_tiled, 0, ggml_nbytes(dst));
|
||||
|
||||
// std vs tiled: identical quantized inputs, so any large difference here is a bug in the tiled kernel
|
||||
float max_err, rms_err;
|
||||
compare_f32(dst_out, dst_tiled, M*K, &max_err, &rms_err);
|
||||
float tol = 1e-3f;
|
||||
|
||||
printf("TEST %lldx%lld * %lldx%lld (%s): %s (max_err: %f, rms: %f, tolerance: %f)\n",
|
||||
(long long)M, (long long)N, (long long)N, (long long)K,
|
||||
ggml_type_name(quant_type), (max_err <= tol) ? "PASS" : "FAIL", max_err, rms_err, tol);
|
||||
|
||||
// if the tiled kernel deviates from std beyond quantization tolerance,
|
||||
// dump a few offenders
|
||||
if (max_err > tol) {
|
||||
int64_t shown = 0;
|
||||
for (int64_t i = 0; i < M*K && shown < 8; ++i) {
|
||||
float err = fabsf(dst_out[i] - dst_tiled[i]);
|
||||
if (err > tol) {
|
||||
printf(" tiled vs std: i=%lld (m=%lld k=%lld) std=%f tiled=%f err=%f\n",
|
||||
(long long)i, (long long)(i/K), (long long)(i%K), dst_out[i], dst_tiled[i], err);
|
||||
++shown;
|
||||
}
|
||||
}
|
||||
++n_failed;
|
||||
}
|
||||
|
||||
ggml_backend_buffer_free(buf);
|
||||
ggml_free(ctx);
|
||||
free(src1_ref); free(src0_ref); free(dst_out); free(dst_tiled);
|
||||
}
|
||||
|
||||
// Higher-dim (ne[2], ne[3] > 1) check, std (ggml_mul_mat) is trusted as the reference
|
||||
//
|
||||
// src1 (F32) : [N, M, src1_2, src1_3] ne0=N (reduction), ne1=M (out0)
|
||||
// src0 (quant): [N, K, src0_2, src0_3] ne0=N (reduction), ne1=K (out1)
|
||||
// dst = ggml_mul_mat(src0, src1) : [K, M, src1_2, src1_3]
|
||||
static void test_matmul_highdim(ggml_backend_t backend, int64_t M, int64_t N, int64_t K,
|
||||
int64_t src1_2, int64_t src1_3,
|
||||
int64_t src0_2, int64_t src0_3,
|
||||
ggml_type quant_type) {
|
||||
srand(0xBEEF);
|
||||
|
||||
const int64_t n_src1 = N*M*src1_2*src1_3;
|
||||
const int64_t n_src0 = N*K*src0_2*src0_3;
|
||||
const int64_t n_dst = K*M*src1_2*src1_3;
|
||||
|
||||
float * src1_ref = gen_rand_f32(n_src1);
|
||||
float * src0_ref = gen_rand_f32(n_src0);
|
||||
float * dst_std = (float *) malloc(n_dst * sizeof(float));
|
||||
float * dst_tiled = (float *) malloc(n_dst * sizeof(float));
|
||||
|
||||
struct ggml_init_params ip = { 1024*1024*1024, nullptr, true };
|
||||
struct ggml_context * ctx = ggml_init(ip);
|
||||
|
||||
int64_t ne_src1[4] = { N, M, src1_2, src1_3 };
|
||||
int64_t ne_src0[4] = { N, K, src0_2, src0_3 };
|
||||
struct ggml_tensor * src1 = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_src1);
|
||||
struct ggml_tensor * src0 = ggml_new_tensor(ctx, quant_type, 4, ne_src0);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
struct ggml_tensor * dst = ggml_mul_mat(ctx, src0, src1);
|
||||
ggml_build_forward_expand(gf, dst);
|
||||
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
|
||||
|
||||
// the batch dims fold into the row count (rows are contiguous, ne0 fastest)
|
||||
fill_tensor(src1, src1_ref, M * src1_2 * src1_3, N, GGML_TYPE_F32);
|
||||
fill_tensor(src0, src0_ref, K * src0_2 * src0_3, N, quant_type);
|
||||
|
||||
// reference = stock path (use_ref keeps the op off the tiled hook), tiled
|
||||
// = the gated path; same weights, same op, run twice into separate buffers
|
||||
cpu_set_use_ref(backend, true);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, dst_std, 0, ggml_nbytes(dst));
|
||||
cpu_set_use_ref(backend, false);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, dst_tiled, 0, ggml_nbytes(dst));
|
||||
|
||||
// max |dst|: scale for the quantization tolerance
|
||||
float scale = 0.0f;
|
||||
for (int64_t i = 0; i < n_dst; ++i) {
|
||||
scale = fmaxf(scale, fabsf(dst_std[i]));
|
||||
}
|
||||
|
||||
// std vs tiled: identical quantized inputs, so a large difference is a bug
|
||||
float max_err, rms_err;
|
||||
compare_f32(dst_std, dst_tiled, n_dst, &max_err, &rms_err);
|
||||
float tol = (quant_type == GGML_TYPE_F32) ? 1e-4f : fmaxf(1e-3f, 1e-3f*scale);
|
||||
|
||||
// which path did the tiled op take (mirrors the hard-constraint gate; the
|
||||
// harness runs with force on, so the profitability check is bypassed)
|
||||
bool tiled_kernel = (N % 256 == 0) && (src0_2 > 0) && (src0_3 > 0)
|
||||
&& (src1_2 % src0_2 == 0) && (src1_3 % src0_3 == 0);
|
||||
|
||||
printf("TEST %lldx%lldx%lldx%lld * %lldx%lldx%lldx%lld (%s, %s): %s (max_err: %f, rms: %f, scale: %f)\n",
|
||||
(long long)M, (long long)N, (long long)src1_2, (long long)src1_3,
|
||||
(long long)K, (long long)N, (long long)src0_2, (long long)src0_3,
|
||||
ggml_type_name(quant_type), tiled_kernel ? "tiled-kernel" : "stock",
|
||||
(max_err <= tol) ? "PASS" : "FAIL", max_err, rms_err, scale);
|
||||
|
||||
if (max_err > tol) {
|
||||
int64_t shown = 0;
|
||||
for (int64_t i = 0; i < n_dst && shown < 8; ++i) {
|
||||
float err = fabsf(dst_std[i] - dst_tiled[i]);
|
||||
if (err > tol) {
|
||||
printf(" tiled vs std: i=%lld std=%f tiled=%f err=%f\n",
|
||||
(long long)i, dst_std[i], dst_tiled[i], err);
|
||||
++shown;
|
||||
}
|
||||
}
|
||||
++n_failed;
|
||||
}
|
||||
|
||||
ggml_backend_buffer_free(buf);
|
||||
ggml_free(ctx);
|
||||
free(src1_ref); free(src0_ref); free(dst_std); free(dst_tiled);
|
||||
}
|
||||
|
||||
// MUL_MAT_ID (MoE) check, std (vec_dot) is trusted as the reference
|
||||
//
|
||||
// src0 (as, quant): [K, R, n_experts] ne0 = K (reduction), ne1 = R rows per expert, ne2 = experts
|
||||
// src1 (b, F32) : [K, b_slots, batch] ne0 = K (reduction), ne1 = b_slots (b rows, broadcast over k columns)
|
||||
// ids : [k, batch] expert picked per (slot, batch row); k % b_slots == 0
|
||||
// dst = ggml_mul_mat_id(as, b, ids) : [R, k, batch]; column (id, t) = GEMV of src1 row (id % b_slots + t*b_slots)
|
||||
// against expert ids[t*k+id]. b_slots = 1 is the common MoE case (all top-k experts see one input row).
|
||||
static void test_mul_mat_id(ggml_backend_t backend, int64_t K, int64_t R, int64_t n_experts,
|
||||
int64_t k, int64_t b_slots, int64_t batch, ggml_type quant_type,
|
||||
bool src1_strided = false) {
|
||||
srand(0xBEEF);
|
||||
|
||||
const int64_t n_as = K * R * n_experts;
|
||||
const int64_t n_b = K * b_slots * batch;
|
||||
const int64_t n_dst = R * k * batch;
|
||||
|
||||
float * as_ref = gen_rand_f32(n_as);
|
||||
float * b_ref = gen_rand_f32(n_b);
|
||||
int32_t * ids = (int32_t *) malloc(k * batch * sizeof(int32_t));
|
||||
float * dst_ref = (float *) malloc(n_dst * sizeof(float));
|
||||
float * dst_tiled = (float *) malloc(n_dst * sizeof(float));
|
||||
|
||||
struct ggml_init_params ip = { 1024*1024*1024, nullptr, true };
|
||||
struct ggml_context * ctx = ggml_init(ip);
|
||||
|
||||
int64_t ne_as[4] = { K, R, n_experts, 1 };
|
||||
int64_t ne_b[4] = { K, b_slots, batch, 1 };
|
||||
int64_t ne_ids[4] = { k, batch, 1, 1 };
|
||||
struct ggml_tensor * src0 = ggml_new_tensor(ctx, quant_type, 4, ne_as);
|
||||
struct ggml_tensor * ids_t = ggml_new_tensor(ctx, GGML_TYPE_I32, 4, ne_ids);
|
||||
struct ggml_tensor * src1;
|
||||
struct ggml_tensor * src1_base = NULL;
|
||||
if (src1_strided) {
|
||||
// strided view: nb[1] = 4 bytes, the tiled path reads src1 through strides into wdata
|
||||
src1_base = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, b_slots * batch);
|
||||
src1 = ggml_view_3d(ctx, src1_base, K, b_slots, batch, 4, (size_t) b_slots * K * 4, 0);
|
||||
} else {
|
||||
src1 = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_b);
|
||||
}
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
struct ggml_tensor * dst = ggml_mul_mat_id(ctx, src0, src1, ids_t);
|
||||
ggml_build_forward_expand(gf, dst);
|
||||
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
|
||||
|
||||
// deterministic balanced routing: each expert gets ~ k*batch/n_experts rows
|
||||
for (int64_t t = 0; t < batch; t++) {
|
||||
for (int64_t e = 0; e < k; e++) {
|
||||
ids[t * k + e] = (int32_t) ((t * k + e) % n_experts);
|
||||
}
|
||||
}
|
||||
|
||||
fill_tensor(src1, b_ref, b_slots * batch, K, GGML_TYPE_F32);
|
||||
fill_tensor(src0, as_ref, R * n_experts, K, quant_type);
|
||||
ggml_backend_tensor_set(ids_t, ids, 0, ggml_nbytes(ids_t));
|
||||
|
||||
cpu_set_use_ref(backend, true);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, dst_ref, 0, ggml_nbytes(dst));
|
||||
cpu_set_use_ref(backend, false);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, dst_tiled, 0, ggml_nbytes(dst));
|
||||
|
||||
// same quantized inputs, so a large difference is a bug in the tiled path
|
||||
float max_err, rms_err;
|
||||
compare_f32(dst_ref, dst_tiled, n_dst, &max_err, &rms_err);
|
||||
float tol = 1e-3f;
|
||||
|
||||
printf("TEST mul_mat_id %lldx%lldx%lld * %lldx%lldx%lld ids[%lld,%lld] (%s%s): %s (max_err: %f, rms: %f, tolerance: %f)\n",
|
||||
(long long)K, (long long)R, (long long)n_experts,
|
||||
(long long)K, (long long)b_slots, (long long)batch,
|
||||
(long long)k, (long long)batch,
|
||||
ggml_type_name(quant_type), src1_strided ? ", strided-b" : "",
|
||||
(max_err <= tol) ? "PASS" : "FAIL", max_err, rms_err, tol);
|
||||
|
||||
if (max_err > tol) {
|
||||
int64_t shown = 0;
|
||||
for (int64_t i = 0; i < n_dst && shown < 8; ++i) {
|
||||
float err = fabsf(dst_ref[i] - dst_tiled[i]);
|
||||
if (err > tol) {
|
||||
printf(" tiled vs std: i=%lld (row=%lld slot=%lld batch=%lld) std=%f tiled=%f err=%f\n",
|
||||
(long long)i, (long long)(i % R), (long long)((i / R) % k), (long long)(i / (R * k)),
|
||||
dst_ref[i], dst_tiled[i], err);
|
||||
++shown;
|
||||
}
|
||||
}
|
||||
++n_failed;
|
||||
}
|
||||
|
||||
ggml_backend_buffer_free(buf);
|
||||
ggml_free(ctx);
|
||||
free(as_ref); free(b_ref); free(ids); free(dst_ref); free(dst_tiled);
|
||||
}
|
||||
|
||||
static double time_graph_compute(ggml_backend_t backend, struct ggml_cgraph * gf) {
|
||||
#if defined(_WIN32)
|
||||
// high-res timer; clock_gettime is not portable across Windows toolchains
|
||||
LARGE_INTEGER freq, t0, t1;
|
||||
QueryPerformanceFrequency(&freq);
|
||||
QueryPerformanceCounter(&t0);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
QueryPerformanceCounter(&t1);
|
||||
return (double) (t1.QuadPart - t0.QuadPart) / (double) freq.QuadPart;
|
||||
#else
|
||||
struct timespec t0, t1;
|
||||
clock_gettime(CLOCK_MONOTONIC, &t0);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
clock_gettime(CLOCK_MONOTONIC, &t1);
|
||||
return (t1.tv_sec - t0.tv_sec) + (t1.tv_nsec - t0.tv_nsec) / 1e9;
|
||||
#endif
|
||||
}
|
||||
|
||||
// Warm up, then time n runs and return the best
|
||||
static double time_graph_compute_best(ggml_backend_t backend, struct ggml_cgraph * gf, int n,
|
||||
void * flush_buf = NULL, size_t flush_size = 0) {
|
||||
|
||||
time_graph_compute(backend, gf); // warmup
|
||||
double best = 1e30;
|
||||
for (int i = 0; i < n; ++i) {
|
||||
// Evict L3 cache first
|
||||
if (flush_buf) { memset(flush_buf, 0xAB, flush_size); }
|
||||
const double t = time_graph_compute(backend, gf);
|
||||
if (t < best) best = t;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
// Fetch the repack extra buffer type through the public proc-address API
|
||||
static ggml_backend_buffer_type_t get_cpu_repack_buft(void) {
|
||||
ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
|
||||
if (!cpu_dev) {
|
||||
return NULL;
|
||||
}
|
||||
ggml_backend_reg_t cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);
|
||||
ggml_backend_dev_get_extra_bufts_t get_extra =
|
||||
(ggml_backend_dev_get_extra_bufts_t) ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_dev_get_extra_bufts");
|
||||
if (!get_extra) {
|
||||
return NULL;
|
||||
}
|
||||
ggml_backend_buffer_type_t * bufts = get_extra(cpu_dev);
|
||||
// the only extra buffer type the CPU backend exposes is CPU_REPACK
|
||||
return (bufts && *bufts) ? *bufts : NULL;
|
||||
}
|
||||
|
||||
struct bench_row {
|
||||
const char * name;
|
||||
double time_std, time_repack, time_tiled;
|
||||
float max_err_repack, rmse_repack;
|
||||
float max_err_tiled, rmse_tiled;
|
||||
bool have_repack;
|
||||
};
|
||||
|
||||
// MUL_MAT three-way bench: std (default optimized GEMM) vs repack (CPU_REPACK buffer) vs
|
||||
// tiled kernel. std is timed in-process with use_ref=true (bypasses the tiled gate); tiled
|
||||
// and repack with use_ref=false (repack's tiled gate declines on the buffer's extra). repack
|
||||
// is n/a where no repack kernel exists for the type. Errors are vs the vec-only reference (use_ref).
|
||||
static bench_row bench_three_way(ggml_backend_t backend, int64_t M, int64_t N, int64_t K, ggml_type quant_type) {
|
||||
bench_row row;
|
||||
row.name = ggml_type_name(quant_type);
|
||||
row.time_std = row.time_repack = row.time_tiled = 0.0;
|
||||
row.max_err_repack = row.rmse_repack = row.max_err_tiled = row.rmse_tiled = 0.0f;
|
||||
row.have_repack = false;
|
||||
|
||||
struct ggml_init_params ip = { 1024*1024*1024, nullptr, true };
|
||||
struct ggml_context * ctx = ggml_init(ip);
|
||||
|
||||
struct ggml_tensor * src1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, N, M);
|
||||
struct ggml_tensor * src0_std = ggml_new_tensor_2d(ctx, quant_type, N, K);
|
||||
struct ggml_tensor * src0_rep = ggml_new_tensor_2d(ctx, quant_type, N, K);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
struct ggml_tensor * dst = ggml_mul_mat(ctx, src0_std, src1);
|
||||
ggml_build_forward_expand(gf, dst);
|
||||
|
||||
struct ggml_cgraph * gf_repack = ggml_new_graph(ctx);
|
||||
struct ggml_tensor * dst_repack = ggml_mul_mat(ctx, src0_rep, src1);
|
||||
ggml_build_forward_expand(gf_repack, dst_repack);
|
||||
|
||||
ggml_backend_buffer_type_t repack_buft = get_cpu_repack_buft();
|
||||
ggml_backend_buffer_t buf_rep = NULL;
|
||||
if (repack_buft && N % 8 == 0 && K % 8 == 0) {
|
||||
buf_rep = ggml_backend_buft_alloc_buffer(repack_buft, ggml_nbytes(src0_rep));
|
||||
src0_rep->buffer = buf_rep;
|
||||
src0_rep->data = ggml_backend_buffer_get_base(buf_rep);
|
||||
ggml_backend_buffer_init_tensor(buf_rep, src0_rep);
|
||||
row.have_repack = (src0_rep->extra != NULL);
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
|
||||
|
||||
srand(0xBEEF);
|
||||
float * src1_data = gen_rand_f32(M * N);
|
||||
float * src0_data = gen_rand_f32(N * K);
|
||||
fill_tensor(src1, src1_data, M, N, GGML_TYPE_F32);
|
||||
fill_tensor(src0_std, src0_data, K, N, quant_type);
|
||||
if (row.have_repack) {
|
||||
fill_tensor(src0_rep, src0_data, K, N, quant_type);
|
||||
}
|
||||
free(src1_data); free(src0_data);
|
||||
|
||||
// one warmup + N timings; best (min) wins. The flush buffer is larger than the
|
||||
// whole-chip cache, so its memset evicts L3 and each timed iteration streams DRAM.
|
||||
const size_t flush_size = 256 * 1024 * 1024;
|
||||
void * flush_buf = malloc(flush_size);
|
||||
const int n_reps = 5;
|
||||
|
||||
// std in-process (use_ref bypasses the tiled gate; the vec_dot is the same optimized one)
|
||||
cpu_set_use_ref(backend, true);
|
||||
row.time_std = time_graph_compute_best(backend, gf, n_reps, flush_buf, flush_size);
|
||||
|
||||
// tiled in-process (the parent holds TILED_MM=1 forced on)
|
||||
cpu_set_use_ref(backend, false);
|
||||
row.time_tiled = time_graph_compute_best(backend, gf, n_reps, flush_buf, flush_size);
|
||||
|
||||
// repack in-process (the tiled gate declines on the buffer's extra)
|
||||
if (row.have_repack) {
|
||||
row.time_repack = time_graph_compute_best(backend, gf_repack, n_reps, flush_buf, flush_size);
|
||||
}
|
||||
free(flush_buf);
|
||||
|
||||
// errors vs the vec-only reference (use_ref)
|
||||
float * out_ref = (float *) malloc(M * K * sizeof(float));
|
||||
float * out_tiled = (float *) malloc(M * K * sizeof(float));
|
||||
float * out_repack = (float *) malloc(M * K * sizeof(float));
|
||||
cpu_set_use_ref(backend, true);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, out_ref, 0, ggml_nbytes(dst));
|
||||
cpu_set_use_ref(backend, false);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, out_tiled, 0, ggml_nbytes(dst));
|
||||
compare_f32(out_ref, out_tiled, M * K, &row.max_err_tiled, &row.rmse_tiled);
|
||||
if (row.have_repack) {
|
||||
ggml_backend_graph_compute(backend, gf_repack);
|
||||
ggml_backend_tensor_get(dst_repack, out_repack, 0, ggml_nbytes(dst_repack));
|
||||
compare_f32(out_ref, out_repack, M * K, &row.max_err_repack, &row.rmse_repack);
|
||||
}
|
||||
free(out_ref); free(out_tiled); free(out_repack);
|
||||
|
||||
ggml_backend_buffer_free(buf_rep);
|
||||
ggml_backend_buffer_free(buf);
|
||||
ggml_free(ctx);
|
||||
return row;
|
||||
}
|
||||
|
||||
static void print_bench_table(int64_t M, int64_t N, int64_t K, const bench_row * rows, size_t n_types) {
|
||||
const double flops = 2.0 * M * N * K;
|
||||
|
||||
printf("\nBENCH dense %lldx%lld * %lldx%lld (std vs repack vs tiled), min of 5 timings, 8 threads\n",
|
||||
(long long)M, (long long)N, (long long)N, (long long)K);
|
||||
printf("%-8s %10s %12s %12s %11s %11s %17s %17s %17s %17s\n",
|
||||
"type", "std TF", "repack TF", "tiled TF", "repack/std", "tiled/std",
|
||||
"max_err(repack)", "rmse(repack)", "max_err(tiled)", "rmse(tiled)");
|
||||
for (size_t i = 0; i < n_types; ++i) {
|
||||
const bench_row * r = &rows[i];
|
||||
if (r->have_repack) {
|
||||
printf("%-8s %10.3f %12.3f %12.3f %11.2f %11.2f %17.5e %17.5e %17.5e %17.5e\n",
|
||||
r->name,
|
||||
flops / (r->time_std * 1e12),
|
||||
flops / (r->time_repack * 1e12), flops / (r->time_tiled * 1e12),
|
||||
r->time_std / r->time_repack, r->time_std / r->time_tiled,
|
||||
r->max_err_repack, r->rmse_repack, r->max_err_tiled, r->rmse_tiled);
|
||||
} else {
|
||||
printf("%-8s %10.3f %12s %12.3f %11s %11.2f %17s %17s %17.5e %17.5e\n",
|
||||
r->name,
|
||||
flops / (r->time_std * 1e12),
|
||||
"n/a", flops / (r->time_tiled * 1e12), "n/a",
|
||||
r->time_std / r->time_tiled,
|
||||
"n/a", "n/a", r->max_err_tiled, r->rmse_tiled);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct bench_row_mmid {
|
||||
const char * name;
|
||||
double time_std, time_repack, time_tiled;
|
||||
float max_err_repack, rmse_repack;
|
||||
float max_err_tiled, rmse_tiled;
|
||||
bool have_repack;
|
||||
};
|
||||
|
||||
// MUL_MAT_ID (MoE) three-way bench: std (default GEMM, use_ref) vs repack (CPU_REPACK buffer)
|
||||
// vs tiled kernel (default; TILED_MM_FORCE is set in main so ragged experts take it too).
|
||||
// Column (id, t) = GEMV of src1 row (id % b_slots + t * b_slots) against expert ids[t*k+id];
|
||||
// cne1 = k * batch / n_experts rows per expert on average. repack is n/a where no repack
|
||||
// kernel exists. Errors are vs the vec-only reference (use_ref).
|
||||
static bench_row_mmid bench_mul_mat_id(ggml_backend_t backend, int64_t K, int64_t R, int64_t n_experts,
|
||||
int64_t k, int64_t b_slots, int64_t batch, ggml_type quant_type) {
|
||||
bench_row_mmid row;
|
||||
row.name = ggml_type_name(quant_type);
|
||||
row.time_std = row.time_repack = row.time_tiled = 0.0;
|
||||
row.max_err_repack = row.rmse_repack = row.max_err_tiled = row.rmse_tiled = 0.0f;
|
||||
row.have_repack = false;
|
||||
|
||||
struct ggml_init_params ip = { 1024*1024*1024, nullptr, true };
|
||||
struct ggml_context * ctx = ggml_init(ip);
|
||||
|
||||
int64_t ne_as[4] = { K, R, n_experts, 1 };
|
||||
int64_t ne_b[4] = { K, b_slots, batch, 1 };
|
||||
int64_t ne_ids[4] = { k, batch, 1, 1 };
|
||||
struct ggml_tensor * src1 = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_b);
|
||||
struct ggml_tensor * src0_std = ggml_new_tensor(ctx, quant_type, 4, ne_as);
|
||||
struct ggml_tensor * src0_rep = ggml_new_tensor(ctx, quant_type, 4, ne_as);
|
||||
struct ggml_tensor * ids_t = ggml_new_tensor(ctx, GGML_TYPE_I32, 4, ne_ids);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
struct ggml_tensor * dst = ggml_mul_mat_id(ctx, src0_std, src1, ids_t);
|
||||
ggml_build_forward_expand(gf, dst);
|
||||
|
||||
struct ggml_cgraph * gf_repack = ggml_new_graph(ctx);
|
||||
struct ggml_tensor * dst_repack = ggml_mul_mat_id(ctx, src0_rep, src1, ids_t);
|
||||
ggml_build_forward_expand(gf_repack, dst_repack);
|
||||
|
||||
ggml_backend_buffer_type_t repack_buft = get_cpu_repack_buft();
|
||||
ggml_backend_buffer_t buf_rep = NULL;
|
||||
if (repack_buft && R % 8 == 0) {
|
||||
buf_rep = ggml_backend_buft_alloc_buffer(repack_buft, ggml_nbytes(src0_rep));
|
||||
src0_rep->buffer = buf_rep;
|
||||
src0_rep->data = ggml_backend_buffer_get_base(buf_rep);
|
||||
ggml_backend_buffer_init_tensor(buf_rep, src0_rep);
|
||||
row.have_repack = (src0_rep->extra != NULL);
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend);
|
||||
|
||||
// deterministic balanced routing: each expert gets ~ k*batch/n_experts rows
|
||||
int32_t * ids = (int32_t *) malloc(k * batch * sizeof(int32_t));
|
||||
for (int64_t t = 0; t < batch; t++) {
|
||||
for (int64_t e = 0; e < k; e++) {
|
||||
ids[t * k + e] = (int32_t) ((t * k + e) % n_experts);
|
||||
}
|
||||
}
|
||||
srand(0xBEEF);
|
||||
float * b_data = gen_rand_f32(K * b_slots * batch);
|
||||
float * as_data = gen_rand_f32(K * R * n_experts);
|
||||
fill_tensor(src1, b_data, b_slots * batch, K, GGML_TYPE_F32);
|
||||
fill_tensor(src0_std, as_data, R * n_experts, K, quant_type);
|
||||
if (row.have_repack) {
|
||||
fill_tensor(src0_rep, as_data, R * n_experts, K, quant_type);
|
||||
}
|
||||
ggml_backend_tensor_set(ids_t, ids, 0, ggml_nbytes(ids_t));
|
||||
free(b_data); free(as_data);
|
||||
|
||||
const size_t flush_size = 256 * 1024 * 1024;
|
||||
void * flush_buf = malloc(flush_size);
|
||||
const int n_reps = 5;
|
||||
|
||||
// std in-process (use_ref bypasses the tiled gate; the vec_dot is the same optimized one)
|
||||
cpu_set_use_ref(backend, true);
|
||||
row.time_std = time_graph_compute_best(backend, gf, n_reps, flush_buf, flush_size);
|
||||
|
||||
// tiled in-process (the parent holds TILED_MM=1 forced on)
|
||||
cpu_set_use_ref(backend, false);
|
||||
row.time_tiled = time_graph_compute_best(backend, gf, n_reps, flush_buf, flush_size);
|
||||
|
||||
// repack in-process (the tiled gate declines on the buffer's extra)
|
||||
if (row.have_repack) {
|
||||
row.time_repack = time_graph_compute_best(backend, gf_repack, n_reps, flush_buf, flush_size);
|
||||
}
|
||||
free(flush_buf);
|
||||
|
||||
// errors vs the vec-only reference (use_ref)
|
||||
float * out_ref = (float *) malloc(R * k * batch * sizeof(float));
|
||||
float * out_tiled = (float *) malloc(R * k * batch * sizeof(float));
|
||||
float * out_repack = (float *) malloc(R * k * batch * sizeof(float));
|
||||
cpu_set_use_ref(backend, true);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, out_ref, 0, ggml_nbytes(dst));
|
||||
cpu_set_use_ref(backend, false);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
ggml_backend_tensor_get(dst, out_tiled, 0, ggml_nbytes(dst));
|
||||
compare_f32(out_ref, out_tiled, R * k * batch, &row.max_err_tiled, &row.rmse_tiled);
|
||||
if (row.have_repack) {
|
||||
ggml_backend_graph_compute(backend, gf_repack);
|
||||
ggml_backend_tensor_get(dst_repack, out_repack, 0, ggml_nbytes(dst_repack));
|
||||
compare_f32(out_ref, out_repack, R * k * batch, &row.max_err_repack, &row.rmse_repack);
|
||||
}
|
||||
free(out_ref); free(out_tiled); free(out_repack); free(ids);
|
||||
|
||||
ggml_backend_buffer_free(buf_rep);
|
||||
ggml_backend_buffer_free(buf);
|
||||
ggml_free(ctx);
|
||||
return row;
|
||||
}
|
||||
|
||||
static void print_mmid_table(int64_t K, int64_t R, int64_t n_experts, int64_t k,
|
||||
int64_t b_slots, int64_t batch, const bench_row_mmid * rows, size_t n_types) {
|
||||
const double flops = 2.0 * (double) K * R * k * batch;
|
||||
|
||||
printf("\nBENCH mmid %lldx%lldx%lld k=%lld b_slots=%lld batch=%lld (cne1=%lld, std vs repack vs tiled), min of 5 timings, 8 threads\n",
|
||||
(long long)K, (long long)R, (long long)n_experts, (long long)k,
|
||||
(long long)b_slots, (long long)batch, (long long)(k * batch / n_experts));
|
||||
printf("%-8s %10s %12s %12s %11s %11s %17s %17s %17s %17s\n",
|
||||
"type", "std TF", "repack TF", "tiled TF", "repack/std", "tiled/std",
|
||||
"max_err(repack)", "rmse(repack)", "max_err(tiled)", "rmse(tiled)");
|
||||
for (size_t i = 0; i < n_types; ++i) {
|
||||
const bench_row_mmid * r = &rows[i];
|
||||
if (r->have_repack) {
|
||||
printf("%-8s %10.3f %12.3f %12.3f %11.2f %11.2f %17.5e %17.5e %17.5e %17.5e\n",
|
||||
r->name,
|
||||
flops / (r->time_std * 1e12),
|
||||
flops / (r->time_repack * 1e12), flops / (r->time_tiled * 1e12),
|
||||
r->time_std / r->time_repack, r->time_std / r->time_tiled,
|
||||
r->max_err_repack, r->rmse_repack, r->max_err_tiled, r->rmse_tiled);
|
||||
} else {
|
||||
printf("%-8s %10.3f %12s %12.3f %11s %11.2f %17s %17s %17.5e %17.5e\n",
|
||||
r->name,
|
||||
flops / (r->time_std * 1e12),
|
||||
"n/a", flops / (r->time_tiled * 1e12), "n/a",
|
||||
r->time_std / r->time_tiled,
|
||||
"n/a", "n/a", r->max_err_tiled, r->rmse_tiled);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
bool run_bench = false;
|
||||
bool run_fuzz = false;
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
if (strcmp(argv[i], "--bench") == 0) {
|
||||
run_bench = true;
|
||||
} else if (strcmp(argv[i], "--fuzz") == 0) {
|
||||
run_fuzz = true;
|
||||
} else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", argv[i]);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Enable tiled MM, also force tiled MM even when unprofitable for benchmarks
|
||||
#if defined(_WIN32)
|
||||
_putenv("GGML_CPU_TILED_MM=1");
|
||||
_putenv("GGML_CPU_TILED_MM_FORCE=1");
|
||||
#else
|
||||
setenv("GGML_CPU_TILED_MM", "1", 1);
|
||||
setenv("GGML_CPU_TILED_MM_FORCE", "1", 1);
|
||||
#endif
|
||||
|
||||
ggml_backend_load_all();
|
||||
ggml_backend_t backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL);
|
||||
if (!backend) {
|
||||
fprintf(stderr, "failed to initialize the CPU backend\n");
|
||||
return 1;
|
||||
}
|
||||
cpu_set_n_threads(backend, 8);
|
||||
// base case, smoke each quant type
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_Q5_K);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_Q4_K);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_Q3_K);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_Q2_K);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_IQ4_XS);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_IQ2_XXS);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_IQ2_XS);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_IQ2_S);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_IQ3_XXS);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_IQ3_S);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_IQ1_S);
|
||||
test_matmul(backend, 256, 512, 256, GGML_TYPE_IQ1_M);
|
||||
test_mul_mat_id(backend, 256, 256, 2, 8, 1, 64, GGML_TYPE_Q4_K); // single block, cne1 = 256
|
||||
test_mul_mat_id(backend, 512, 300, 4, 2, 1, 150, GGML_TYPE_IQ2_XXS); // iq type, imatrix quant
|
||||
|
||||
if (run_fuzz) {
|
||||
printf("Running fuzz tests.\n");
|
||||
// ragged edges, both subblock lengths (Q5_K = 32, Q6_K = 16)
|
||||
test_matmul(backend, 256, 1024, 8192, GGML_TYPE_Q6_K); // long K, int32 accumulation
|
||||
test_matmul(backend, 357, 1024, 137, GGML_TYPE_Q6_K); // ragged M and K
|
||||
test_matmul(backend, 16, 1024, 16, GGML_TYPE_Q5_K);
|
||||
test_matmul(backend, 18, 1024, 7, GGML_TYPE_Q5_K); // ragged M tail, ragged K in first subblock
|
||||
test_matmul(backend, 8, 256, 8, GGML_TYPE_Q5_K);
|
||||
test_matmul(backend, 18, 1024, 256, GGML_TYPE_Q5_K);
|
||||
test_matmul(backend, 9, 256, 256, GGML_TYPE_Q5_K);
|
||||
test_matmul(backend, 8, 256, 8, GGML_TYPE_Q6_K); // tiny M, single QK_K block
|
||||
test_matmul(backend, 256, 1024, 8192, GGML_TYPE_Q3_K);
|
||||
test_matmul(backend, 357, 1024, 137, GGML_TYPE_Q3_K);
|
||||
test_matmul(backend, 8, 256, 8, GGML_TYPE_Q3_K);
|
||||
test_matmul(backend, 17, 1024, 257, GGML_TYPE_Q3_K); // K past a full 256 tile
|
||||
test_matmul(backend, 257, 1024, 17, GGML_TYPE_Q3_K); // K past a full microtile
|
||||
test_matmul(backend, 256, 1024, 8192, GGML_TYPE_Q2_K);
|
||||
test_matmul(backend, 357, 1024, 137, GGML_TYPE_Q2_K);
|
||||
test_matmul(backend, 8, 256, 8, GGML_TYPE_Q2_K);
|
||||
test_matmul(backend, 17, 1024, 257, GGML_TYPE_Q2_K);
|
||||
test_matmul(backend, 257, 1024, 17, GGML_TYPE_Q2_K);
|
||||
test_matmul(backend, 357, 1024, 137, GGML_TYPE_IQ4_XS); // ragged M and K
|
||||
test_matmul(backend, 8, 256, 8, GGML_TYPE_IQ4_XS); // tiny M, single QK_K block
|
||||
test_matmul(backend, 17, 1024, 257, GGML_TYPE_IQ4_XS); // K past a full 256 tile
|
||||
|
||||
// fuzz: M/K around the microtile (16) and tile (256) boundaries
|
||||
// Q4_K = subblock 32, Q6_K = subblock 16
|
||||
test_matmul(backend, 1, 1024, 1, GGML_TYPE_Q4_K);
|
||||
test_matmul(backend, 2, 1024, 3, GGML_TYPE_Q4_K);
|
||||
test_matmul(backend, 15, 1024, 15, GGML_TYPE_Q4_K); // M and K in the first microtile
|
||||
test_matmul(backend, 17, 1024, 17, GGML_TYPE_Q4_K); // M and K past the first microtile
|
||||
test_matmul(backend, 255, 1024, 255, GGML_TYPE_Q4_K); // M and K one short of a tile
|
||||
test_matmul(backend, 257, 1024, 257, GGML_TYPE_Q4_K); // M and K one past a tile
|
||||
test_matmul(backend, 271, 1024, 271, GGML_TYPE_Q4_K); // 1 tile + 15
|
||||
test_matmul(backend, 272, 1024, 272, GGML_TYPE_Q4_K); // 1 tile + 16
|
||||
test_matmul(backend, 511, 1024, 511, GGML_TYPE_Q4_K); // 2 tiles - 1
|
||||
test_matmul(backend, 513, 1024, 513, GGML_TYPE_Q4_K); // 2 tiles + 1
|
||||
test_matmul(backend, 17, 512, 257, GGML_TYPE_Q4_K); // ragged M and K, N = 2 blocks
|
||||
|
||||
test_matmul(backend, 1, 1024, 1, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 2, 1024, 3, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 15, 1024, 15, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 17, 1024, 17, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 255, 1024, 255, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 257, 1024, 257, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 271, 1024, 271, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 272, 1024, 272, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 511, 1024, 511, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 513, 1024, 513, GGML_TYPE_Q6_K);
|
||||
test_matmul(backend, 17, 512, 257, GGML_TYPE_Q6_K);
|
||||
|
||||
// MUL_MAT_ID (MoE): K = reduction (tiled gate needs K % 256 == 0), R = output rows per expert,
|
||||
// k = top-k slots, b_slots = b rows (1 = broadcast MoE, k = i11-diverse gather), cne1 = k*batch/E
|
||||
test_mul_mat_id(backend, 512, 300, 4, 2, 1, 150, GGML_TYPE_Q4_K); // ragged R, broadcast, cne1 = 75
|
||||
test_mul_mat_id(backend, 512, 256, 4, 2, 2, 150, GGML_TYPE_Q5_K); // i11-diverse gather, HAS_MIN
|
||||
test_mul_mat_id(backend, 512, 256, 4, 2, 1, 100, GGML_TYPE_Q6_K); // ragged cne1 = 50
|
||||
test_mul_mat_id(backend, 768, 256, 4, 2, 1, 150, GGML_TYPE_Q3_K); // K = 3 slabs
|
||||
test_mul_mat_id(backend, 1024, 2048, 128, 8, 1, 2048, GGML_TYPE_Q4_K); // large, cne1 = 128
|
||||
test_mul_mat_id(backend, 512, 256, 32, 2, 1, 8, GGML_TYPE_Q4_K); // tiny cne1 (~0.5), forced only
|
||||
|
||||
// cne1 > 256: the k-outer ring sweeps multiple 256-row windows per expert (the multi-window path)
|
||||
test_mul_mat_id(backend, 256, 256, 8, 8, 1, 512, GGML_TYPE_Q4_K); // cne1 = 512, two full windows
|
||||
test_mul_mat_id(backend, 512, 256, 8, 8, 1, 300, GGML_TYPE_Q4_K); // cne1 = 300, 2nd window ragged (44 rows)
|
||||
test_mul_mat_id(backend, 256, 64, 8, 8, 1, 257, GGML_TYPE_Q4_K); // cne1 = 257, 1-row tail window, R = 1 group
|
||||
test_mul_mat_id(backend, 512, 128, 8, 8, 1, 511, GGML_TYPE_Q4_K); // cne1 = 511, 15-row ragged tail
|
||||
test_mul_mat_id(backend, 768, 300, 8, 8, 1, 300, GGML_TYPE_Q4_K); // 3 K slabs x 2 windows, ragged R and cne1
|
||||
test_mul_mat_id(backend, 768, 300, 8, 2, 2, 300, GGML_TYPE_Q5_K); // cne1 = 600, 3 slabs, 3 windows (last ragged), i11-diverse, HAS_MIN
|
||||
|
||||
// cne1 = 255: one short of a full ring, last ring row zero-padded by the unpack
|
||||
test_mul_mat_id(backend, 512, 256, 4, 4, 1, 255, GGML_TYPE_Q4_K);
|
||||
|
||||
// long K: 8 slabs per ring row
|
||||
test_mul_mat_id(backend, 2048, 300, 8, 2, 1, 400, GGML_TYPE_Q4_K); // cne1 = 100
|
||||
|
||||
// R group edges: 64 = TILED_MMID_GROUP (ceil div, ragged group tail, single-group early return)
|
||||
test_mul_mat_id(backend, 512, 64, 4, 2, 1, 128, GGML_TYPE_Q4_K); // ngroups = 1, most threads idle
|
||||
test_mul_mat_id(backend, 512, 65, 4, 2, 1, 128, GGML_TYPE_Q4_K); // ceil-div boundary, 1-row tail group
|
||||
test_mul_mat_id(backend, 512, 63, 4, 2, 1, 128, GGML_TYPE_Q4_K); // ragged group tail
|
||||
test_mul_mat_id(backend, 512, 1, 4, 1, 1, 256, GGML_TYPE_Q4_K); // R = 1 row, cne1 = 256
|
||||
|
||||
// strided (non-contiguous) F32 src1: nb[1] = 4 bytes, the tiled path reads through strides
|
||||
test_mul_mat_id(backend, 512, 256, 8, 8, 1, 128, GGML_TYPE_Q4_K, true); // cne1 = 128
|
||||
|
||||
// narrow path (cne1 in [8,16], K a multiple of k_extent): the weight is read as long per-row K streams
|
||||
test_mul_mat_id(backend, 1024, 256, 2, 2, 1, 16, GGML_TYPE_Q4_K); // cne1 = 16
|
||||
test_mul_mat_id(backend, 1024, 256, 2, 1, 1, 16, GGML_TYPE_Q4_K); // cne1 = 8
|
||||
test_mul_mat_id(backend, 512, 128, 2, 3, 1, 8, GGML_TYPE_Q5_K); // cne1 = 12
|
||||
|
||||
// higher-dim (ne[2], ne[3] > 1)
|
||||
// equal-batch and broadcast shapes (src1_2=2*src0_2, src1_3=2*src0_3)
|
||||
test_matmul_highdim(backend, 1024, 1024, 1024, 2, 1, 2, 1, GGML_TYPE_Q4_K); // 3D, tiled
|
||||
test_matmul_highdim(backend, 1024, 1024, 1024, 1, 2, 1, 2, GGML_TYPE_Q4_K); // 3D, tiled
|
||||
test_matmul_highdim(backend, 512, 1024, 512, 2, 2, 2, 2, GGML_TYPE_Q4_K); // 4D, tiled
|
||||
test_matmul_highdim(backend, 1024, 1024, 1024, 2, 1, 1, 1, GGML_TYPE_Q4_K); // broadcast r2=2 (src1_2>src0_2)
|
||||
test_matmul_highdim(backend, 1024, 1024, 1024, 2, 2, 2, 1, GGML_TYPE_Q4_K); // broadcast r3=2 (src1_3>src0_3)
|
||||
test_matmul_highdim(backend, 1024, 1024, 1024, 2, 1, 2, 1, GGML_TYPE_Q6_K); // 3D, tiled, q6_K
|
||||
test_matmul_highdim(backend, 1024, 1024, 1024, 2, 2, 2, 2, GGML_TYPE_Q3_K); // 4D, tiled, q3_K
|
||||
test_matmul_highdim(backend, 1024, 1024, 1024, 2, 2, 2, 2, GGML_TYPE_Q2_K); // 4D, tiled, q2_K
|
||||
}
|
||||
|
||||
|
||||
// benchmark: std (use_ref) vs repack (CPU_REPACK buffer) vs tiled kernel, over all tiled
|
||||
// types. std is timed with use_ref=true (bypasses the tiled gate); tiled and repack with
|
||||
// use_ref=false. Errors are vs the vec-only reference (use_ref).
|
||||
if (run_bench) {
|
||||
printf("Starting benchmark runs.\n");
|
||||
const ggml_type bench_types[] = { GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K,
|
||||
GGML_TYPE_IQ4_XS, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S,
|
||||
GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M };
|
||||
const size_t n_types = sizeof(bench_types) / sizeof(bench_types[0]);
|
||||
|
||||
struct { int64_t M, N, K; } shapes[] = {
|
||||
{ 4096, 4096, 4096 },
|
||||
{ 4096, 4096, 64 },
|
||||
{ 4096, 4096, 32 },
|
||||
{ 4096, 4096, 24 },
|
||||
{ 4096, 4096, 16 },
|
||||
{ 4096, 4096, 8 },
|
||||
{ 4096, 4096, 1 },
|
||||
};
|
||||
for (size_t s = 0; s < sizeof(shapes) / sizeof(shapes[0]); ++s) {
|
||||
bench_row rows[n_types];
|
||||
for (size_t i = 0; i < n_types; ++i) {
|
||||
rows[i] = bench_three_way(backend, shapes[s].M, shapes[s].N, shapes[s].K, bench_types[i]);
|
||||
}
|
||||
print_bench_table(shapes[s].M, shapes[s].N, shapes[s].K, rows, n_types);
|
||||
}
|
||||
|
||||
// MUL_MAT_ID (MoE): K must be a multiple of 256 (the tiled slab). cne1 = k*batch/E.
|
||||
struct { int64_t K, R, E, k, b_slots, batch; } mmid_shapes[] = {
|
||||
{ 1024, 1024, 8, 8, 1, 1 }, // cne1 = 1, single-token decode
|
||||
{ 1024, 1024, 4, 2, 1, 16 }, // cne1 = 8, narrow path
|
||||
{ 1024, 1024, 4, 2, 1, 32 }, // cne1 = 16, narrow path
|
||||
{ 512, 512, 8, 2, 1, 128 }, // cne1 = 32
|
||||
{ 1024, 1024, 16, 8, 1, 64 }, // cne1 = 32
|
||||
{ 1024, 1024, 16, 8, 1, 256 }, // cne1 = 128
|
||||
{ 1024, 2048, 32, 8, 1, 512 }, // cne1 = 128, wide experts
|
||||
{ 2048, 1024, 16, 8, 1, 1024 }, // cne1 = 512, long dot
|
||||
};
|
||||
for (size_t s = 0; s < sizeof(mmid_shapes) / sizeof(mmid_shapes[0]); ++s) {
|
||||
bench_row_mmid rows[n_types];
|
||||
for (size_t i = 0; i < n_types; ++i) {
|
||||
rows[i] = bench_mul_mat_id(backend, mmid_shapes[s].K, mmid_shapes[s].R, mmid_shapes[s].E,
|
||||
mmid_shapes[s].k, mmid_shapes[s].b_slots, mmid_shapes[s].batch, bench_types[i]);
|
||||
}
|
||||
print_mmid_table(mmid_shapes[s].K, mmid_shapes[s].R, mmid_shapes[s].E, mmid_shapes[s].k,
|
||||
mmid_shapes[s].b_slots, mmid_shapes[s].batch, rows, n_types);
|
||||
}
|
||||
}
|
||||
|
||||
ggml_backend_free(backend);
|
||||
return n_failed ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
#include "llama.h"
|
||||
#include "common.h"
|
||||
#include "console.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
#include <fstream>
|
||||
#include <thread>
|
||||
|
||||
//static const std::map<std::string, std::vector<llama_token>> & k_tests() {
|
||||
// static std::map<std::string, std::vector<llama_token>> _k_tests = {
|
||||
// { "" , { }, },
|
||||
// { " " , { 220, }, },
|
||||
// { " " , { 256, }, },
|
||||
// { " " , { 262, }, },
|
||||
// { "\t" , { 197, }, },
|
||||
// { "\n" , { 198, }, },
|
||||
// { "\n\n" , { 271, }, },
|
||||
// { "\n\n\n" , { 1432, }, },
|
||||
// { "\t\n" , { 1602, }, },
|
||||
// { "Hello world" , { 9906, 1917, }, },
|
||||
// { " Hello world" , { 22691, 1917, }, },
|
||||
// { "Hello World" , { 9906, 4435, }, },
|
||||
// { " Hello World" , { 22691, 4435, }, },
|
||||
// { " Hello World!" , { 22691, 4435, 0, }, },
|
||||
// { "Hello, world!" , { 9906, 11, 1917, 0, }, },
|
||||
// { " Hello, world!" , { 22691, 11, 1917, 0, }, },
|
||||
// { " this is 🦙.cpp" , { 420, 374, 11410, 99, 247, 13, 11055, }, },
|
||||
// { "w048 7tuijk dsdfhu" , { 86, 23904, 220, 22, 83, 2005, 42908, 11729, 3013, 17156, }, },
|
||||
// { "нещо на Български" , { 79862, 102118, 13373, 64571, 34694, 3114, 112203, 80112, }, },
|
||||
// { "កាន់តែពិសេសអាចខលចេញ" , { 21549, 222, 98629, 241, 45358, 233, 21549, 237, 45358, 224, 21549, 244, 21549, 115, 21549, 253, 45358, 223, 21549, 253, 21549, 95, 98629, 227, 21549, 223, 21549, 249, 21549, 227, 45358, 223, 21549, 231, }, },
|
||||
// { "🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)", { 9468, 248, 222, 320, 8416, 8, 27623, 114, 102470, 9468, 234, 104, 31643, 320, 36773, 100166, 98634, 8, 26602, 227, 320, 3323, 43465, 430, 706, 1202, 1866, 4037, 8, }, },
|
||||
// { "Hello" , { 9906, }, },
|
||||
// { " Hello" , { 22691, }, },
|
||||
// { " Hello" , { 220, 22691, }, },
|
||||
// { " Hello" , { 256, 22691, }, },
|
||||
// { " Hello" , { 262, 22691, }, },
|
||||
// { " Hello\n Hello" , { 262, 22691, 198, 262, 22691, }, },
|
||||
// { " (" , { 320, }, },
|
||||
// { "\n =" , { 198, 284, }, },
|
||||
// { "' era" , { 6, 11639, }, },
|
||||
// { "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～", { 9906, 11, 379, 65948, 0, 2650, 527, 499, 27623, 223, 949, 37046, 101067, 19000, 23182, 102301, 9263, 18136, 16, 36827, 21909, }, },
|
||||
// { "3" , { 18, }, },
|
||||
// { "33" , { 1644, }, },
|
||||
// { "333" , { 8765, }, },
|
||||
// { "3333" , { 8765, 18, }, },
|
||||
// { "33333" , { 8765, 1644, }, },
|
||||
// { "333333" , { 8765, 8765, }, },
|
||||
// { "3333333" , { 8765, 8765, 18, }, },
|
||||
// { "33333333" , { 8765, 8765, 1644, }, },
|
||||
// { "333333333" , { 8765, 8765, 8765, }, },
|
||||
// };
|
||||
//
|
||||
// return _k_tests;
|
||||
//}
|
||||
|
||||
using llama_tests = std::map<std::string, std::vector<llama_token>>;
|
||||
|
||||
static llama_tests read_tests(const std::string & fname_inp, const std::string & fname_out) {
|
||||
llama_tests tests;
|
||||
|
||||
std::ifstream ifs_inp(fname_inp);
|
||||
if (!ifs_inp) {
|
||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_inp.c_str());
|
||||
return tests;
|
||||
}
|
||||
|
||||
std::string sraw((std::istreambuf_iterator<char>(ifs_inp)), std::istreambuf_iterator<char>());
|
||||
|
||||
std::ifstream ifs_out(fname_out);
|
||||
if (!ifs_out) {
|
||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
|
||||
return tests;
|
||||
}
|
||||
|
||||
std::vector<std::string> sout;
|
||||
for (std::string line; std::getline(ifs_out, line);) {
|
||||
sout.push_back(line);
|
||||
}
|
||||
|
||||
const std::string sep = "\n__ggml_vocab_test__\n";
|
||||
|
||||
std::vector<std::string> sinp;
|
||||
|
||||
size_t pos = 0;
|
||||
while (pos < sraw.size()) {
|
||||
const size_t next = sraw.find(sep, pos);
|
||||
if (next == std::string::npos) {
|
||||
sinp.push_back(sraw.substr(pos));
|
||||
break;
|
||||
}
|
||||
sinp.push_back(sraw.substr(pos, next - pos));
|
||||
pos = next + sep.size();
|
||||
}
|
||||
|
||||
if (sinp.size() != sout.size()) {
|
||||
fprintf(stderr, "%s : error: input and output files have different number of tests\n", __func__);
|
||||
return tests;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < sinp.size(); ++i) {
|
||||
const std::string & s = sinp[i];
|
||||
const std::string & o = string_strip(sout[i]);
|
||||
|
||||
std::vector<llama_token> toks;
|
||||
|
||||
size_t pos = 0;
|
||||
while (pos < o.size()) {
|
||||
size_t next = o.find(' ', pos);
|
||||
if (next == std::string::npos) {
|
||||
next = o.size();
|
||||
}
|
||||
const std::string stok = o.substr(pos, next - pos);
|
||||
toks.push_back(std::stoi(stok));
|
||||
pos = next + 1;
|
||||
}
|
||||
|
||||
tests[s] = toks;
|
||||
}
|
||||
|
||||
return tests;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "Usage: %s vocab-file [text-file] [--check-decode]\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const std::string fname = argv[1];
|
||||
|
||||
const std::string fname_inp = fname + ".inp";
|
||||
const std::string fname_out = fname + ".out";
|
||||
|
||||
std::string fname_text;
|
||||
bool check_decode = false; // also check that the expected tokens detokenize to the input text
|
||||
for (int i = 2; i < argc; i++) {
|
||||
if (std::string(argv[i]) == "--check-decode") {
|
||||
check_decode = true;
|
||||
} else {
|
||||
fname_text = argv[i];
|
||||
}
|
||||
}
|
||||
|
||||
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
|
||||
|
||||
llama_model * model;
|
||||
llama_context * ctx;
|
||||
|
||||
llama_backend_init();
|
||||
|
||||
// load the vocab
|
||||
{
|
||||
auto mparams = llama_model_default_params();
|
||||
|
||||
mparams.vocab_only = true;
|
||||
|
||||
model = llama_model_load_from_file(fname.c_str(), mparams);
|
||||
|
||||
if (model == NULL) {
|
||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
auto cparams = llama_context_default_params();
|
||||
|
||||
ctx = llama_init_from_model(model, cparams);
|
||||
|
||||
if (ctx == NULL) {
|
||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
||||
llama_model_free(model);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
// We need this for unicode console support
|
||||
console::init(false, false);
|
||||
atexit([]() { console::cleanup(); });
|
||||
#endif
|
||||
|
||||
bool success = true;
|
||||
|
||||
const auto k_tests = [&]() -> llama_tests {
|
||||
if (!fname_text.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const auto res = read_tests(fname_inp, fname_out);
|
||||
|
||||
if (res.empty()) {
|
||||
fprintf(stderr, "%s : error: no tests found\n", __func__);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
return res;
|
||||
}();
|
||||
|
||||
const bool add_special = false;
|
||||
|
||||
// multi-threaded tokenization
|
||||
const int nthread = std::thread::hardware_concurrency();
|
||||
std::vector<std::thread> threads(nthread);
|
||||
|
||||
for (int i = 0; i < nthread; i++) {
|
||||
threads[i] = std::thread([&, i]() {
|
||||
for (const auto & test_kv : k_tests) {
|
||||
const std::vector<llama_token> res = common_tokenize(ctx, test_kv.first, add_special, false);
|
||||
|
||||
// here only print the result of the first thread
|
||||
// because the other threads are running the same tests
|
||||
if (i != 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
printf("\n");
|
||||
printf("src: '%s'\n", test_kv.first.c_str());
|
||||
printf("res: '%s'\n", common_detokenize(ctx, res).c_str());
|
||||
printf("tok: ");
|
||||
for (const auto & tok : res) {
|
||||
printf("%d ", tok);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
bool correct = res.size() == test_kv.second.size();
|
||||
for (int i = 0; i < (int) res.size() && correct; ++i) {
|
||||
if (test_kv.second[i] != res[i]) {
|
||||
correct = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!correct) {
|
||||
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str());
|
||||
fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__,
|
||||
common_detokenize(ctx, res).c_str(),
|
||||
common_detokenize(ctx, test_kv.second).c_str());
|
||||
fprintf(stderr, "%s : expected tokens: ", __func__);
|
||||
for (const auto & t : test_kv.second) {
|
||||
fprintf(stderr, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str());
|
||||
}
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "%s : got tokens: ", __func__);
|
||||
for (const auto & t : res) {
|
||||
fprintf(stderr, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str());
|
||||
}
|
||||
fprintf(stderr, "\n");
|
||||
|
||||
success = false;
|
||||
}
|
||||
|
||||
if (check_decode) {
|
||||
const std::string dec = common_detokenize(ctx, test_kv.second);
|
||||
if (dec != test_kv.first) {
|
||||
fprintf(stderr, "%s : failed decode: '%s' detokenized to '%s'\n", __func__, test_kv.first.c_str(), dec.c_str());
|
||||
success = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (int i = 0; i < nthread; i++) {
|
||||
threads[i].join();
|
||||
}
|
||||
|
||||
// single threaded tokenization
|
||||
if (!fname_text.empty()) {
|
||||
fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str());
|
||||
|
||||
std::string text;
|
||||
{
|
||||
std::ifstream ifs(fname_text);
|
||||
if (!ifs) {
|
||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str());
|
||||
return 1;
|
||||
}
|
||||
text = std::string(std::istreambuf_iterator<char>(ifs), std::istreambuf_iterator<char>());
|
||||
}
|
||||
|
||||
fprintf(stderr, "%s : text size: %zu\n", __func__, text.size());
|
||||
|
||||
std::vector<llama_token> res;
|
||||
|
||||
{
|
||||
const auto t_start = ggml_time_us();
|
||||
|
||||
res = common_tokenize(ctx, text, add_special, false);
|
||||
|
||||
const auto t_end = ggml_time_us();
|
||||
|
||||
fprintf(stderr, "%s : tokenized in %.3f ms (cpp)\n", __func__, (t_end - t_start) / 1000.0);
|
||||
}
|
||||
|
||||
fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());
|
||||
|
||||
{
|
||||
const std::string fname_out = fname_text + ".tokcpp";
|
||||
|
||||
std::ofstream ofs(fname_out);
|
||||
if (!ofs) {
|
||||
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
for (const auto & tok : res) {
|
||||
//ofs << tok << " '" << string_strip(llama_detokenize(ctx, std::vector<int>{tok})) << "'" << std::endl;
|
||||
ofs << tok << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str());
|
||||
}
|
||||
|
||||
llama_free(ctx);
|
||||
llama_model_free(model);
|
||||
|
||||
llama_backend_free();
|
||||
|
||||
printf("\n");
|
||||
printf("Tests %s\n", success ? "passed" : "failed");
|
||||
|
||||
return success ? 0 : 3;
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import time
|
||||
import argparse
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("dir_tokenizer", help="directory containing 'tokenizer.model' file")
|
||||
parser.add_argument("--fname-tok", help="path to a text file to tokenize", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
dir_tokenizer = args.dir_tokenizer
|
||||
fname_tok = args.fname_tok
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(dir_tokenizer)
|
||||
|
||||
print('tokenizing file: ', fname_tok) # noqa: NP100
|
||||
fname_out = fname_tok + '.tok'
|
||||
with open(fname_tok, 'r', encoding='utf-8') as f:
|
||||
lines = f.readlines()
|
||||
s = ''.join(lines)
|
||||
t_start = time.time()
|
||||
res = tokenizer.encode(s, add_special_tokens=False) # ty: ignore[unresolved-attribute]
|
||||
t_end = time.time()
|
||||
print('\nmain : tokenized in', "{:.3f}".format(1000.0 * (t_end - t_start)), 'ms (py)') # noqa: NP100
|
||||
with open(fname_out, 'w', encoding='utf-8') as f:
|
||||
for x in res:
|
||||
# LLaMA v3 for some reason strips the space for these tokens (and others)
|
||||
# if x == 662:
|
||||
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
||||
# elif x == 1174:
|
||||
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
||||
# elif x == 2564:
|
||||
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
||||
# elif x == 758:
|
||||
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
||||
# elif x == 949:
|
||||
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
||||
# elif x == 5354:
|
||||
# f.write(str(x) + ' \' ' + tokenizer.decode(x) + '\'\n')
|
||||
# else:
|
||||
# f.write(str(x) + ' \'' + tokenizer.decode(x) + '\'\n')
|
||||
# f.write(str(x) + ' \'' + tokenizer.decode(x).strip() + '\'\n')
|
||||
f.write(str(x) + '\n')
|
||||
print('len(res): ', len(res)) # noqa: NP100
|
||||
print('len(lines): ', len(lines)) # noqa: NP100
|
||||
print('results written to: ', fname_out) # noqa: NP100
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Usage:
|
||||
#
|
||||
# test-tokenizer-0.sh <name> <input>
|
||||
#
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
printf "Usage: $0 <name> <input>\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
name=$1
|
||||
input=$2
|
||||
|
||||
# Build using CMake if binary doesn't exist
|
||||
if [ ! -f ./build/bin/test-tokenizer-0 ]; then
|
||||
printf "Building test-tokenizer-0 with CMake...\n"
|
||||
cmake -B build -DLLAMA_BUILD_TESTS=ON
|
||||
cmake --build build --target test-tokenizer-0 -j
|
||||
fi
|
||||
|
||||
printf "Testing %s on %s ...\n" $name $input
|
||||
|
||||
set -e
|
||||
|
||||
printf "Tokenizing using (py) Python AutoTokenizer ...\n"
|
||||
python3 ./tests/test-tokenizer-0.py ./models/tokenizers/$name --fname-tok $input > /tmp/test-tokenizer-0-$name-py.log 2>&1
|
||||
|
||||
printf "Tokenizing using (cpp) llama.cpp ...\n"
|
||||
./build/bin/test-tokenizer-0 ./models/ggml-vocab-$name.gguf $input > /tmp/test-tokenizer-0-$name-cpp.log 2>&1
|
||||
|
||||
cat /tmp/test-tokenizer-0-$name-py.log | grep "tokenized in"
|
||||
cat /tmp/test-tokenizer-0-$name-cpp.log | grep "tokenized in"
|
||||
|
||||
set +e
|
||||
|
||||
diff $input.tok $input.tokcpp > /dev/null 2>&1
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
printf "Tokenization is correct!\n"
|
||||
else
|
||||
diff $input.tok $input.tokcpp | head -n 32
|
||||
|
||||
printf "Tokenization differs!\n"
|
||||
fi
|
||||
@@ -0,0 +1,155 @@
|
||||
#include "llama.h"
|
||||
#include "common.h"
|
||||
#include "console.h"
|
||||
|
||||
#include "../src/unicode.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <codecvt>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <locale>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <atomic>
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 2 || argc > 3) {
|
||||
fprintf(stderr, "Usage: %s <vocab-file> [--ignore-merges]\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const std::string fname = argv[1];
|
||||
bool ignore_merges = false;
|
||||
if (argc == 3) {
|
||||
if (std::strcmp(argv[2], "--ignore-merges") != 0) {
|
||||
fprintf(stderr, "Usage: %s <vocab-file> [--ignore-merges]\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
ignore_merges = true;
|
||||
}
|
||||
|
||||
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
|
||||
|
||||
if (ignore_merges) {
|
||||
fprintf(stderr, "%s : ignoring merges for tokens inside vocab\n", __func__);
|
||||
}
|
||||
|
||||
llama_model * model;
|
||||
llama_context * ctx;
|
||||
|
||||
llama_backend_init();
|
||||
|
||||
// load the vocab
|
||||
{
|
||||
auto mparams = llama_model_default_params();
|
||||
|
||||
mparams.vocab_only = true;
|
||||
|
||||
model = llama_model_load_from_file(fname.c_str(), mparams);
|
||||
|
||||
if (model == NULL) {
|
||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
auto cparams = llama_context_default_params();
|
||||
|
||||
ctx = llama_init_from_model(model, cparams);
|
||||
|
||||
if (ctx == NULL) {
|
||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
||||
llama_model_free(model);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
const llama_vocab * vocab = llama_model_get_vocab(model);
|
||||
|
||||
//GGML_ASSERT(llama_vocab_type(vocab) == LLAMA_VOCAB_TYPE_BPE);
|
||||
if (llama_vocab_type(vocab) != LLAMA_VOCAB_TYPE_BPE) {
|
||||
return 99;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
// We need this for unicode console support
|
||||
console::init(false, false);
|
||||
atexit([]() { console::cleanup(); });
|
||||
#endif
|
||||
|
||||
const int n_vocab = llama_vocab_n_tokens(vocab);
|
||||
|
||||
for (int i = 0; i < n_vocab; ++i) {
|
||||
std::string str = common_detokenize(ctx, std::vector<int>(1, i));
|
||||
try {
|
||||
auto cps = unicode_cpts_from_utf8(str);
|
||||
std::vector<llama_token> tokens = common_tokenize(ctx, str, false, true);
|
||||
if (ignore_merges && tokens.size() > 1) {
|
||||
fprintf(stderr,
|
||||
"%s : error: token %d detokenizes to '%s'(%zu) but "
|
||||
"tokenization of this to multiple tokens: [",
|
||||
__func__, i, str.c_str(), str.length());
|
||||
fprintf(stderr, "%d", tokens[0]);
|
||||
for (size_t i = 1; i < tokens.size(); i++) {
|
||||
fprintf(stderr, ", %d", tokens[i]);
|
||||
}
|
||||
fprintf(stderr, "]\n");
|
||||
return 2;
|
||||
}
|
||||
std::string check = common_detokenize(ctx, tokens);
|
||||
if (check != str) {
|
||||
fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
|
||||
__func__, i, str.c_str(), str.length(), check.c_str(), check.length());
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
catch (const std::invalid_argument &) {
|
||||
//fprintf(stderr, "%s : info: utf8 conversion %d '%s'\n", __func__, i, str.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
// unicode
|
||||
{
|
||||
const int nthread = std::thread::hardware_concurrency();
|
||||
|
||||
std::vector<std::thread> threads(nthread);
|
||||
|
||||
std::atomic_int errcode = {};
|
||||
|
||||
for (int i = 0; i < nthread; ++i) {
|
||||
threads[i] = std::thread([i, nthread, ctx, &errcode]() {
|
||||
for (uint32_t cp = i; !errcode && cp < 0x00110000; cp += nthread) {
|
||||
if ((0x0000D800 <= cp && cp <= 0x0000DFFF) || // surrogates \p{Cs}
|
||||
(0x00040000 <= cp && cp <= 0x000E0000)) { // undefined \p{Cn}
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string str = unicode_cpt_to_utf8(cp);
|
||||
std::vector<llama_token> tokens = common_tokenize(ctx, str, false);
|
||||
std::string check = common_detokenize(ctx, tokens);
|
||||
if (cp != 9601 && str != check) {
|
||||
fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
|
||||
cp, check.c_str(), check.length(), str.c_str(), str.length());
|
||||
errcode = 3;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (auto & t : threads) {
|
||||
t.join();
|
||||
}
|
||||
|
||||
if (errcode) {
|
||||
return errcode;
|
||||
}
|
||||
}
|
||||
|
||||
llama_free(ctx);
|
||||
llama_model_free(model);
|
||||
|
||||
llama_backend_free();
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
#include "llama.h"
|
||||
#include "common.h"
|
||||
#include "console.h"
|
||||
|
||||
#include "../src/unicode.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <codecvt>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <locale>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <atomic>
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "Usage: %s <vocab-file>\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const std::string fname = argv[1];
|
||||
|
||||
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
|
||||
|
||||
llama_model * model;
|
||||
llama_context * ctx;
|
||||
|
||||
llama_backend_init();
|
||||
|
||||
// load the vocab
|
||||
{
|
||||
auto mparams = llama_model_default_params();
|
||||
|
||||
mparams.vocab_only = true;
|
||||
|
||||
model = llama_model_load_from_file(fname.c_str(), mparams);
|
||||
|
||||
if (model == NULL) {
|
||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
auto cparams = llama_context_default_params();
|
||||
|
||||
ctx = llama_init_from_model(model, cparams);
|
||||
|
||||
if (ctx == NULL) {
|
||||
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
|
||||
llama_model_free(model);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
const llama_vocab * vocab = llama_model_get_vocab(model);
|
||||
|
||||
//GGML_ASSERT(llama_vocab_type(model) == LLAMA_VOCAB_TYPE_SPM);
|
||||
if (llama_vocab_type(vocab) != LLAMA_VOCAB_TYPE_SPM) {
|
||||
return 99;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
// We need this for unicode console support
|
||||
console::init(false, false);
|
||||
atexit([]() { console::cleanup(); });
|
||||
#endif
|
||||
|
||||
const int n_vocab = llama_vocab_n_tokens(vocab);
|
||||
|
||||
for (int i = 0; i < n_vocab; ++i) {
|
||||
std::string str = common_detokenize(ctx, std::vector<int>(1, i), true);
|
||||
std::vector<llama_token> tokens = common_tokenize(ctx, str, false, true);
|
||||
std::string check = common_detokenize(ctx, tokens);
|
||||
if (check != str) {
|
||||
fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
|
||||
__func__, i, str.c_str(), str.length(), check.c_str(), check.length());
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
// unicode
|
||||
{
|
||||
const int nthread = std::thread::hardware_concurrency();
|
||||
|
||||
std::vector<std::thread> threads(nthread);
|
||||
|
||||
std::atomic_int errcode = {};
|
||||
|
||||
for (int i = 0; i < nthread; ++i) {
|
||||
threads[i] = std::thread([i, nthread, ctx, &errcode]() {
|
||||
for (uint32_t cp = i; !errcode && cp < 0x00110000; cp += nthread) {
|
||||
if ((0x0000D800 <= cp && cp <= 0x0000DFFF) || // surrogates \p{Cs}
|
||||
(0x00040000 <= cp && cp <= 0x000E0000)) { // undefined \p{Cn}
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string str = unicode_cpt_to_utf8(cp);
|
||||
std::vector<llama_token> tokens = common_tokenize(ctx, str, false, true);
|
||||
std::string check = common_detokenize(ctx, tokens);
|
||||
if (cp != 9601 && str != check) {
|
||||
fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
|
||||
cp, check.c_str(), check.length(), str.c_str(), str.length());
|
||||
errcode = 3;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (auto & t : threads) {
|
||||
t.join();
|
||||
}
|
||||
|
||||
if(errcode) {
|
||||
return errcode;
|
||||
}
|
||||
}
|
||||
|
||||
llama_free(ctx);
|
||||
llama_model_free(model);
|
||||
|
||||
llama_backend_free();
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,565 @@
|
||||
# Test libllama tokenizer == AutoTokenizer.
|
||||
# Brute force random words/text generation.
|
||||
#
|
||||
# Sample usage:
|
||||
#
|
||||
# python3 tests/test-tokenizer-random.py ./models/ggml-vocab-llama-bpe.gguf ./models/tokenizers/llama-bpe
|
||||
#
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import logging
|
||||
import argparse
|
||||
import subprocess
|
||||
import random
|
||||
import unicodedata
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
import cffi
|
||||
from transformers import AutoTokenizer, PreTrainedTokenizer
|
||||
|
||||
|
||||
logger = logging.getLogger("test-tokenizer-random")
|
||||
|
||||
|
||||
class LibLlama:
|
||||
|
||||
DEFAULT_PATH_LLAMA_H = "./include/llama.h"
|
||||
DEFAULT_PATH_INCLUDES = ["./ggml/include/", "./include/"]
|
||||
DEFAULT_PATH_LIBLLAMA = "./build/src/libllama.so" # CMakeLists.txt: BUILD_SHARED_LIBS ON
|
||||
|
||||
def __init__(self, path_llama_h: str | None = None, path_includes: list[str] = [], path_libllama: str | None = None):
|
||||
path_llama_h = path_llama_h or self.DEFAULT_PATH_LLAMA_H
|
||||
path_includes = path_includes or self.DEFAULT_PATH_INCLUDES
|
||||
path_libllama = path_libllama or self.DEFAULT_PATH_LIBLLAMA
|
||||
(self.ffi, self.lib) = self._load_libllama_cffi(path_llama_h, path_includes, path_libllama)
|
||||
self.lib.llama_backend_init()
|
||||
|
||||
def _load_libllama_cffi(self, path_llama_h: str, path_includes: list[str], path_libllama: str) -> tuple[cffi.FFI, Any]:
|
||||
cmd = ["gcc", "-O0", "-E", "-P", "-D__restrict=", "-D__attribute__(x)=", "-D__asm__(x)="]
|
||||
cmd += ["-I" + path for path in path_includes] + [path_llama_h]
|
||||
res = subprocess.run(cmd, stdout=subprocess.PIPE)
|
||||
assert (res.returncode == 0)
|
||||
source = res.stdout.decode()
|
||||
ffi = cffi.FFI()
|
||||
if True: # workarounds for pycparser
|
||||
source = "typedef struct { } __builtin_va_list;" + "\n" + source
|
||||
source = source.replace("sizeof (int)", str(ffi.sizeof("int")))
|
||||
source = source.replace("sizeof (void *)", str(ffi.sizeof("void*")))
|
||||
source = source.replace("sizeof (size_t)", str(ffi.sizeof("size_t")))
|
||||
source = source.replace("sizeof(int32_t)", str(ffi.sizeof("int32_t")))
|
||||
ffi.cdef(source, override=True)
|
||||
lib = ffi.dlopen(path_libllama)
|
||||
return (ffi, lib)
|
||||
|
||||
def model_default_params(self, **kwargs):
|
||||
mparams = self.lib.llama_model_default_params()
|
||||
for k, v in kwargs.items():
|
||||
setattr(mparams, k, v)
|
||||
return mparams
|
||||
|
||||
def context_default_params(self, **kwargs):
|
||||
cparams = self.lib.llama_context_default_params()
|
||||
for k, v in kwargs.items():
|
||||
setattr(cparams, k, v)
|
||||
return cparams
|
||||
|
||||
|
||||
class LibLlamaModel:
|
||||
|
||||
def __init__(self, libllama: LibLlama, path_model: str, mparams={}, cparams={}):
|
||||
self.lib: Any = libllama.lib
|
||||
self.ffi = libllama.ffi
|
||||
if isinstance(mparams, dict):
|
||||
mparams = libllama.model_default_params(**mparams)
|
||||
self.model = self.lib.llama_model_load_from_file(path_model.encode(), mparams)
|
||||
if not self.model:
|
||||
raise RuntimeError("error: failed to load model '%s'" % path_model)
|
||||
if isinstance(cparams, dict):
|
||||
cparams = libllama.context_default_params(**cparams)
|
||||
self.ctx = self.lib.llama_new_context_with_model(self.model, cparams)
|
||||
if not self.ctx:
|
||||
raise RuntimeError("error: failed to create context for model '%s'" % path_model)
|
||||
n_tokens_max = self.lib.llama_n_ctx(self.ctx)
|
||||
self.token_ids = self.ffi.new("llama_token[]", n_tokens_max)
|
||||
self.text_buff = self.ffi.new("uint8_t[]", 1024)
|
||||
|
||||
def free(self):
|
||||
if self.ctx:
|
||||
self.lib.llama_free(self.ctx)
|
||||
if self.model:
|
||||
self.lib.llama_model_free(self.model)
|
||||
self.ctx = None
|
||||
self.model = None
|
||||
self.lib = None
|
||||
|
||||
def tokenize(self, text: str, add_special: bool = False, parse_special: bool = False) -> list[int]:
|
||||
encoded_text: bytes = text.encode("utf-8")
|
||||
num = self.lib.llama_tokenize(self.model, encoded_text, len(encoded_text), self.token_ids, len(self.token_ids), add_special, parse_special)
|
||||
while num < 0 and len(self.token_ids) < (16 << 20):
|
||||
self.token_ids = self.ffi.new("llama_token[]", -2 * num)
|
||||
num = self.lib.llama_tokenize(self.model, encoded_text, len(encoded_text), self.token_ids, len(self.token_ids), add_special, parse_special)
|
||||
return list(self.token_ids[0:num])
|
||||
|
||||
def detokenize(self, ids: list[int], remove_special: bool = False, unparse_special: bool = False) -> str:
|
||||
if len(self.token_ids) < len(ids):
|
||||
self.token_ids = self.ffi.new("llama_token[]", 2 * len(ids))
|
||||
for i, id in enumerate(ids):
|
||||
self.token_ids[i] = id
|
||||
num = self.lib.llama_detokenize(self.model, self.token_ids, len(ids), self.text_buff, len(self.text_buff), remove_special, unparse_special)
|
||||
while num < 0 and len(self.text_buff) < (16 << 20):
|
||||
self.text_buff = self.ffi.new("uint8_t[]", -2 * num)
|
||||
num = self.lib.llama_detokenize(self.model, self.token_ids, len(ids), self.text_buff, len(self.text_buff), remove_special, unparse_special)
|
||||
return str(self.ffi.buffer(self.text_buff, num), encoding="utf-8", errors="replace") # replace errors with '\uFFFD' # pyright: ignore[reportArgumentType]
|
||||
|
||||
|
||||
class Tokenizer:
|
||||
|
||||
def encode(self, text: str) -> list[int]:
|
||||
raise NotImplementedError
|
||||
|
||||
def decode(self, ids: list[int]) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class TokenizerGroundtruth (Tokenizer):
|
||||
|
||||
def __init__(self, dir_tokenizer: str):
|
||||
self.model: PreTrainedTokenizer = AutoTokenizer.from_pretrained(dir_tokenizer) # ty: ignore[invalid-assignment]
|
||||
# guess BOS and EOS
|
||||
ids = self.encode("a")
|
||||
assert 1 <= len(ids) <= 3
|
||||
add_bos_token = len(ids) > 1 and self.model.bos_token_id == ids[0]
|
||||
add_eos_token = len(ids) > 1 and self.model.eos_token_id == ids[-1]
|
||||
self.add_bos_token = getattr(self.model, "add_bos_token", add_bos_token)
|
||||
self.add_eos_token = getattr(self.model, "add_eos_token", add_eos_token)
|
||||
# build vocab
|
||||
tokens = list(self.model.get_vocab().values())
|
||||
self.vocab = self.model.batch_decode(tokens, skip_special_tokens=True)
|
||||
self.vocab = list(sorted(self.vocab))
|
||||
# tokens and lists
|
||||
self.special_tokens = list(self.model.all_special_tokens)
|
||||
self.added_tokens = self.model.batch_decode(list(self.model.added_tokens_encoder.values()), skip_special_tokens=False)
|
||||
self.bos_token = self.model.bos_token
|
||||
self.eos_token = self.model.eos_token
|
||||
|
||||
def encode(self, text: str) -> list[int]:
|
||||
return self.model.encode(text, add_special_tokens=True)
|
||||
|
||||
def decode(self, ids: list[int]) -> str:
|
||||
return self.model.decode(ids, skip_special_tokens=False) # ty: ignore[invalid-return-type]
|
||||
|
||||
|
||||
class TokenizerLlamaCpp (Tokenizer):
|
||||
|
||||
libllama: LibLlama | None = None
|
||||
|
||||
def __init__(self, vocab_file: str):
|
||||
if not self.libllama:
|
||||
self.libllama = LibLlama()
|
||||
self.model = LibLlamaModel(self.libllama, vocab_file, mparams=dict(vocab_only=True), cparams=dict(n_ctx=4096))
|
||||
|
||||
def encode(self, text: str) -> list[int]:
|
||||
return self.model.tokenize(text, add_special=True, parse_special=True)
|
||||
|
||||
def decode(self, ids: list[int]) -> str:
|
||||
return self.model.detokenize(ids, remove_special=False, unparse_special=True)
|
||||
|
||||
|
||||
def generator_custom_text() -> Iterator[str]:
|
||||
"""General tests"""
|
||||
yield from [
|
||||
"",
|
||||
" ",
|
||||
" ",
|
||||
" ",
|
||||
"\t",
|
||||
"\n",
|
||||
"\n\n",
|
||||
"\n\n\n",
|
||||
"\t\n",
|
||||
"Hello world",
|
||||
" Hello world",
|
||||
"Hello World",
|
||||
" Hello World",
|
||||
" Hello World!",
|
||||
"Hello, world!",
|
||||
" Hello, world!",
|
||||
" this is 🦙.cpp",
|
||||
"w048 7tuijk dsdfhu",
|
||||
"нещо на Български",
|
||||
"កាន់តែពិសេសអាចខលចេញ",
|
||||
"🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)",
|
||||
"Hello",
|
||||
" Hello",
|
||||
" Hello",
|
||||
" Hello",
|
||||
" Hello",
|
||||
" Hello\n Hello",
|
||||
" (",
|
||||
"\n =",
|
||||
"' era",
|
||||
"Hello, y'all! How are you 😁 ?我想在apple工作1314151天～",
|
||||
"3",
|
||||
"33",
|
||||
"333",
|
||||
"3333",
|
||||
"33333",
|
||||
"333333",
|
||||
"3333333",
|
||||
"33333333",
|
||||
"333333333",
|
||||
]
|
||||
|
||||
|
||||
def generator_custom_text_edge_cases() -> Iterator[str]:
|
||||
"""Edge cases found while debugging"""
|
||||
yield from [
|
||||
'\x1f-a', # unicode_ranges_control, {0x00001C, 0x00001F}
|
||||
'¼-a', # unicode_ranges_digit, 0x00BC
|
||||
'½-a', # unicode_ranges_digit, 0x00BD
|
||||
'¾-a', # unicode_ranges_digit, 0x00BE
|
||||
'a 〇b', # unicode_ranges_digit, 0x3007
|
||||
'Ⅵ-a', # unicode_ranges_digit, {0x00002150, 0x0000218F} // Number Forms
|
||||
'\uFEFF//', # unicode_ranges_control, 0xFEFF (BOM)
|
||||
'Cửa Việt', # llama-3, ignore_merges = true
|
||||
'<s>a', # Phi-3 fail
|
||||
'<unk><|endoftext|><s>', # Phi-3 fail
|
||||
'a\na', # bert fail
|
||||
'"`', # falcon
|
||||
' \u2e4e', # falcon
|
||||
'\n\x0b ', # falcon
|
||||
'a\xa0\xa0\x00b', # jina-v2-es
|
||||
'one <mask>', # jina-v2-es <mask> lstrip=true
|
||||
'a </s> b', # rstrip phi-3
|
||||
'a <mask> b', # lstrip jina-v2
|
||||
'\xa0aC', # deepseek
|
||||
'\u2029 \uA3E4', # deepseek-llm
|
||||
"a ?",
|
||||
'å', # mpt
|
||||
'\U000ac517', # utf-8 encode error, falcon
|
||||
'\U000522f4', # utf-8 encode error, starcoder
|
||||
"<s><s><unk><s>a<s>b<s>c<unk>d<unk></s>",
|
||||
"<s> <s> <unk><s>a<s>b<s>c<unk>d<unk></s>",
|
||||
]
|
||||
|
||||
|
||||
def generator_vocab_words(tokenizer: TokenizerGroundtruth) -> Iterator[str]:
|
||||
"""Brute force check all vocab words"""
|
||||
yield from tokenizer.vocab
|
||||
|
||||
|
||||
def generator_ascii_lr_strip() -> Iterator[str]:
|
||||
WHITESPACES = ["", " ", " "]
|
||||
CHARACTERS = list(chr(i) for i in range(1, 0x80)) + [""]
|
||||
for char1 in CHARACTERS:
|
||||
for char2 in CHARACTERS:
|
||||
for lstrip in WHITESPACES:
|
||||
for rstrip in WHITESPACES:
|
||||
yield lstrip + char1 + char2 + rstrip
|
||||
yield lstrip + char1 + rstrip + char2
|
||||
yield char1 + lstrip + char2 + rstrip
|
||||
|
||||
|
||||
def generator_apostrophe() -> Iterator[str]:
|
||||
WHITESPACES = ["", " ", " "]
|
||||
CHARACTERS = list(chr(i) for i in range(1, 0x80)) + [""]
|
||||
for char1 in CHARACTERS:
|
||||
for char2 in CHARACTERS:
|
||||
for lstrip in WHITESPACES:
|
||||
for rstrip in WHITESPACES:
|
||||
yield char1 + lstrip + "'" + rstrip + char2
|
||||
yield char1 + char2 + lstrip + "'" + rstrip + "z"
|
||||
yield "a" + lstrip + "'" + rstrip + char1 + char2
|
||||
|
||||
|
||||
def generator_added_lr_strip(tokenizer: TokenizerGroundtruth) -> Iterator[str]:
|
||||
WHITESPACES = ["", " ", " ", "\n", "\r\n", "\n\n", "\t", "\t\t"]
|
||||
all_tokens = list(sorted(set(tokenizer.special_tokens + tokenizer.added_tokens)))
|
||||
for token in all_tokens:
|
||||
for lstrip in WHITESPACES:
|
||||
for rstrip in WHITESPACES:
|
||||
yield lstrip + token + rstrip
|
||||
yield "a" + lstrip + token + rstrip
|
||||
yield lstrip + token + rstrip + "z"
|
||||
yield "a" + lstrip + token + rstrip + "z"
|
||||
|
||||
|
||||
def generator_random_added_tokens(tokenizer: TokenizerGroundtruth, iterations=100) -> Iterator[str]:
|
||||
separations = [" ", "\n", "\t", "-", "!", "one", "1", "<s>", "</s>"]
|
||||
all_tokens = list(sorted(set(tokenizer.special_tokens + tokenizer.added_tokens + separations)))
|
||||
rand = random.Random()
|
||||
for m in range(iterations):
|
||||
rand.seed(m)
|
||||
words = rand.choices(all_tokens, k=500)
|
||||
if words and words[0] == tokenizer.bos_token: # skip spam warning of double BOS
|
||||
while len(words) > 1 and words[1] == tokenizer.bos_token: # leave one starting BOS
|
||||
words.pop(0)
|
||||
if tokenizer.add_bos_token: # drop all starting BOS
|
||||
words.pop(0)
|
||||
if words and words[-1] == tokenizer.eos_token: # skip spam warning of double EOS
|
||||
while len(words) > 1 and words[-2] == tokenizer.eos_token: # leave one trailing EOS
|
||||
words.pop(-1)
|
||||
if tokenizer.add_bos_token: # drop all trailing EOS
|
||||
words.pop(-1)
|
||||
yield "".join(words)
|
||||
|
||||
|
||||
def generator_random_chars(iterations=100) -> Iterator[str]:
|
||||
"""Brute force random text with simple characters"""
|
||||
|
||||
NUM_WORDS = 400
|
||||
WHITESPACES = list(" " * 20 + "\n" * 5 + "\r\n" * 5 + "\t" * 5)
|
||||
CHARS = list(sorted(set("""
|
||||
ABCDEFGHIJKLMNOPQRSTUVWXYZ
|
||||
abcdefghijklmnopqrstuvwxyz
|
||||
ÁÉÍÓÚÀÈÌÒÙÂÊÎÔÛÄËÏÖÜ
|
||||
áéíóúàèìòùâêîôûäëïöü
|
||||
.-,*/-+ª!"·$%&/()=?¿[]{}<>\\|@#~½¬~;:_
|
||||
""")))
|
||||
|
||||
rand = random.Random()
|
||||
for m in range(iterations):
|
||||
rand.seed(m)
|
||||
text = []
|
||||
for _ in range(NUM_WORDS):
|
||||
k = rand.randint(1, 7)
|
||||
word = rand.choices(CHARS, k=k)
|
||||
word.append(rand.choice(WHITESPACES))
|
||||
text.append("".join(word))
|
||||
yield "".join(text)
|
||||
|
||||
|
||||
def generator_unicodes() -> Iterator[str]:
|
||||
"""Iterate unicode characters"""
|
||||
|
||||
MAX_CODEPOINTS = 0x30000 # 0x110000
|
||||
|
||||
def _valid(cpt):
|
||||
if cpt >= 0x30000: # unassigned and supplementary
|
||||
return False
|
||||
# if cpt == 0x2029: # deepseek-llm
|
||||
# return False
|
||||
if unicodedata.category(chr(cpt)) in ("Cn", "Cs", "Co"): # undefined, surrogates, private
|
||||
return False
|
||||
return True
|
||||
|
||||
characters = [chr(cpt) for cpt in range(0, MAX_CODEPOINTS) if _valid(cpt)]
|
||||
|
||||
yield from characters
|
||||
|
||||
|
||||
def generator_random_unicodes(iterations=100) -> Iterator[str]:
|
||||
"""Brute force random text with unicode characters"""
|
||||
|
||||
NUM_WORDS = 200
|
||||
WHITESPACES = list(" " * 20 + "\n" * 5 + "\r\n" * 5 + "\t" * 5)
|
||||
|
||||
characters = list(generator_unicodes())
|
||||
|
||||
rand = random.Random()
|
||||
for m in range(iterations):
|
||||
rand.seed(m)
|
||||
text = []
|
||||
for _ in range(NUM_WORDS):
|
||||
k = rand.randint(1, 7)
|
||||
word = rand.choices(characters, k=k)
|
||||
word.append(rand.choice(WHITESPACES))
|
||||
text.append("".join(word))
|
||||
yield "".join(text)
|
||||
|
||||
|
||||
def generator_random_vocab_chars(tokenizer: TokenizerGroundtruth, iterations=100) -> Iterator[str]:
|
||||
"""Brute force random text with vocab characters"""
|
||||
|
||||
vocab_chars = set()
|
||||
for word in tokenizer.vocab:
|
||||
vocab_chars.update(word)
|
||||
vocab_chars = list(sorted(vocab_chars))
|
||||
|
||||
rand = random.Random()
|
||||
for m in range(iterations):
|
||||
rand.seed(m)
|
||||
text = rand.choices(vocab_chars, k=1024)
|
||||
yield "".join(text)
|
||||
|
||||
|
||||
def generator_random_vocab_words(tokenizer: TokenizerGroundtruth, iterations=100) -> Iterator[str]:
|
||||
"""Brute force random text from vocab words"""
|
||||
|
||||
vocab = [w.strip() for w in tokenizer.vocab]
|
||||
yield from vocab
|
||||
|
||||
rand = random.Random()
|
||||
for m in range(iterations):
|
||||
rand.seed(m)
|
||||
text = []
|
||||
num_words = rand.randint(300, 400)
|
||||
for i in range(num_words):
|
||||
k = rand.randint(1, 3)
|
||||
words = rand.choices(vocab, k=k)
|
||||
sep = rand.choice(" \n\r\t")
|
||||
text.append("".join(words) + sep)
|
||||
yield "".join(text)
|
||||
|
||||
|
||||
def compare_tokenizers(tokenizer1: TokenizerGroundtruth, tokenizer2: TokenizerLlamaCpp, generator: Iterator[str]):
|
||||
|
||||
def find_first_mismatch(ids1: list[int] | str, ids2: list[int] | str):
|
||||
for i, (a, b) in enumerate(zip(ids1, ids2)):
|
||||
if a != b:
|
||||
return i
|
||||
if len(ids1) == len(ids2):
|
||||
return -1
|
||||
return min(len(ids1), len(ids2))
|
||||
|
||||
def check_detokenizer(text: str, text1: str, text2: str) -> bool:
|
||||
if text1 == text2: # equal to TokenizerGroundtruth?
|
||||
return True
|
||||
# equal to source text?
|
||||
if tokenizer1.add_bos_token and tokenizer1.bos_token and isinstance(tokenizer1.bos_token, str): # remove BOS
|
||||
if text2.startswith(tokenizer1.bos_token):
|
||||
text2 = text2[len(tokenizer1.bos_token):]
|
||||
if tokenizer1.add_eos_token and tokenizer1.eos_token and isinstance(tokenizer1.eos_token, str): # remove EOS
|
||||
if text2.endswith(tokenizer1.eos_token):
|
||||
text2 = text2[:-len(tokenizer1.eos_token)]
|
||||
return text == text2
|
||||
|
||||
t_encode1 = 0
|
||||
t_encode2 = 0
|
||||
t_decode1 = 0
|
||||
t_decode2 = 0
|
||||
t_start = time.perf_counter()
|
||||
encode_errors = 0
|
||||
decode_errors = 0
|
||||
MAX_ERRORS = 10
|
||||
|
||||
logger.info("%s: %s" % (getattr(generator, "__qualname__", ""), "ini"))
|
||||
for text in generator:
|
||||
# print(repr(text), text.encode())
|
||||
# print(repr(text), hex(ord(text[0])), text.encode())
|
||||
t0 = time.perf_counter()
|
||||
ids1 = tokenizer1.encode(text)
|
||||
t1 = time.perf_counter()
|
||||
ids2 = tokenizer2.encode(text)
|
||||
t2 = time.perf_counter()
|
||||
text1 = tokenizer1.decode(ids1)
|
||||
t3 = time.perf_counter()
|
||||
text2 = tokenizer2.decode(ids1)
|
||||
t4 = time.perf_counter()
|
||||
t_encode1 += t1 - t0
|
||||
t_encode2 += t2 - t1
|
||||
t_decode1 += t3 - t2
|
||||
t_decode2 += t4 - t3
|
||||
if encode_errors < MAX_ERRORS and ids1 != ids2:
|
||||
i = find_first_mismatch(ids1, ids2)
|
||||
ids1 = list(ids1)[max(0, i - 2) : i + 5 + 1]
|
||||
ids2 = list(ids2)[max(0, i - 2) : i + 5 + 1]
|
||||
logger.error(" Expected: " + str(ids1))
|
||||
logger.error(" Result: " + str(ids2))
|
||||
encode_errors += 1
|
||||
logger.error(f" {encode_errors=}")
|
||||
if decode_errors < MAX_ERRORS and not check_detokenizer(text, text1, text2):
|
||||
i = find_first_mismatch(text1, text2)
|
||||
text1 = list(text1[max(0, i - 2) : i + 5 + 1])
|
||||
text2 = list(text2[max(0, i - 2) : i + 5 + 1])
|
||||
logger.error(" Expected: " + " ".join(hex(ord(x)) for x in text1))
|
||||
logger.error(" Result: " + " ".join(hex(ord(x)) for x in text2))
|
||||
decode_errors += 1
|
||||
logger.error(f" {decode_errors=}")
|
||||
if encode_errors >= MAX_ERRORS and decode_errors >= MAX_ERRORS:
|
||||
logger.error(f" EXIT: {encode_errors=} {decode_errors=}")
|
||||
# raise Exception()
|
||||
break
|
||||
|
||||
t_total = time.perf_counter() - t_start
|
||||
logger.info(f"{getattr(generator, '__qualname__', '')}: end, {t_encode1=:.3f} {t_encode2=:.3f} {t_decode1=:.3f} {t_decode2=:.3f} {t_total=:.3f}")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("vocab_file", type=str, help="path to vocab 'gguf' file")
|
||||
parser.add_argument("dir_tokenizer", type=str, help="directory containing 'tokenizer.model' file")
|
||||
parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
logging.basicConfig(level = logging.DEBUG if args.verbose else logging.INFO)
|
||||
logger.info(f"VOCABFILE: '{args.vocab_file}'")
|
||||
|
||||
tokenizer1 = TokenizerGroundtruth(args.dir_tokenizer)
|
||||
tokenizer2 = TokenizerLlamaCpp(args.vocab_file)
|
||||
|
||||
# compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text())
|
||||
# compare_tokenizers(tokenizer1, tokenizer2, generator_custom_text_edge_cases())
|
||||
compare_tokenizers(tokenizer1, tokenizer2, generator_ascii_lr_strip())
|
||||
compare_tokenizers(tokenizer1, tokenizer2, generator_apostrophe())
|
||||
compare_tokenizers(tokenizer1, tokenizer2, generator_unicodes())
|
||||
compare_tokenizers(tokenizer1, tokenizer2, generator_vocab_words(tokenizer1))
|
||||
compare_tokenizers(tokenizer1, tokenizer2, generator_added_lr_strip(tokenizer1))
|
||||
# compare_tokenizers(tokenizer1, tokenizer2, generator_random_added_tokens(tokenizer1, 10_000))
|
||||
# compare_tokenizers(tokenizer1, tokenizer2, generator_random_chars(10_000))
|
||||
# compare_tokenizers(tokenizer1, tokenizer2, generator_random_unicodes(10_000))
|
||||
# compare_tokenizers(tokenizer1, tokenizer2, generator_random_vocab_chars(tokenizer1, 10_000))
|
||||
# compare_tokenizers(tokenizer1, tokenizer2, generator_random_vocab_words(tokenizer1, 5_000))
|
||||
|
||||
tokenizer2.model.free()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# main()
|
||||
|
||||
if True:
|
||||
logging.basicConfig(
|
||||
level = logging.DEBUG,
|
||||
format = "%(asctime)s.%(msecs)03d %(name)s %(levelname)s %(message)s",
|
||||
datefmt = "%Y-%m-%d %H:%M:%S",
|
||||
filename = logger.name + ".log",
|
||||
filemode = "a"
|
||||
)
|
||||
logging.basicConfig(
|
||||
level = logging.DEBUG,
|
||||
format = "%(levelname)s %(message)s",
|
||||
)
|
||||
|
||||
path_tokenizers = Path("./models/tokenizers/")
|
||||
path_vocab_format = "./models/ggml-vocab-%s.gguf"
|
||||
|
||||
tokenizers = [
|
||||
"llama-spm", # SPM
|
||||
"phi-3", # SPM
|
||||
"gemma", # SPM
|
||||
"gemma-2", # SPM
|
||||
"baichuan", # SPM
|
||||
"bert-bge", # WPM
|
||||
"jina-v2-en", # WPM
|
||||
"llama-bpe", # BPE
|
||||
"phi-2", # BPE
|
||||
"deepseek-llm", # BPE
|
||||
"deepseek-coder", # BPE
|
||||
"falcon", # BPE
|
||||
"mpt", # BPE
|
||||
"starcoder", # BPE
|
||||
"gpt-2", # BPE
|
||||
"stablelm2", # BPE
|
||||
"refact", # BPE
|
||||
"qwen2", # BPE
|
||||
"olmo", # BPE
|
||||
"jina-v2-es", # BPE
|
||||
"jina-v2-de", # BPE
|
||||
"smaug-bpe", # BPE
|
||||
"poro-chat", # BPE
|
||||
"jina-v2-code", # BPE
|
||||
"viking", # BPE
|
||||
"jais", # BPE
|
||||
]
|
||||
|
||||
logger.info("=" * 50)
|
||||
for tokenizer in tokenizers:
|
||||
logger.info("-" * 50)
|
||||
logger.info(f"TOKENIZER: '{tokenizer}'")
|
||||
vocab_file = Path(path_vocab_format % tokenizer)
|
||||
dir_tokenizer = path_tokenizers / tokenizer
|
||||
main([str(vocab_file), str(dir_tokenizer), "--verbose"])
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
if [ $# -lt 2 ]; then
|
||||
printf "Usage: $0 <git-repo> <target-folder> [<test-exe>]\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $# -eq 3 ]; then
|
||||
toktest=$3
|
||||
else
|
||||
toktest="./test-tokenizer-0"
|
||||
fi
|
||||
|
||||
if [ ! -x $toktest ]; then
|
||||
printf "Test executable \"$toktest\" not found!\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
repo=$1
|
||||
folder=$2
|
||||
|
||||
if [ -d $folder ] && [ -d $folder/.git ]; then
|
||||
(cd $folder; git pull)
|
||||
else
|
||||
git clone $repo $folder
|
||||
|
||||
# byteswap models if on big endian
|
||||
if [ "$(uname -m)" = s390x ]; then
|
||||
for f in $folder/*/*.gguf; do
|
||||
echo YES | python3 "$(dirname $0)/../gguf-py/gguf/scripts/gguf_convert_endian.py" $f big
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
shopt -s globstar
|
||||
for gguf in $folder/**/*.gguf; do
|
||||
if [ -f $gguf.inp ] && [ -f $gguf.out ]; then
|
||||
$toktest $gguf
|
||||
else
|
||||
printf "Found \"$gguf\" without matching inp/out files, ignoring...\n"
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
#include "../src/unicode.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
int main() {
|
||||
bool success = true;
|
||||
const std::vector<std::pair<std::vector<uint32_t>, std::vector<uint32_t>>> normalization = {
|
||||
{{}, {}},
|
||||
{{0x65, 0x301}, {0xE9}},
|
||||
{{0xE9}, {0xE9}},
|
||||
{{0x212B}, {0xC5}},
|
||||
{{0x61, 0x315, 0x300}, {0xE0, 0x315}},
|
||||
{{0x61, 0x305, 0x301}, {0x61, 0x305, 0x301}},
|
||||
{{0x301, 0x327}, {0x327, 0x301}},
|
||||
{{0x1100, 0x1161, 0x11A8}, {0xAC01}},
|
||||
{{0xAC01}, {0xAC01}},
|
||||
{{0xFB01, 0xFF21}, {0xFB01, 0xFF21}},
|
||||
{{0x958}, {0x915, 0x93C}},
|
||||
{{0x1D15E}, {0x1D157, 0x1D165}},
|
||||
};
|
||||
for (const auto & test : normalization) {
|
||||
if (unicode_cpts_normalize_nfc(test.first) != test.second || unicode_cpts_normalize_nfc(test.second) != test.second) {
|
||||
fprintf(stderr, "unexpected NFC normalization\n");
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const struct {
|
||||
std::string text;
|
||||
std::vector<uint32_t> expected;
|
||||
} utf8_tests[] = {
|
||||
{ "\x7f", { 0x7f } },
|
||||
{ "\xc2\x80", { 0x80 } },
|
||||
{ "\xdf\xbf", { 0x7ff } },
|
||||
{ "\xe0\xa0\x80", { 0x800 } },
|
||||
{ "\xef\xbf\xbf", { 0xffff } },
|
||||
{ "\xf0\x90\x80\x80", { 0x10000 } },
|
||||
{ "\xf4\x8f\xbf\xbf", { 0x10ffff } },
|
||||
// Invalid-input cases preserve the existing recovery behavior: one U+FFFD per byte.
|
||||
{ "\xc0\xa0", { 0xfffd, 0xfffd } },
|
||||
{ "\xc1\xbf", { 0xfffd, 0xfffd } },
|
||||
{ "\xe0\x80\xa0", { 0xfffd, 0xfffd, 0xfffd } },
|
||||
{ "\xe0\x9f\xbf", { 0xfffd, 0xfffd, 0xfffd } },
|
||||
{ "\xf0\x80\x80\xa0", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
|
||||
{ "\xf0\x8f\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
|
||||
{ "\xf4\x90\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
|
||||
{ "\xf5\x80\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
|
||||
{ "\xf7\xbf\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
|
||||
{ "\xe2\x80", { 0xfffd, 0xfffd } },
|
||||
};
|
||||
|
||||
for (const auto & test : utf8_tests) {
|
||||
auto expected = test.expected;
|
||||
expected.insert(expected.begin(), 'a');
|
||||
expected.push_back('b');
|
||||
if (unicode_cpts_from_utf8("a" + test.text + "b") != expected) {
|
||||
fprintf(stderr, "unexpected UTF-8 decoding for bytes:");
|
||||
for (const unsigned char byte : test.text) {
|
||||
fprintf(stderr, " %02x", byte);
|
||||
}
|
||||
fprintf(stderr, "\n");
|
||||
success = false;
|
||||
}
|
||||
}
|
||||
|
||||
const std::vector<std::string> regex_exprs = {
|
||||
"[~][A-Za-z]+| ?[\\p{S}]+|\\s+",
|
||||
};
|
||||
const std::vector<std::string> expected = { " ~", "foo" };
|
||||
const auto actual = unicode_regex_split(" ~foo", regex_exprs, false);
|
||||
|
||||
if (actual != expected) {
|
||||
fprintf(stderr, "unexpected split:");
|
||||
for (const auto & piece : actual) {
|
||||
fprintf(stderr, " [%s]", piece.c_str());
|
||||
}
|
||||
fprintf(stderr, "\n");
|
||||
success = false;
|
||||
}
|
||||
|
||||
return success ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <exception>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <regex>
|
||||
#include <vector>
|
||||
|
||||
struct testing {
|
||||
std::ostream &out;
|
||||
std::vector<std::string> stack;
|
||||
std::regex filter;
|
||||
bool filter_tests = false;
|
||||
bool throw_exception = false;
|
||||
bool verbose = false;
|
||||
int tests = 0;
|
||||
int assertions = 0;
|
||||
int failures = 0;
|
||||
int unnamed = 0;
|
||||
int exceptions = 0;
|
||||
int skipped = 0;
|
||||
|
||||
// set by skip(), read by the innermost test()
|
||||
bool skip_current = false;
|
||||
std::string skip_reason;
|
||||
|
||||
static constexpr std::size_t status_column = 80;
|
||||
|
||||
explicit testing(std::ostream &os = std::cout) : out(os) {}
|
||||
|
||||
std::string indent() const {
|
||||
if (stack.empty()) {
|
||||
return "";
|
||||
}
|
||||
return std::string((stack.size() - 1) * 2, ' ');
|
||||
}
|
||||
|
||||
std::string full_name() const {
|
||||
std::string name;
|
||||
for (std::size_t i = 0; i < stack.size(); ++i) {
|
||||
if (i != 0) {
|
||||
name += ".";
|
||||
}
|
||||
name += stack[i];
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
void log(const std::string & msg) {
|
||||
if (verbose) {
|
||||
out << indent() << " " << msg << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
void set_filter(const std::string & re) {
|
||||
filter = std::regex(re);
|
||||
filter_tests = true;
|
||||
}
|
||||
|
||||
bool should_run() const {
|
||||
if (filter_tests) {
|
||||
if (!std::regex_match(full_name(), filter)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
void run_with_exceptions(F &&f, const char *ctx) {
|
||||
try {
|
||||
f();
|
||||
} catch (const std::exception &e) {
|
||||
++failures;
|
||||
++exceptions;
|
||||
out << indent() << "UNHANDLED EXCEPTION (" << ctx << "): " << e.what() << "\n";
|
||||
if (throw_exception) {
|
||||
throw;
|
||||
}
|
||||
} catch (...) {
|
||||
++failures;
|
||||
++exceptions;
|
||||
out << indent() << "UNHANDLED EXCEPTION (" << ctx << "): unknown\n";
|
||||
if (throw_exception) {
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void skip(const std::string &reason = "") {
|
||||
skip_current = true;
|
||||
skip_reason = reason;
|
||||
}
|
||||
|
||||
void print_result(const std::string &label, int new_failures, int new_assertions, const std::string &extra = "", bool was_skipped = false) const {
|
||||
std::string line = indent() + label;
|
||||
|
||||
std::string details;
|
||||
if (new_assertions > 0) {
|
||||
if (new_failures == 0) {
|
||||
details = std::to_string(new_assertions) + " assertion(s)";
|
||||
} else {
|
||||
details = std::to_string(new_failures) + " of " +
|
||||
std::to_string(new_assertions) + " assertion(s) failed";
|
||||
}
|
||||
}
|
||||
if (!extra.empty()) {
|
||||
if (!details.empty()) {
|
||||
details += ", ";
|
||||
}
|
||||
details += extra;
|
||||
}
|
||||
|
||||
if (!details.empty()) {
|
||||
line += " (" + details + ")";
|
||||
}
|
||||
|
||||
std::string status = new_failures != 0 ? "[FAIL]" : (was_skipped ? "[SKIP]" : "[PASS]");
|
||||
|
||||
if (line.size() + 1 < status_column) {
|
||||
line.append(status_column - line.size(), ' ');
|
||||
} else {
|
||||
line.push_back(' ');
|
||||
}
|
||||
|
||||
out << line << status << "\n";
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
void test(const std::string &name, F f) {
|
||||
stack.push_back(name);
|
||||
if (!should_run()) {
|
||||
stack.pop_back();
|
||||
return;
|
||||
}
|
||||
|
||||
++tests;
|
||||
out << indent() << name << "\n";
|
||||
|
||||
int before_failures = failures;
|
||||
int before_assertions = assertions;
|
||||
|
||||
// do not let a skipped subtest also mark its parent as skipped
|
||||
bool outer_skip = skip_current;
|
||||
std::string outer_skip_reason = skip_reason;
|
||||
skip_current = false;
|
||||
skip_reason.clear();
|
||||
|
||||
run_with_exceptions([&] { f(*this); }, "test");
|
||||
|
||||
int new_failures = failures - before_failures;
|
||||
int new_assertions = assertions - before_assertions;
|
||||
|
||||
bool was_skipped = skip_current && new_failures == 0;
|
||||
if (was_skipped) {
|
||||
++skipped;
|
||||
}
|
||||
|
||||
print_result(name, new_failures, new_assertions, was_skipped ? skip_reason : "", was_skipped);
|
||||
|
||||
skip_current = outer_skip;
|
||||
skip_reason = outer_skip_reason;
|
||||
|
||||
stack.pop_back();
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
void test(F f) {
|
||||
test("test #" + std::to_string(++unnamed), f);
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
void bench(const std::string &name, F f, int iterations = 100) {
|
||||
stack.push_back(name);
|
||||
if (!should_run()) {
|
||||
stack.pop_back();
|
||||
return;
|
||||
}
|
||||
|
||||
++tests;
|
||||
out << indent() << "[bench] " << name << "\n";
|
||||
|
||||
int before_failures = failures;
|
||||
int before_assertions = assertions;
|
||||
|
||||
using clock = std::chrono::high_resolution_clock;
|
||||
|
||||
std::chrono::microseconds duration(0);
|
||||
|
||||
run_with_exceptions([&] {
|
||||
for (auto i = 0; i < iterations; i++) {
|
||||
auto start = clock::now();
|
||||
f();
|
||||
duration += std::chrono::duration_cast<std::chrono::microseconds>(clock::now() - start);
|
||||
}
|
||||
}, "bench");
|
||||
|
||||
auto avg_elapsed = duration.count() / iterations;
|
||||
auto avg_elapsed_s = std::chrono::duration_cast<std::chrono::duration<double>>(duration).count() / iterations;
|
||||
auto rate = (avg_elapsed_s > 0.0) ? (1.0 / avg_elapsed_s) : 0.0;
|
||||
|
||||
int new_failures = failures - before_failures;
|
||||
int new_assertions = assertions - before_assertions;
|
||||
|
||||
std::string extra =
|
||||
"n=" + std::to_string(iterations) +
|
||||
" avg=" + std::to_string(avg_elapsed) + "us" +
|
||||
" rate=" + std::to_string(int(rate)) + "/s";
|
||||
|
||||
print_result("[bench] " + name, new_failures, new_assertions, extra);
|
||||
|
||||
stack.pop_back();
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
void bench(F f, int iterations = 100) {
|
||||
bench("bench #" + std::to_string(++unnamed), f, iterations);
|
||||
}
|
||||
|
||||
// Assertions
|
||||
bool assert_true(bool cond) {
|
||||
return assert_true("", cond);
|
||||
}
|
||||
|
||||
bool assert_true(const std::string &msg, bool cond) {
|
||||
++assertions;
|
||||
if (!cond) {
|
||||
++failures;
|
||||
out << indent() << "ASSERTION FAILED";
|
||||
if (!msg.empty()) {
|
||||
out << " : " << msg;
|
||||
}
|
||||
out << "\n";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename A, typename B>
|
||||
bool assert_equal(const A &expected, const B &actual) {
|
||||
return assert_equal("", expected, actual);
|
||||
}
|
||||
|
||||
template <typename A, typename B>
|
||||
bool assert_equal(const std::string &msg, const A &expected, const B &actual) {
|
||||
++assertions;
|
||||
if (!(actual == expected)) {
|
||||
++failures;
|
||||
out << indent() << "ASSERT EQUAL FAILED";
|
||||
if (!msg.empty()) {
|
||||
out << " : " << msg;
|
||||
}
|
||||
out << "\n";
|
||||
|
||||
out << indent() << " expected: " << expected << "\n";
|
||||
out << indent() << " actual : " << actual << "\n";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Print summary and return an exit code
|
||||
int summary() const {
|
||||
out << "\n";
|
||||
out << "tests : " << tests << "\n";
|
||||
out << "assertions : " << assertions << "\n";
|
||||
out << "failures : " << failures << "\n";
|
||||
out << "exceptions : " << exceptions << "\n";
|
||||
out << "skipped : " << skipped << "\n";
|
||||
return failures == 0 ? 0 : 1;
|
||||
}
|
||||
};
|
||||
Reference in new issue
Block a user