zefie/nvfp4_gguf_quant
1
0 Fork 0
Code Actions Activity

include llama.cpp since there were some edits

Browse Source
This commit is contained in:
zefie committed 2026-11-01 16:32:35 -05:00
1 parent db7f28d087
commit 8c66990853
3918 files changed
+1640170 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

llama.cpp/ggml/src/ggml-hexagon/CMakeLists.txt
+128
View File
@@ -0,0 +1,128 @@
file(TO_CMAKE_PATH "${HEXAGON_SDK_ROOT}" HEXAGON_SDK_ROOT)
file(TO_CMAKE_PATH "${HEXAGON_TOOLS_ROOT}" HEXAGON_TOOLS_ROOT)
if (NOT IS_DIRECTORY "${HEXAGON_SDK_ROOT}")
message(FATAL_ERROR "Make sure HEXAGON_SDK_ROOT point to the correct Hexagon SDK installation.")
endif()
if (NOT IS_DIRECTORY "${HEXAGON_TOOLS_ROOT}")
message("Try to read HEXAGON_TOOLS_ROOT from hexagon_sdk.json")
file(READ "${HEXAGON_SDK_ROOT}/hexagon_sdk.json" HEXAGON_SDK_CONFIG_PATH)
string(JSON HEXAGON_TOOLS_PATH GET ${HEXAGON_SDK_CONFIG_PATH} "root" "tools" "info" 0 "path")
message("Found HEXAGON_TOOLS_PATH: ${HEXAGON_TOOLS_PATH}")
set(HEXAGON_TOOLS_ROOT "${HEXAGON_SDK_ROOT}/${HEXAGON_TOOLS_PATH}")
file(TO_CMAKE_PATH "${HEXAGON_TOOLS_ROOT}" HEXAGON_TOOLS_ROOT)
if (NOT IS_DIRECTORY "${HEXAGON_TOOLS_ROOT}")
message(FATAL_ERROR "Make sure HEXAGON_TOOLS_ROOT point to the correct Hexagon SDK installation.")
endif()
endif()
message(STATUS "hexagon: using ${HEXAGON_SDK_ROOT} and ${HEXAGON_TOOLS_ROOT} for building libggml-htp skels")
include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake)
include(ExternalProject)
find_package(Python3 REQUIRED COMPONENTS Interpreter)
option(GGML_HEXAGON_HTP_DEBUG "ggml-hexagon: enable HTP debug output" OFF)
set(GGML_HEXAGON_HTP_BUILD_TYPE "Release" CACHE STRING "ggml-hexagon: HTP skel build type (Release, RelWithDebInfo, Debug)")
set(GGML_HEXAGON_HTP_CERT "$ENV{HEXAGON_HTP_CERT}" CACHE PATH "ggml-hexagon: enable HTP library signing using certificate")
add_library(htp_iface OBJECT
${CMAKE_CURRENT_BINARY_DIR}/htp_iface_stub.c)
set_target_properties(htp_iface PROPERTIES POSITION_INDEPENDENT_CODE ON)
target_include_directories(htp_iface PUBLIC
${HEXAGON_SDK_ROOT}/incs
${HEXAGON_SDK_ROOT}/incs/stddef
${HEXAGON_SDK_ROOT}/utils/examples
${CMAKE_CURRENT_SOURCE_DIR}/htp
${CMAKE_CURRENT_BINARY_DIR})
build_idl(htp/htp_iface.idl htp_iface)
if (CMAKE_SYSTEM_NAME MATCHES Android)
target_link_options(htp_iface PUBLIC -llog -ldl)
elseif (CMAKE_SYSTEM_NAME MATCHES Windows)
target_precompile_headers(htp_iface PUBLIC <sal.h>)
else()
target_link_options(htp_iface PUBLIC -ldl)
endif()
set(TARGET_NAME ggml-hexagon)
ggml_add_backend_library(${TARGET_NAME}
ggml-hexagon.cpp
htp-common.h
htp-kparams.cpp
htp-kparams.h
htp-opnode.cpp
htp-opnode.h
htp-fusion.cpp
htp-fusion.h
htp-repack.cpp
htp-repack.h
htp-drv.cpp
htp-drv.h
libdl.h
../../include/ggml-hexagon.h)
target_link_libraries(${TARGET_NAME} PRIVATE htp_iface)
target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/htp ${CMAKE_CURRENT_BINARY_DIR})
# Build HTP skels
set(HTP_SKELS)
set(HTP_PROJECTS)
function(build_htp_skel V)
ExternalProject_Add(htp-${V}
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/htp BUILD_ALWAYS ON
CMAKE_ARGS
-DCMAKE_BUILD_TYPE=${GGML_HEXAGON_HTP_BUILD_TYPE}
-DCMAKE_TOOLCHAIN_FILE=${CMAKE_CURRENT_SOURCE_DIR}/htp/cmake-toolchain.cmake
-DCMAKE_INSTALL_LIBDIR=${CMAKE_CURRENT_BINARY_DIR}
-DHEXAGON_SDK_ROOT=${HEXAGON_SDK_ROOT}
-DHEXAGON_TOOLS_ROOT=${HEXAGON_TOOLS_ROOT}
-DHEXAGON_HTP_DEBUG=${GGML_HEXAGON_HTP_DEBUG}
-DDSP_VERSION=${V}
-DPython3_EXECUTABLE=${Python3_EXECUTABLE}
-DPREBUILT_LIB_DIR="toolv21_${V}")
list(APPEND HTP_SKELS ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp-${V}.so)
list(APPEND HTP_PROJECTS htp-${V})
set(HTP_SKELS ${HTP_SKELS} PARENT_SCOPE)
set(HTP_PROJECTS ${HTP_PROJECTS} PARENT_SCOPE)
endfunction()
build_htp_skel(v73)
build_htp_skel(v75)
build_htp_skel(v79)
build_htp_skel(v81)
build_htp_skel(v85)
# Install Hexagon skels required at runtime
install(FILES ${HTP_SKELS} TYPE LIB)
if (CMAKE_SYSTEM_NAME MATCHES Windows AND GGML_HEXAGON_HTP_CERT)
file(TO_CMAKE_PATH "$ENV{WINDOWS_SDK_BIN}/arm64" WINSDK_BIN0_ARM64)
file(TO_CMAKE_PATH "$ENV{WINDOWS_SDK_BIN}/x86" WINSDK_BIN0_X86)
file(TO_CMAKE_PATH "$ENV{WindowsSdkVerBinPath}/arm64" WINSDK_BIN1_ARM64)
file(TO_CMAKE_PATH "$ENV{WindowsSdkVerBinPath}/x86" WINSDK_BIN1_X86)
set(WINSDK_PATHS ${WINSDK_BIN0_ARM64} ${WINSDK_BIN0_X86} ${WINSDK_BIN1_ARM64} ${WINSDK_BIN1_X86})
find_program(INF2CAT NAMES inf2cat.exe PATHS ${WINSDK_PATHS} REQUIRED)
find_program(SIGNTOOL NAMES signtool.exe PATHS ${WINSDK_PATHS} REQUIRED)
message(STATUS "hexagon: using ${GGML_HEXAGON_HTP_CERT} to sign libggml-htp skels")
set(LIBGGML_HTP_CAT ${CMAKE_CURRENT_BINARY_DIR}/libggml-htp.cat)
add_custom_target(libggml-htp-cat
BYPRODUCTS ${LIBGGML_HTP_CAT}
DEPENDS libggml-htp.inf ${HTP_PROJECTS}
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/libggml-htp.inf ${CMAKE_CURRENT_BINARY_DIR}
COMMAND ${INF2CAT} /driver:${CMAKE_CURRENT_BINARY_DIR} /os:10_25H2_ARM64
COMMAND ${SIGNTOOL} sign /fd sha256 /f ${GGML_HEXAGON_HTP_CERT} ${LIBGGML_HTP_CAT}
COMMENT "generating and signing libggml-htp.cat file"
VERBATIM
)
add_dependencies(${TARGET_NAME} libggml-htp-cat)
install(FILES ${LIBGGML_HTP_CAT} TYPE LIB)
endif()
llama.cpp/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+3729
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp-common.h
+106
View File
@@ -0,0 +1,106 @@
#ifndef HTP_COMMON_H
#define HTP_COMMON_H
#include "htp-opnode.h"
#include "htp-repack.h"
struct htp_tensor_extra {
std::vector<uint8_t> shadow_buf;
size_t shadow_size { 0 };
uint32_t flags { 0 };
};
extern int opt_arch;
extern size_t opt_ndev;
extern size_t opt_nhvx;
extern int opt_nhmx;
extern size_t opt_vmem;
extern int opt_etm;
extern int opt_verbose;
extern int opt_profile;
extern bool opt_hostbuf;
extern bool opt_dma64;
extern int opt_power_mode;
extern size_t opt_mbuf_dyn;
extern size_t opt_mbuf_static;
extern size_t opt_mbuf_total;
extern int opt_mm_select;
extern int opt_fa_select;
extern int opt_fa_split;
extern int opt_gdn_select;
extern int opt_ar_select;
extern int opt_ar_scatter;
extern int opt_opbatch;
extern int opt_opqueue;
extern int opt_optrace;
extern int opt_oppoll;
extern int opt_opfusion;
extern int opt_rt_repack;
#ifndef HEX_VERBOSE
#define HEX_VERBOSE(...) \
if (opt_verbose) GGML_LOG_DEBUG(__VA_ARGS__)
#endif
static inline bool htp_tensor_is_fuseable(const struct ggml_tensor * t) {
if (!t || !t->extra) return false;
const auto * extra = (const struct htp_tensor_extra *) t->extra;
return (extra->flags & HTP_TENSOR_FUSEABLE) != 0;
}
static inline bool htp_can_rt_repack(const struct ggml_tensor * t) {
if (opt_rt_repack == 0 || !htp_is_tiled_weight_tensor(t)) {
return false;
}
if (t->buffer && ggml_backend_buft_is_host(t->buffer->buft)) {
return false;
}
return opt_rt_repack > 1 || ggml_is_quantized(t->type);
}
static inline bool htp_tensor_is_tiled(const struct ggml_tensor * t) {
const auto * extra = (const struct htp_tensor_extra *) t->extra;
if (extra && (extra->flags & HTP_TENSOR_TILED)) {
return true;
}
return htp_can_rt_repack(t);
}
static inline bool htp_needs_repack(const struct ggml_tensor * t) {
const auto * extra = (const struct htp_tensor_extra *) t->extra;
if (extra && (extra->flags & HTP_TENSOR_TILED)) {
return false;
}
return htp_can_rt_repack(t);
}
static inline bool htp_tensors_overlap(const struct ggml_tensor * a, const struct ggml_tensor * b) {
const uintptr_t a0 = (uintptr_t) a->data;
const uintptr_t b0 = (uintptr_t) b->data;
const uintptr_t a1 = a0 + ggml_nbytes(a);
const uintptr_t b1 = b0 + ggml_nbytes(b);
return a0 < b1 && b0 < a1;
}
static inline bool htp_can_row_partition(const struct ggml_tensor * t) {
if (t->ne[1] > 1 && (t->nb[1] & 127) != 0) return false;
if (t->ne[2] > 1 && (t->nb[2] & 127) != 0) return false;
if (t->ne[3] > 1 && (t->nb[3] & 127) != 0) return false;
return true;
}
struct ggml_hexagon_session;
const char * htp_session_name(const struct ggml_hexagon_session * sess);
size_t htp_session_vtcm_size(const struct ggml_hexagon_session * sess);
uint32_t htp_session_n_threads(const struct ggml_hexagon_session * sess);
uint32_t htp_session_n_hmx(const struct ggml_hexagon_session * sess);
uint32_t htp_session_mdev_count(const struct ggml_hexagon_session * sess);
htp_op_code htp_op_remap(const struct ggml_tensor * t);
#endif // HTP_COMMON_H
llama.cpp/ggml/src/ggml-hexagon/htp-drv.cpp
+425
View File
@@ -0,0 +1,425 @@
#include <filesystem>
#include <set>
#include <sstream>
#include <string>
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
# ifndef NOMINMAX
# define NOMINMAX
# endif
# include <windows.h>
# include <winevt.h>
#else
# include <dlfcn.h>
# include <unistd.h>
#endif
#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wsign-compare"
#pragma clang diagnostic ignored "-Wlanguage-extension-token"
#pragma clang diagnostic ignored "-Wmicrosoft-enum-value"
#pragma clang diagnostic ignored "-Wnested-anon-types"
#include "ggml-impl.h"
#include "htp-drv.h"
#include "libdl.h"
#include <domain.h>
//
// Driver API types
//
typedef void * (*rpcmem_alloc_pfn_t)(int heapid, uint32_t flags, int size);
typedef void * (*rpcmem_alloc2_pfn_t)(int heapid, uint32_t flags, size_t size);
typedef void (*rpcmem_free_pfn_t)(void * po);
typedef int (*rpcmem_to_fd_pfn_t)(void * po);
typedef AEEResult (*dspqueue_create_pfn_t)(int domain,
uint32_t flags,
uint32_t req_queue_size,
uint32_t resp_queue_size,
dspqueue_callback_t packet_callback,
dspqueue_callback_t error_callback,
void * callback_context,
dspqueue_t * queue);
typedef AEEResult (*dspqueue_close_pfn_t)(dspqueue_t queue);
typedef AEEResult (*dspqueue_export_pfn_t)(dspqueue_t queue, uint64_t *queue_id);
typedef AEEResult (*dspqueue_write_pfn_t)(dspqueue_t queue, uint32_t flags,
uint32_t num_buffers,
struct dspqueue_buffer *buffers,
uint32_t message_length,
const uint8_t *message,
uint32_t timeout_us);
typedef AEEResult (*dspqueue_read_pfn_t)(dspqueue_t queue, uint32_t *flags,
uint32_t max_buffers, uint32_t *num_buffers,
struct dspqueue_buffer *buffers,
uint32_t max_message_length,
uint32_t *message_length, uint8_t *message,
uint32_t timeout_us);
typedef AEEResult (*dspqueue_read_noblock_pfn_t)(dspqueue_t queue, uint32_t *flags,
uint32_t max_buffers, uint32_t *num_buffers,
struct dspqueue_buffer *buffers,
uint32_t max_message_length,
uint32_t *message_length, uint8_t *message);
typedef int (*fastrpc_mmap_pfn_t)(int domain, int fd, void *addr, int offset, size_t length, enum fastrpc_map_flags flags);
typedef int (*fastrpc_munmap_pfn_t)(int domain, int fd, void *addr, size_t length);
typedef int (*remote_handle64_open_pfn_t)(const char* name, remote_handle64 *ph);
typedef int (*remote_handle64_invoke_pfn_t)(remote_handle64 h, uint32_t dwScalars, remote_arg *pra);
typedef int (*remote_handle64_close_pfn_t)(remote_handle h);
typedef int (*remote_handle_control_pfn_t)(uint32_t req, void* data, uint32_t datalen);
typedef int (*remote_handle64_control_pfn_t)(remote_handle64 h, uint32_t req, void* data, uint32_t datalen);
typedef int (*remote_session_control_pfn_t)(uint32_t req, void *data, uint32_t datalen);
typedef int (*remote_system_request_pfn_t)(system_req_payload * req);
//
// Driver API pfns
//
rpcmem_alloc_pfn_t rpcmem_alloc_pfn = nullptr;
rpcmem_alloc2_pfn_t rpcmem_alloc2_pfn = nullptr;
rpcmem_free_pfn_t rpcmem_free_pfn = nullptr;
rpcmem_to_fd_pfn_t rpcmem_to_fd_pfn = nullptr;
fastrpc_mmap_pfn_t fastrpc_mmap_pfn = nullptr;
fastrpc_munmap_pfn_t fastrpc_munmap_pfn = nullptr;
dspqueue_create_pfn_t dspqueue_create_pfn = nullptr;
dspqueue_close_pfn_t dspqueue_close_pfn = nullptr;
dspqueue_export_pfn_t dspqueue_export_pfn = nullptr;
dspqueue_write_pfn_t dspqueue_write_pfn = nullptr;
dspqueue_read_pfn_t dspqueue_read_pfn = nullptr;
dspqueue_read_noblock_pfn_t dspqueue_read_noblock_pfn = nullptr;
remote_handle64_open_pfn_t remote_handle64_open_pfn = nullptr;
remote_handle64_invoke_pfn_t remote_handle64_invoke_pfn = nullptr;
remote_handle64_close_pfn_t remote_handle64_close_pfn = nullptr;
remote_handle_control_pfn_t remote_handle_control_pfn = nullptr;
remote_handle64_control_pfn_t remote_handle64_control_pfn = nullptr;
remote_session_control_pfn_t remote_session_control_pfn = nullptr;
remote_system_request_pfn_t remote_system_request_pfn = nullptr;
//
// Driver API
//
void * rpcmem_alloc(int heapid, uint32_t flags, int size) {
return rpcmem_alloc_pfn(heapid, flags, size);
}
void * rpcmem_alloc2(int heapid, uint32_t flags, size_t size) {
if (rpcmem_alloc2_pfn) {
return rpcmem_alloc2_pfn(heapid, flags, size);
} else {
GGML_LOG_INFO("ggml-hex: rpcmem_alloc2 not found, falling back to rpcmem_alloc\n");
return rpcmem_alloc_pfn(heapid, flags, size);
}
}
void rpcmem_free(void * po) {
return rpcmem_free_pfn(po);
}
int rpcmem_to_fd(void * po) {
return rpcmem_to_fd_pfn(po);
}
HTPDRV_API int fastrpc_mmap(int domain, int fd, void * addr, int offset, size_t length, enum fastrpc_map_flags flags) {
return fastrpc_mmap_pfn(domain, fd, addr, offset, length, flags);
}
HTPDRV_API int fastrpc_munmap(int domain, int fd, void * addr, size_t length) {
return fastrpc_munmap_pfn(domain, fd, addr, length);
}
AEEResult dspqueue_create(int domain,
uint32_t flags,
uint32_t req_queue_size,
uint32_t resp_queue_size,
dspqueue_callback_t packet_callback,
dspqueue_callback_t error_callback,
void * callback_context,
dspqueue_t * queue) {
return dspqueue_create_pfn(domain, flags, req_queue_size, resp_queue_size, packet_callback, error_callback,
callback_context, queue);
}
AEEResult dspqueue_close(dspqueue_t queue) {
return dspqueue_close_pfn(queue);
}
AEEResult dspqueue_export(dspqueue_t queue, uint64_t * queue_id) {
return dspqueue_export_pfn(queue, queue_id);
}
AEEResult dspqueue_write(dspqueue_t queue,
uint32_t flags,
uint32_t num_buffers,
struct dspqueue_buffer * buffers,
uint32_t message_length,
const uint8_t * message,
uint32_t timeout_us) {
return dspqueue_write_pfn(queue, flags, num_buffers, buffers, message_length, message, timeout_us);
}
AEEResult dspqueue_read(dspqueue_t queue,
uint32_t * flags,
uint32_t max_buffers,
uint32_t * num_buffers,
struct dspqueue_buffer * buffers,
uint32_t max_message_length,
uint32_t * message_length,
uint8_t * message,
uint32_t timeout_us) {
#ifdef _WIN32
if (timeout_us == 0) {
return dspqueue_read_noblock_pfn(queue, flags, max_buffers, num_buffers, buffers, max_message_length,
message_length, message);
}
#endif
return dspqueue_read_pfn(queue, flags, max_buffers, num_buffers, buffers, max_message_length, message_length,
message, timeout_us);
}
HTPDRV_API int remote_handle64_open(const char * name, remote_handle64 * ph) {
return remote_handle64_open_pfn(name, ph);
}
HTPDRV_API int remote_handle64_invoke(remote_handle64 h, uint32_t dwScalars, remote_arg * pra) {
return remote_handle64_invoke_pfn(h, dwScalars, pra);
}
HTPDRV_API int remote_handle64_close(remote_handle64 h) {
return remote_handle64_close_pfn(h);
}
HTPDRV_API int remote_handle_control(uint32_t req, void * data, uint32_t datalen) {
return remote_handle_control_pfn(req, data, datalen);
}
HTPDRV_API int remote_handle64_control(remote_handle64 h, uint32_t req, void * data, uint32_t datalen) {
return remote_handle64_control_pfn(h, req, data, datalen);
}
HTPDRV_API int remote_session_control(uint32_t req, void * data, uint32_t datalen) {
return remote_session_control_pfn(req, data, datalen);
}
HTPDRV_API int remote_system_request(system_req_payload * req) {
if (!remote_system_request_pfn) {
return AEE_EUNSUPPORTEDAPI;
}
return remote_system_request_pfn(req);
}
#ifdef _WIN32
static std::string wstr_to_str(std::wstring_view wstr) {
std::string result;
if (wstr.empty()) {
return result;
}
auto bytes_needed = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS,
wstr.data(), (int) wstr.size(),
nullptr, 0, nullptr, nullptr);
if (bytes_needed == 0) {
GGML_LOG_ERROR("ggml-hex: WideCharToMultiByte failed. Error %lu\n", GetLastError());
throw std::runtime_error("Invalid wstring input");
}
result.resize(bytes_needed, '\0');
int bytes_written = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS,
wstr.data(), (int) wstr.size(),
result.data(), bytes_needed,
nullptr, nullptr);
if (bytes_written == 0) {
GGML_LOG_ERROR("ggml-hex: WideCharToMultiByte failed. Error %lu\n", GetLastError());
throw std::runtime_error("Wstring conversion failed");
}
return result;
}
static std::string get_driver_path() {
std::wstring serviceName = L"qcnspmcdm";
std::string result;
// Get a handle to the SCM database.
SC_HANDLE schSCManager = OpenSCManagerW(NULL, NULL, STANDARD_RIGHTS_READ);
if (nullptr == schSCManager) {
GGML_LOG_ERROR("ggml-hex: Failed to open SCManager. Error: %lu\n", GetLastError());
return result;
}
// Get a handle to the service.
SC_HANDLE schService = OpenServiceW(schSCManager, // SCM database
serviceName.c_str(), // name of service
SERVICE_QUERY_CONFIG); // need query config access
if (nullptr == schService) {
GGML_LOG_ERROR("ggml-hex: Failed to open qcnspmcdm service. Error: %lu\n", GetLastError());
CloseServiceHandle(schSCManager);
return result;
}
// Store the size of buffer used as an output.
DWORD bufferSize;
if (!QueryServiceConfigW(schService, NULL, 0, &bufferSize) &&
(GetLastError() != ERROR_INSUFFICIENT_BUFFER)) {
GGML_LOG_ERROR("ggml-hex: Failed to query service config. Error: %lu\n", GetLastError());
CloseServiceHandle(schService);
CloseServiceHandle(schSCManager);
return result;
}
// Get the configuration of the service.
LPQUERY_SERVICE_CONFIGW serviceConfig =
static_cast<LPQUERY_SERVICE_CONFIGW>(LocalAlloc(LMEM_FIXED, bufferSize));
if (!QueryServiceConfigW(schService, serviceConfig, bufferSize, &bufferSize)) {
fprintf(stderr, "ggml-hex: Failed to query service config. Error: %lu\n", GetLastError());
LocalFree(serviceConfig);
CloseServiceHandle(schService);
CloseServiceHandle(schSCManager);
return result;
}
// Read the driver file path get its parent directory
std::wstring driverPath = std::wstring(serviceConfig->lpBinaryPathName);
driverPath = driverPath.substr(0, driverPath.find_last_of(L"\\"));
// Clean up resources
LocalFree(serviceConfig);
CloseServiceHandle(schService);
CloseServiceHandle(schSCManager);
// Driver path would contain invalid path string, like:
// \SystemRoot\System32\DriverStore\FileRepository\qcadsprpc8280.inf_arm64_c2b9460c9a072f37
// "\SystemRoot" should be replace with a correct one (e.g. C:\Windows)
const std::wstring systemRootPlaceholder = L"\\SystemRoot";
if (0 != driverPath.compare(0, systemRootPlaceholder.length(), systemRootPlaceholder)) {
GGML_LOG_ERROR("ggml-hex: String pattern not found in driver path.\n");
return result;
}
// Replace \SystemRoot with an absolute path from system ENV windir
const std::wstring systemRootEnv = L"windir";
// Query the number of wide characters this variable requires
DWORD numWords = GetEnvironmentVariableW(systemRootEnv.c_str(), NULL, 0);
if (numWords == 0) {
GGML_LOG_ERROR("ggml-hex: Failed get systemRoot environment variable\n");
return result;
}
// Query the actual system root name from environment variable
std::vector<wchar_t> systemRoot(numWords + 1);
numWords = GetEnvironmentVariableW(systemRootEnv.c_str(), systemRoot.data(), numWords + 1);
if (numWords == 0) {
GGML_LOG_ERROR("ggml-hex: Failed to read windir environment variable\n");
return result;
}
driverPath.replace(0, systemRootPlaceholder.length(), std::wstring(systemRoot.data()));
return wstr_to_str(driverPath);
}
#endif
using dl_handle_ptr = std::unique_ptr<dl_handle, dl_handle_deleter>;
int htpdrv_init() {
static dl_handle_ptr lib_cdsp_rpc_handle = nullptr;
static bool initialized = false;
#ifdef _WIN32
std::string drv_path = get_driver_path() + "\\" + "libcdsprpc.dll";
#else
std::string drv_path = "libcdsprpc.so";
#endif
if (initialized) {
GGML_LOG_INFO("ggml-hex: Driver already loaded\n");
return AEE_SUCCESS;
}
GGML_LOG_INFO("ggml-hex: Loading driver %s\n", drv_path.c_str());
fs::path path{ drv_path.c_str() };
dl_handle_ptr handle { dl_load_library(path) };
if (!handle) {
GGML_LOG_ERROR("ggml-hex: failed to load %s: %s\n", path.u8string().c_str(), dl_error());
return AEE_EUNABLETOLOAD;
}
#define dlsym(drv, type, pfn, symbol, ignore) \
do { \
pfn = (type) dl_get_sym(drv, #symbol); \
if (!ignore && nullptr == pfn) { \
GGML_LOG_ERROR("ggml-hex: failed to dlsym %s\n", #symbol); \
return AEE_EUNABLETOLOAD; \
} \
} while (0)
dlsym(handle.get(), rpcmem_alloc_pfn_t, rpcmem_alloc_pfn, rpcmem_alloc, false);
dlsym(handle.get(), rpcmem_alloc2_pfn_t, rpcmem_alloc2_pfn, rpcmem_alloc2, true);
dlsym(handle.get(), rpcmem_free_pfn_t, rpcmem_free_pfn, rpcmem_free, false);
dlsym(handle.get(), rpcmem_to_fd_pfn_t, rpcmem_to_fd_pfn, rpcmem_to_fd, false);
dlsym(handle.get(), fastrpc_mmap_pfn_t, fastrpc_mmap_pfn, fastrpc_mmap, false);
dlsym(handle.get(), fastrpc_munmap_pfn_t, fastrpc_munmap_pfn, fastrpc_munmap, false);
dlsym(handle.get(), dspqueue_create_pfn_t, dspqueue_create_pfn, dspqueue_create, false);
dlsym(handle.get(), dspqueue_close_pfn_t, dspqueue_close_pfn, dspqueue_close, false);
dlsym(handle.get(), dspqueue_export_pfn_t, dspqueue_export_pfn, dspqueue_export, false);
dlsym(handle.get(), dspqueue_write_pfn_t, dspqueue_write_pfn, dspqueue_write, false);
dlsym(handle.get(), dspqueue_read_pfn_t, dspqueue_read_pfn, dspqueue_read, false);
dlsym(handle.get(), dspqueue_read_noblock_pfn_t, dspqueue_read_noblock_pfn, dspqueue_read_noblock, false);
dlsym(handle.get(), remote_handle64_open_pfn_t, remote_handle64_open_pfn, remote_handle64_open, false);
dlsym(handle.get(), remote_handle64_invoke_pfn_t, remote_handle64_invoke_pfn, remote_handle64_invoke, false);
dlsym(handle.get(), remote_handle_control_pfn_t, remote_handle_control_pfn, remote_handle_control, false);
dlsym(handle.get(), remote_handle64_control_pfn_t, remote_handle64_control_pfn, remote_handle64_control, false);
dlsym(handle.get(), remote_session_control_pfn_t, remote_session_control_pfn, remote_session_control, false);
dlsym(handle.get(), remote_handle64_close_pfn_t, remote_handle64_close_pfn, remote_handle64_close, false);
dlsym(handle.get(), remote_system_request_pfn_t, remote_system_request_pfn, remote_system_request, true);
lib_cdsp_rpc_handle = std::move(handle);
initialized = true;
return AEE_SUCCESS;
}
domain * htpdrv_get_domain(int domain_id) {
int i = 0;
int size = sizeof(supported_domains) / sizeof(domain);
for (i = 0; i < size; i++) {
if (supported_domains[i].id == domain_id) {
return &supported_domains[i];
}
}
return NULL;
}
int htpdrv_get_arch(int domain, int * arch) {
if (!remote_handle_control_pfn) {
GGML_LOG_ERROR("ggml-hex: remote_handle_control is not supported on this device\n");
return AEE_EUNSUPPORTEDAPI;
}
struct remote_dsp_capability arch_ver;
arch_ver.domain = (uint32_t) domain;
arch_ver.attribute_ID = ARCH_VER;
arch_ver.capability = (uint32_t) 0;
int err = remote_handle_control(DSPRPC_GET_DSP_INFO, &arch_ver, sizeof(arch_ver));
if ((err & 0xff) == (AEE_EUNSUPPORTEDAPI & 0xff)) {
GGML_LOG_ERROR("ggml-hex: FastRPC capability API is not supported on this device\n");
return AEE_EUNSUPPORTEDAPI;
}
if (err != AEE_SUCCESS) {
GGML_LOG_ERROR("ggml-hex: FastRPC capability query failed (err %d)\n", err);
return err;
}
uint32_t val = arch_ver.capability & 0xff;
*arch = (int) ((val >> 4) * 10 + (val & 0x0f));
return 0;
}
llama.cpp/ggml/src/ggml-hexagon/htp-drv.h
+123
View File
@@ -0,0 +1,123 @@
#pragma once
#ifdef __cplusplus
extern "C" {
#endif
#ifdef _WIN32
# pragma clang diagnostic ignored "-Wignored-attributes"
#endif
#include <AEEStdErr.h>
#include <rpcmem.h>
#include <remote.h>
#include <dspqueue.h>
#if defined(_WIN32) && !defined(__MINGW32__)
# ifdef GGML_BACKEND_BUILD
# define HTPDRV_API __declspec(dllexport) extern
# else
# define HTPDRV_API __declspec(dllimport) extern
# endif
#else
# define HTPDRV_API __attribute__ ((visibility ("default"))) extern
#endif
/* Offset to differentiate HLOS and Hexagon error codes.
Stores the value of AEE_EOFFSET for Hexagon. */
#ifndef DSP_OFFSET
# define DSP_OFFSET 0x80000400
#endif
/* Errno for connection reset by peer. */
#ifndef ECONNRESET
# ifdef __hexagon__
# define ECONNRESET 104
# endif
#endif
/* Abstraction of different OS specific sleep APIs.
SLEEP accepts input in seconds. */
#ifndef SLEEP
# ifdef __hexagon__
# define SLEEP(x) \
{ /* Do nothing for simulator. */ \
}
# else
# ifdef _WIN32
# define SLEEP(x) Sleep(1000 * x) /* Sleep accepts input in milliseconds. */
# else
# define SLEEP(x) sleep(x) /* sleep accepts input in seconds. */
# endif
# endif
#endif
/* Include windows specific header files. */
#ifdef _WIN32
# include <windows.h>
# include <sysinfoapi.h>
# define _CRT_SECURE_NO_WARNINGS 1
# define _WINSOCK_DEPRECATED_NO_WARNINGS 1
#endif
/* Includes and defines for all HLOS except windows */
#if !defined(__hexagon__) && !defined(_WIN32)
# include "unistd.h"
# include <sys/time.h>
#endif
/* Includes and defines for Hexagon and all HLOS except Windows. */
#if !defined(_WIN32)
/* Weak reference to remote symbol for compilation. */
# pragma weak remote_session_control
# pragma weak remote_handle_control
# pragma weak remote_handle64_control
# pragma weak fastrpc_mmap
# pragma weak fastrpc_munmap
# pragma weak rpcmem_alloc2
#endif
#if !defined(_WIN32)
# pragma weak remote_system_request
#endif
#ifdef _WIN32
# define DSPQUEUE_TIMEOUT DSPQUEUE_TIMEOUT_NONE
#else
# define DSPQUEUE_TIMEOUT 1000000
#endif
/**
* htpdrv_init API: driver interface entry point
*
* @return Return AEE error codes as defined in Hexagon SDK.
*/
HTPDRV_API int htpdrv_init(void);
/**
* htpdrv_get_domain API: get domain struct from domain value.
*
* @param[in] domain value of a domain
* @return Returns domain struct of the domain if it is supported or else
* returns NULL.
*
*/
HTPDRV_API domain * htpdrv_get_domain(int domain_id);
/**
* htpdrv_get_arch API: query the Hexagon processor architecture version information
*
* @param[in] domain_id value of a domain
* @param[out] Arch version (73, 75, ...)
* @return 0 if query is successful.
* non-zero if error, return value points to the error.
*
*/
HTPDRV_API int htpdrv_get_arch(int domain, int * arch);
HTPDRV_API int remote_system_request(system_req_payload * req);
#ifdef __cplusplus
}
#endif
llama.cpp/ggml/src/ggml-hexagon/htp-fusion.cpp
+649
View File
@@ -0,0 +1,649 @@
#include "htp-fusion.h"
#include "htp-kparams.h"
#include "ggml-impl.h"
#include <algorithm>
#include <cstring>
static inline bool htp_same_tensor_data(const ggml_tensor * a, const ggml_tensor * b) {
return a && b && (a == b || a->data == b->data);
}
static inline bool htp_same_shape_and_contiguity(const ggml_tensor * a, const ggml_tensor * b) {
return ggml_are_same_shape(a, b) && (ggml_is_contiguous(a) == ggml_is_contiguous(b));
}
static inline bool htp_inplace_overlap_unsafe(const ggml_tensor * dst, const ggml_tensor * in) {
return htp_tensors_overlap(dst, in) && (dst->data != in->data || !ggml_are_same_stride(dst, in));
}
static inline bool htp_inputs_overlap_unsafe(const htp_opnode & prev, const ggml_tensor * dst, const ggml_tensor * extra = nullptr) {
if (extra && htp_inplace_overlap_unsafe(dst, extra)) return true;
for (const auto * in : prev.inputs) {
if (htp_inplace_overlap_unsafe(dst, in)) return true;
}
return false;
}
static inline bool htp_match_consumer(
const htp_opnode & prev,
const htp_opnode & cur,
bool commutative,
const ggml_tensor ** other_src = nullptr
) {
if (prev.outputs.size() != 1) {
return false;
}
const ggml_tensor * prev_out = prev.dst();
if (!htp_tensor_is_fuseable(prev_out)) {
return false;
}
if (htp_same_tensor_data(cur.src0(), prev_out)) {
if (other_src) *other_src = cur.src1();
return true;
}
if (commutative && htp_same_tensor_data(cur.src1(), prev_out)) {
if (other_src) *other_src = cur.src0();
return true;
}
return false;
}
static bool is_supported_mul_mat_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {
if (kparams->n_hmx) {
return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
}
return htp_tensor_is_tiled(src0) && (kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW);
}
static bool is_supported_mul_mat_id_nx_kernel(const ggml_tensor * src0, const struct htp_mm_kernel_params * kparams) {
if (kparams->n_hmx) {
return kparams->kernel_type == HTP_MM_KERNEL_HMX_2D;
}
return htp_tensor_is_tiled(src0) &&
(kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK);
}
// --- Fusion rules ---
static bool fuse_allreduce_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
if (opt_ar_select != 2 || prev.opcode != HTP_OP_ALLREDUCE) return false;
const auto * ar_kparams = prev.kparams<htp_allreduce_kernel_params>();
const uint32_t rank = (uint32_t) ar_kparams->rank;
const uint32_t n_ranks = (uint32_t) ar_kparams->n_ranks;
const ggml_tensor * ar_local = prev.inputs[rank];
if (!htp_tensor_is_fuseable(ar_local)) return false;
const ggml_tensor * add_dst = cur.dst();
const ggml_tensor * res_tensor = nullptr;
if (htp_same_tensor_data(cur.src0(), ar_local)) {
res_tensor = cur.src1();
} else if (htp_same_tensor_data(cur.src1(), ar_local)) {
res_tensor = cur.src0();
} else {
return false;
}
if (ar_local->type != res_tensor->type) return false;
const bool is_same_shape = ggml_are_same_shape(ar_local, res_tensor);
const bool is_row_bcast = !is_same_shape && (ar_local->ne[0] == res_tensor->ne[0] &&
res_tensor->ne[1] == 1 && res_tensor->ne[2] == 1 && res_tensor->ne[3] == 1);
if (!is_same_shape && !is_row_bcast) return false;
if (is_same_shape && !ggml_are_same_stride(ar_local, res_tensor)) {
return false;
}
if (ggml_is_contiguous(ar_local) != ggml_is_contiguous(add_dst)) return false;
const bool is_shard_ok = n_ranks <= HTP_OP_MAX_OUTPUTS && add_dst->data == ar_local->data;
struct htp_allreduce_kernel_params kparams;
if (!htp_kparams_solve_allreduce(sess, add_dst, rank, n_ranks, true, is_row_bcast, is_shard_ok, &kparams)) {
HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: solver failed\n", htp_session_name(sess));
return false;
}
const bool scatter_ok = kparams.n_dsts > 1;
kparams.mode = scatter_ok ? HTP_ALLREDUCE_SHARDED_FANOUT : HTP_ALLREDUCE_FULL;
for (uint32_t r = 0; r < n_ranks; r++) {
const ggml_tensor * ar_src = prev.inputs[r];
if (!htp_tensors_overlap(add_dst, ar_src)) continue;
if (scatter_ok && r == rank) continue;
HEX_VERBOSE("ggml-hex: %s skip ALLREDUCE_ADD fusion: dst overlaps allreduce src %u\n", htp_session_name(sess), r);
return false;
}
prev.opcode = HTP_OP_ALLREDUCE_ADD;
prev.inputs.push_back(res_tensor);
if (kparams.mode != HTP_ALLREDUCE_SHARDED_FANOUT) {
prev.outputs = { add_dst };
}
prev.fused.push_back(cur.node);
prev.set_kparams(kparams);
return true;
}
static bool fuse_rms_norm_mul(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
if (prev.opcode != HTP_OP_RMS_NORM) return false;
const ggml_tensor * weight = nullptr;
if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &weight)) return false;
const ggml_tensor * src0 = prev.src0();
const ggml_tensor * dst = cur.dst();
if (src0->type != GGML_TYPE_F32 || src0->ne[0] != weight->ne[0] || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false;
if (htp_inputs_overlap_unsafe(prev, dst, weight)) return false;
const bool is_row_bcast = (weight->ne[1] == 1 && weight->ne[2] == 1 && weight->ne[3] == 1);
const bool is_same_shape = ggml_are_same_shape(src0, weight);
if (!is_row_bcast && !is_same_shape) return false;
struct htp_unary_kernel_params kparams;
if (!htp_kparams_solve_rms_norm_mul(sess, src0, weight, dst, &kparams)) {
HEX_VERBOSE("ggml-hex: %s skip RMS_NORM_MUL fusion: VTCM needed (%d) > budget (%zu)\n",
htp_session_name(sess), kparams.vtcm_size, htp_session_vtcm_size(sess));
return false;
}
prev.opcode = HTP_OP_RMS_NORM_MUL;
prev.inputs = { src0, weight };
prev.outputs = { dst };
prev.fused.push_back(cur.node);
prev.set_kparams(kparams);
return true;
}
static bool fuse_rms_norm_scale(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
GGML_UNUSED(sess);
if (prev.opcode != HTP_OP_RMS_NORM && prev.opcode != HTP_OP_RMS_NORM_MUL) return false;
if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false;
const ggml_tensor * src0 = prev.src0();
const ggml_tensor * dst = cur.dst();
if (src0->type != GGML_TYPE_F32 || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false;
if (htp_inputs_overlap_unsafe(prev, dst)) return false;
float scale = 0.0f;
float bias = 0.0f;
memcpy(&scale, (const float *) cur.params + 0, sizeof(float));
memcpy(&bias, (const float *) cur.params + 1, sizeof(float));
if (bias != 0.0f || scale == 0.0f) return false;
float prev_scale = 0.0f;
memcpy(&prev_scale, &prev.params[1], sizeof(float));
if (prev_scale != 0.0f) {
scale *= prev_scale;
}
memcpy(&prev.params[1], &scale, sizeof(float));
prev.outputs[0] = dst;
prev.fused.push_back(cur.node);
return true;
}
static bool fuse_mul_mat_add(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
if (prev.opcode != HTP_OP_MUL_MAT) return false;
const ggml_tensor * src2 = nullptr;
if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &src2)) return false;
if (src2->type != GGML_TYPE_F32) return false;
const ggml_tensor * src0 = prev.node->src[0];
const ggml_tensor * src1 = prev.node->src[1];
const ggml_tensor * dst = cur.node;
if (!ggml_are_same_shape(prev.node, dst)) return false;
const bool prev_collapsed = htp_matmul_can_collapse(src0, src1, prev.node);
const bool cur_collapsed = htp_matmul_add_can_collapse(src0, src1, src2, dst);
if (prev_collapsed != cur_collapsed) {
return false;
}
const auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
struct htp_mm_kernel_params kparams;
if (!htp_kparams_solve_matmul_add(sess, src0, src1, src2, dst, &kparams)) {
return false;
}
const int src1_nrows = src1->ne[1] * src1->ne[2] * src1->ne[3];
if (kparams.n_hmx == 0 && src1_nrows != 1) return false;
if (kparams.n_hmx > 0 && orig_kparams->n_hmx > 0) {
if (kparams.m_chunk < orig_kparams->m_chunk ||
kparams.n_chunk < orig_kparams->n_chunk ||
kparams.n_act_threads < orig_kparams->n_act_threads) {
HEX_VERBOSE("ggml-hex: %s skip MUL_MAT_ADD fusion: HMX efficiency reduced (m %d->%d, n %d->%d, th %d->%d)\n",
htp_session_name(sess), orig_kparams->m_chunk, kparams.m_chunk,
orig_kparams->n_chunk, kparams.n_chunk,
orig_kparams->n_act_threads, kparams.n_act_threads);
return false;
}
}
prev.opcode = HTP_OP_MUL_MAT_ADD;
prev.inputs = { src0, src1, src2 };
prev.outputs = { dst };
htp_opnode_collapse(sess, prev);
prev.fused.push_back(cur.node);
prev.set_kparams(kparams);
return true;
}
static bool fuse_mul_mat_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
if (prev.opcode != HTP_OP_MUL_MAT && prev.opcode != HTP_OP_MUL_MAT_NX) return false;
const uint32_t curr_n = (uint32_t) prev.outputs.size();
if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 1 >= HTP_OP_MAX_INPUTS) return false;
const ggml_tensor * w0 = prev.inputs[0];
const ggml_tensor * x_orig = prev.node->src[1];
const ggml_tensor * w_in = cur.node->src[0];
const ggml_tensor * x_in = cur.node->src[1];
const ggml_tensor * d_in = cur.node;
auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>();
if (x_in != x_orig || w_in->type != w0->type || w_in->ne[0] != w0->ne[0] ||
w0->ne[2] != 1 || w0->ne[3] != 1 || w_in->ne[2] != 1 || w_in->ne[3] != 1 ||
htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
orig_kparams->n_hmx != cur_kparams->n_hmx) {
return false;
}
const bool collapse = htp_matmul_can_collapse(w0, x_orig, prev.node) && htp_matmul_can_collapse(w_in, x_in, d_in);
if ((x_orig->ne[2] > 1 || x_orig->ne[3] > 1) && !collapse) return false;
if (prev.opcode == HTP_OP_MUL_MAT_NX) {
orig_kparams->n_weights = curr_n + 1;
} else {
struct htp_mm_kernel_params kparams;
const ggml_tensor x_collapsed = collapse ? htp_tensor_collapse_rows(x_orig) : *x_orig;
if (!htp_kparams_solve_mmnx(sess, w0, &x_collapsed, curr_n + 1, &kparams)) return false;
if (!is_supported_mul_mat_nx_kernel(w0, &kparams)) return false;
prev.set_kparams(kparams);
prev.opcode = HTP_OP_MUL_MAT_NX;
}
prev.inputs.insert(prev.inputs.begin() + curr_n, w_in);
prev.outputs.push_back(collapse ? prev.add_dummy(htp_tensor_collapse_rows(d_in)) : d_in);
prev.fused.push_back(cur.node);
return true;
}
static bool fuse_mul_mat_id_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
if (prev.opcode != HTP_OP_MUL_MAT_ID && prev.opcode != HTP_OP_MUL_MAT_ID_NX) return false;
const uint32_t curr_n = (uint32_t) prev.outputs.size();
if (curr_n >= HTP_OP_MAX_OUTPUTS || curr_n + 2 >= HTP_OP_MAX_INPUTS) return false;
const ggml_tensor * w0 = prev.inputs[0];
const ggml_tensor * x = prev.inputs[curr_n];
const ggml_tensor * ids = prev.inputs[curr_n + 1];
const ggml_tensor * w_in = cur.src0();
const ggml_tensor * x_in = cur.src1();
const ggml_tensor * ids_in = cur.node->src[2];
const ggml_tensor * d_in = cur.dst();
auto * orig_kparams = prev.kparams<htp_mm_kernel_params>();
const auto * cur_kparams = cur.kparams<htp_mm_kernel_params>();
if (x_in != x || ids_in != ids || w_in->type != w0->type ||
w_in->ne[0] != w0->ne[0] || w_in->ne[2] != w0->ne[2] ||
htp_tensor_is_tiled(w_in) != htp_tensor_is_tiled(w0) ||
orig_kparams->n_hmx != cur_kparams->n_hmx) {
return false;
}
struct htp_mm_kernel_params kparams;
if (d_in->nb[1] == prev.node->nb[1]) {
kparams = *orig_kparams;
kparams.n_weights = curr_n + 1;
} else if (!htp_kparams_solve_mmidnx(sess, w0, x, d_in, curr_n + 1, &kparams)) {
return false;
}
if (!is_supported_mul_mat_id_nx_kernel(w0, &kparams)) return false;
prev.opcode = HTP_OP_MUL_MAT_ID_NX;
prev.inputs.insert(prev.inputs.begin() + curr_n, w_in);
prev.outputs.push_back(d_in);
prev.fused.push_back(cur.node);
prev.set_kparams(kparams);
return true;
}
static bool fuse_gdn_cpy(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
GGML_UNUSED(sess);
if (prev.opcode != HTP_OP_GATED_DELTA_NET || prev.outputs.size() != 1) return false;
const ggml_tensor * gdn_out = prev.dst();
const ggml_tensor * cpy_node = cur.node;
const ggml_tensor * cpy_src = cur.src0();
const ggml_tensor * cpy_dst = cur.dst();
if (gdn_out->type != GGML_TYPE_F32 || cpy_src->type != GGML_TYPE_F32 || cpy_dst->type != GGML_TYPE_F32) return false;
if ((gdn_out->flags & GGML_TENSOR_FLAG_OUTPUT) || (cpy_node->flags & GGML_TENSOR_FLAG_OUTPUT)) return false;
const ggml_tensor * v = prev.node->src[2];
if (!v) return false;
const int64_t S_v = v->ne[0];
const int64_t H = v->ne[1];
const int64_t n_tokens = v->ne[2];
const int64_t n_seqs = v->ne[3];
const int64_t K = ggml_get_op_params_i32(prev.node, 0);
const size_t tail_off = (size_t) S_v * H * n_tokens * n_seqs * sizeof(float);
const int64_t D = S_v * S_v * H;
const int64_t n_written = std::min<int64_t>(n_tokens, K);
if (cpy_src->op != GGML_OP_VIEW || !htp_same_tensor_data(cpy_src->view_src, gdn_out) ||
cpy_src->view_offs != tail_off || !ggml_is_contiguous(cpy_src)) {
return false;
}
if (cpy_dst->ne[0] != D || cpy_dst->ne[1] != n_seqs || cpy_dst->ne[2] != n_written || !ggml_is_contiguous(cpy_dst)) {
return false;
}
prev.outputs.push_back(cpy_dst);
prev.fused.push_back(cur.node);
return true;
}
static bool fuse_ssm_conv_silu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
GGML_UNUSED(sess);
if (prev.opcode != HTP_OP_SSM_CONV) return false;
auto * kparams = prev.kparams<htp_ssm_conv_kernel_params>();
if (kparams->silu) return false;
if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false;
const ggml_tensor * conv_dst = prev.dst();
const ggml_tensor * dst = cur.dst();
if (!ggml_are_same_layout(conv_dst, dst)) return false;
if (htp_tensors_overlap(dst, prev.src0()) || htp_tensors_overlap(dst, prev.src1())) return false;
kparams->silu = 1;
prev.outputs[0] = dst;
prev.fused.push_back(cur.node);
return true;
}
static bool fuse_repeat_get_rows(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
if (prev.opcode != HTP_OP_REPEAT) return false;
if (!htp_match_consumer(prev, cur, /*commutative=*/ false)) return false;
const ggml_tensor * repeat_out = prev.dst();
const ggml_tensor * repeat_src0 = prev.src0();
const ggml_tensor * gr_src1 = cur.src1();
const ggml_tensor * gr_dst = cur.dst();
if (repeat_src0->ne[0] != repeat_out->ne[0] || repeat_src0->ne[1] != repeat_out->ne[1]) return false;
if (repeat_src0->type != repeat_out->type) return false;
struct htp_get_rows_kernel_params kparams;
if (!htp_kparams_solve_repeat_get_rows(sess, repeat_src0, gr_src1, gr_dst, &kparams) || kparams.vtcm_size == 0) return false;
prev.opcode = HTP_OP_GET_ROWS;
prev.inputs = { repeat_src0, gr_src1 };
prev.outputs = { gr_dst };
prev.fused.push_back(cur.node);
prev.set_kparams(kparams);
return true;
}
static bool fuse_unary_mul_glu(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
htp_op_code glu_op = HTP_OP_INVALID;
switch (prev.opcode) {
case HTP_OP_UNARY_SILU: glu_op = HTP_OP_GLU_SWIGLU; break;
case HTP_OP_UNARY_GELU: glu_op = HTP_OP_GLU_GEGLU; break;
case HTP_OP_UNARY_GELU_QUICK: glu_op = HTP_OP_GLU_GEGLU_QUICK; break;
case HTP_OP_UNARY_GELU_ERF: glu_op = HTP_OP_GLU_GEGLU_ERF; break;
case HTP_OP_UNARY_RELU: glu_op = HTP_OP_GLU_REGLU; break;
case HTP_OP_UNARY_SIGMOID: glu_op = HTP_OP_GLU_SIGMOID; break;
default: return false;
}
const ggml_tensor * up = nullptr;
if (!htp_match_consumer(prev, cur, /*commutative=*/ true, &up)) return false;
const ggml_tensor * gate = prev.src0();
const ggml_tensor * dst = cur.dst();
if (htp_inputs_overlap_unsafe(prev, dst, up)) return false;
prev.opcode = glu_op;
prev.inputs = { gate, up };
prev.outputs = { dst };
if (!htp_kparams_solve(sess, prev)) return false;
prev.fused.push_back(cur.node);
memset(prev.params, 0, sizeof(prev.params));
return true;
}
static bool fuse_mul_mul_scalar(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
if (prev.opcode != HTP_OP_MUL || prev.inputs.size() != 2) return false;
const auto * k0 = prev.kparams<htp_binary_kernel_params>();
const auto * k1 = cur.kparams<htp_binary_kernel_params>();
if (k0->kernel_type != HTP_BINARY_KERNEL_SCALAR || k1->kernel_type != HTP_BINARY_KERNEL_SCALAR) return false;
if (k0->src1_strided || k1->src1_strided) return false;
const ggml_tensor * src2 = nullptr;
if (!htp_match_consumer(prev, cur, /*commutative=*/ false, &src2)) return false;
const ggml_tensor * src1 = prev.src1();
const ggml_tensor * dst = cur.dst();
if (!ggml_are_same_shape(src1, src2) || !htp_same_shape_and_contiguity(prev.dst(), dst)) return false;
if (htp_inputs_overlap_unsafe(prev, dst, src2)) return false;
const size_t vtcm_size = htp_session_vtcm_size(sess);
struct htp_binary_kernel_params kparams = *k0;
kparams.src2_size = kparams.src1_size;
kparams.rows_per_buffer = 0;
if (kparams.src1_size > 65536 || kparams.src1_size + kparams.src2_size > vtcm_size) return false;
struct htp_binary_vtcm_layout L;
htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size);
if (L.rows_per_buffer == 0 || L.total_bytes > vtcm_size) return false;
kparams.rows_per_buffer = L.rows_per_buffer;
kparams.vtcm_size = L.total_bytes;
prev.inputs.push_back(src2);
prev.outputs[0] = dst;
prev.fused.push_back(cur.node);
prev.set_kparams(kparams);
return true;
}
static bool fuse_binary_nx(const ggml_hexagon_session * sess, htp_opnode & prev, const htp_opnode & cur) {
if (prev.opcode != cur.opcode) return false;
const uint32_t curr_n = (uint32_t) prev.inputs.size();
if (curr_n < 2 || curr_n >= HTP_OP_MAX_INPUTS) return false;
const auto * k0 = prev.kparams<htp_binary_kernel_params>();
const auto * k1 = cur.kparams<htp_binary_kernel_params>();
const bool k0_ok = (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ||
(k0->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k0->is_scalar);
const bool k1_ok = (k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) ||
(k1->kernel_type == HTP_BINARY_KERNEL_CHUNKED && !k1->is_scalar);
if (!k0_ok || !k1_ok) return false;
const bool is_comm = (cur.opcode == HTP_OP_ADD || cur.opcode == HTP_OP_MUL);
const ggml_tensor * new_src = nullptr;
if (!htp_match_consumer(prev, cur, is_comm, &new_src)) return false;
const ggml_tensor * src0 = prev.inputs[0];
const ggml_tensor * dst = cur.dst();
if (!ggml_are_same_shape(src0, prev.inputs[1]) ||
!ggml_are_same_shape(src0, new_src) ||
!htp_same_shape_and_contiguity(prev.dst(), dst)) {
return false;
}
if (htp_inputs_overlap_unsafe(prev, dst, new_src)) return false;
const size_t vtcm_size = htp_session_vtcm_size(sess);
struct htp_binary_kernel_params kparams = *k0;
struct htp_binary_vtcm_layout L;
memset(&L, 0, sizeof(L));
const uint32_t n_srcs = curr_n + 1;
if (k0->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE || k1->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE) {
if (k0->kernel_type != HTP_BINARY_KERNEL_SAME_SHAPE) {
kparams = *k1;
}
kparams.n_srcs = (uint8_t) n_srcs;
kparams.rows_per_buffer = 0;
auto check_split = [&](const ggml_tensor * t) {
if (src0->ne[2] > 1 && t->nb[2] != t->ne[1] * t->nb[1]) kparams.split_at_ne01 = 1;
if (src0->ne[3] > 1 && t->nb[3] != t->ne[2] * t->nb[2]) kparams.split_at_ne02 = 1;
};
for (const auto * in : prev.inputs) check_split(in);
check_split(new_src);
check_split(dst);
htp_binary_vtcm_layout_build(&L, &kparams, vtcm_size);
}
if (L.rows_per_buffer == 0 || L.total_bytes == 0 || L.total_bytes > vtcm_size) {
bool all_contig = ggml_is_contiguous(new_src) && ggml_is_contiguous(dst);
for (const auto * in : prev.inputs) {
all_contig = all_contig && ggml_is_contiguous(in);
}
if (!all_contig) return false;
const uint32_t total_elems = (uint32_t) ggml_nelements(src0);
const uint32_t elem_size = (uint32_t) ggml_type_size(src0->type);
const uint32_t max_threads = htp_session_n_threads(sess);
if (!htp_binary_solve_chunked(&L, &kparams, total_elems, elem_size, max_threads, n_srcs, false, vtcm_size)) {
return false;
}
}
kparams.rows_per_buffer = L.rows_per_buffer;
kparams.vtcm_size = (uint32_t) L.total_bytes;
prev.inputs.push_back(new_src);
prev.outputs[0] = dst;
prev.fused.push_back(cur.node);
prev.set_kparams(kparams);
return true;
}
bool htp_fusion_can_stack(const struct ggml_tensor * a) {
if (!a || !a->src[0] || !a->src[1]) return false;
switch (a->op) {
case GGML_OP_MUL_MAT:
if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX)) return false;
if (htp_is_hadamard(a)) return false;
if (a->src[0]->ne[2] != 1 || a->src[0]->ne[3] != 1) return false;
return ggml_is_quantized(a->src[0]->type);
case GGML_OP_MUL_MAT_ID:
if (!htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX)) return false;
return a->src[2] != nullptr && ggml_is_quantized(a->src[0]->type);
default:
return false;
}
}
bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b) {
if (!htp_fusion_can_stack(b) || a->op != b->op || a->src[1] != b->src[1]) {
return false;
}
if (a->op == GGML_OP_MUL_MAT_ID && a->src[2] != b->src[2]) {
return false;
}
const ggml_tensor * w0 = a->src[0];
const ggml_tensor * w1 = b->src[0];
return w0->type == w1->type && w0->ne[0] == w1->ne[0] && w0->ne[2] == w1->ne[2];
}
int htp_fusion_max(const struct ggml_cgraph * graph, int idx) {
const int n = graph->n_nodes;
const ggml_op op0 = graph->nodes[idx]->op;
if (op0 != GGML_OP_ADD && op0 != GGML_OP_NORM && op0 != GGML_OP_RMS_NORM) {
return 1;
}
enum ggml_op ops[HTP_OP_MAX_FUSED];
ops[0] = op0;
int f = idx + 1;
while (f < n && f < idx + HTP_OP_MAX_FUSED) {
const ggml_op op = graph->nodes[f]->op;
if (op != GGML_OP_ADD && op != GGML_OP_MUL && op != GGML_OP_SCALE &&
op != GGML_OP_NORM && op != GGML_OP_RMS_NORM) {
break;
}
ops[f - idx] = op;
f++;
}
f -= idx;
for (; f > 1; f--) {
if (ggml_can_fuse(graph, idx, ops, f)) {
break;
}
}
return f;
}
void htp_fusion_tag_graph(struct ggml_cgraph * graph) {
for (int i = 0; i < graph->n_nodes; i++) {
auto * extra = (htp_tensor_extra *) graph->nodes[i]->extra;
if (!extra) continue;
extra->flags &= ~HTP_TENSOR_FUSEABLE;
const ggml_op op = graph->nodes[i]->op;
if ((op == GGML_OP_RMS_NORM || op == GGML_OP_SCALE || op == GGML_OP_REPEAT ||
op == GGML_OP_UNARY || op == GGML_OP_ADD || op == GGML_OP_SUB ||
op == GGML_OP_MUL || op == GGML_OP_DIV || op == GGML_OP_MUL_MAT_ID ||
op == GGML_OP_SSM_CONV ||
(op == GGML_OP_MUL_MAT && !htp_is_hadamard(graph->nodes[i]))) &&
ggml_node_has_n_uses(graph, i, 1)) {
extra->flags |= HTP_TENSOR_FUSEABLE;
}
}
}
bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur) {
if (!opt_opfusion) return false;
switch (cur.opcode) {
case HTP_OP_ADD:
return (htp_fusion_enabled(HTP_FUSE_ALLREDUCE_ADD) && fuse_allreduce_add(sess, prev, cur)) ||
(htp_fusion_enabled(HTP_FUSE_MUL_MAT_ADD) && fuse_mul_mat_add(sess, prev, cur)) ||
(htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur));
case HTP_OP_MUL:
return (htp_fusion_enabled(HTP_FUSE_RMS_NORM_MUL) && fuse_rms_norm_mul(sess, prev, cur)) ||
(htp_fusion_enabled(HTP_FUSE_UNARY_MUL_GLU) && fuse_unary_mul_glu(sess, prev, cur)) ||
(htp_fusion_enabled(HTP_FUSE_MUL_MUL_SCALAR) && fuse_mul_mul_scalar(sess, prev, cur)) ||
(htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur));
case HTP_OP_SUB:
case HTP_OP_DIV:
return htp_fusion_enabled(HTP_FUSE_BINARY_NX) && fuse_binary_nx(sess, prev, cur);
case HTP_OP_SCALE:
return htp_fusion_enabled(HTP_FUSE_RMS_NORM_SCALE) && fuse_rms_norm_scale(sess, prev, cur);
case HTP_OP_UNARY_SILU:
return htp_fusion_enabled(HTP_FUSE_SSM_CONV_SILU) && fuse_ssm_conv_silu(sess, prev, cur);
case HTP_OP_GET_ROWS:
return htp_fusion_enabled(HTP_FUSE_REPEAT_GET_ROWS) && fuse_repeat_get_rows(sess, prev, cur);
case HTP_OP_CPY:
return htp_fusion_enabled(HTP_FUSE_GDN_CPY) && fuse_gdn_cpy(sess, prev, cur);
case HTP_OP_MUL_MAT:
return htp_fusion_enabled(HTP_FUSE_MUL_MAT_NX) && fuse_mul_mat_nx(sess, prev, cur);
case HTP_OP_MUL_MAT_ID:
return htp_fusion_enabled(HTP_FUSE_MUL_MAT_ID_NX) && fuse_mul_mat_id_nx(sess, prev, cur);
default:
return false;
}
}
llama.cpp/ggml/src/ggml-hexagon/htp-fusion.h
+33
View File
@@ -0,0 +1,33 @@
#ifndef HTP_FUSION_H
#define HTP_FUSION_H
#include "htp-common.h"
enum htp_fusion_flags {
HTP_FUSE_ALLREDUCE_ADD = (1 << 1), // 2
HTP_FUSE_RMS_NORM_MUL = (1 << 2), // 4
HTP_FUSE_MUL_MAT_ADD = (1 << 3), // 8
HTP_FUSE_MUL_MAT_NX = (1 << 4), // 16
HTP_FUSE_MUL_MAT_ID_NX = (1 << 5), // 32
HTP_FUSE_GDN_CPY = (1 << 6), // 64
HTP_FUSE_REPEAT_GET_ROWS = (1 << 7), // 128
HTP_FUSE_UNARY_MUL_GLU = (1 << 8), // 256
HTP_FUSE_MUL_MUL_SCALAR = (1 << 9), // 512
HTP_FUSE_BINARY_NX = (1 << 10), // 1024
HTP_FUSE_RMS_NORM_SCALE = (1 << 11), // 2048
HTP_FUSE_SSM_CONV_SILU = (1 << 12), // 4096
};
static inline bool htp_fusion_enabled(int flag) {
if (opt_opfusion <= 0) return false;
if (opt_opfusion == 1) return true; // 1 enables all
return (opt_opfusion & flag) != 0;
}
bool htp_fusion_can_stack(const struct ggml_tensor * a);
bool htp_fusion_can_stack(const struct ggml_tensor * a, const struct ggml_tensor * b);
int htp_fusion_max(const struct ggml_cgraph * graph, int idx);
void htp_fusion_tag_graph(struct ggml_cgraph * graph);
bool htp_fusion_try_fuse(const struct ggml_hexagon_session * sess, struct htp_opnode & prev, const struct htp_opnode & cur);
#endif // HTP_FUSION_H
llama.cpp/ggml/src/ggml-hexagon/htp-kparams.cpp
+3514
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp-kparams.h
+136
View File
@@ -0,0 +1,136 @@
#ifndef HTP_KPARAMS_H
#define HTP_KPARAMS_H
#include "htp-common.h"
#include "htp/allreduce-ops.h"
#include "htp/argsort-ops.h"
#include "htp/binary-ops.h"
#include "htp/concat-ops.h"
#include "htp/cpy-ops.h"
#include "htp/cumsum-ops.h"
#include "htp/diag-ops.h"
#include "htp/fill-ops.h"
#include "htp/flash-attn-ops.h"
#include "htp/gated-delta-net-ops.h"
#include "htp/get-rows-ops.h"
#include "htp/glu-ops.h"
#include "htp/im2col-ops.h"
#include "htp/matmul-ops.h"
#include "htp/pad-ops.h"
#include "htp/pool-ops.h"
#include "htp/repeat-ops.h"
#include "htp/roll-ops.h"
#include "htp/rope-ops.h"
#include "htp/set-rows-ops.h"
#include "htp/softmax-ops.h"
#include "htp/solve-tri-ops.h"
#include "htp/ssm-conv.h"
#include "htp/sum-rows-ops.h"
#include "htp/unary-ops.h"
#include <type_traits>
#include <utility>
bool htp_rows_stride(const int64_t * ne, const size_t * nb, size_t * stride);
bool htp_matmul_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * dst);
bool htp_matmul_add_can_collapse(const struct ggml_tensor * src0, const struct ggml_tensor * src1, const struct ggml_tensor * src2, const struct ggml_tensor * dst);
ggml_tensor htp_tensor_collapse_rows(const struct ggml_tensor * t);
template <typename... Args>
auto htp_kparams_type(bool (*)(Args...)) -> std::remove_pointer_t<std::remove_reference_t<decltype((std::declval<Args>(), ...))>>;
template <auto SolveFn, typename... Args>
static inline bool htp_kparams_check(Args &&... args) {
decltype(htp_kparams_type(SolveFn)) kparams;
return SolveFn(std::forward<Args>(args)..., &kparams);
}
struct htp_opnode;
bool htp_kparams_solve(
const struct ggml_hexagon_session * sess,
struct htp_opnode & node
);
void htp_opnode_collapse(
const struct ggml_hexagon_session * sess,
struct htp_opnode & node
);
bool htp_kparams_solve(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
void * kparams = nullptr
);
static inline bool htp_kparams_check(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op
) {
return htp_kparams_solve(sess, op, nullptr);
}
bool htp_kparams_solve_allreduce(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * dst,
uint32_t rank,
uint32_t n_ranks,
bool has_add,
bool is_row_bcast,
bool is_shard_ok,
struct htp_allreduce_kernel_params * kparams
);
bool htp_kparams_solve_rms_norm_mul(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * dst,
struct htp_unary_kernel_params * kparams
);
bool htp_kparams_solve_matmul_add(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * src2,
const struct ggml_tensor * dst,
struct htp_mm_kernel_params * kparams
);
bool htp_kparams_solve_mmnx(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
const struct ggml_tensor * act,
int32_t n_weights,
struct htp_mm_kernel_params * kparams
);
bool htp_kparams_solve_mmidnx(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
const struct ggml_tensor * act,
const struct ggml_tensor * dst,
int32_t n_weights,
struct htp_mm_kernel_params * kparams
);
bool htp_kparams_solve_repeat_get_rows(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * dst,
struct htp_get_rows_kernel_params * kparams
);
bool htp_kparams_solve_unary_mul_glu(
const struct ggml_hexagon_session * sess,
uint32_t op,
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * dst,
struct htp_glu_kernel_params * kparams
);
#endif // HTP_KPARAMS_H
llama.cpp/ggml/src/ggml-hexagon/htp-opnode.cpp
+260
View File
@@ -0,0 +1,260 @@
#include "htp-opnode.h"
#include "htp/allreduce-ops.h"
#include "htp/argsort-ops.h"
#include "htp/binary-ops.h"
#include "htp/cumsum-ops.h"
#include "htp/diag-ops.h"
#include "htp/fill-ops.h"
#include "htp/flash-attn-ops.h"
#include "htp/gated-delta-net-ops.h"
#include "htp/get-rows-ops.h"
#include "htp/glu-ops.h"
#include "htp/matmul-ops.h"
#include "htp/pad-ops.h"
#include "htp/repeat-ops.h"
#include "htp/roll-ops.h"
#include "htp/softmax-ops.h"
#include "htp/solve-tri-ops.h"
#include "htp/ssm-conv.h"
#include "htp/sum-rows-ops.h"
#include "htp/unary-ops.h"
#include <stdio.h>
std::string htp_opnode::name() const {
switch (opcode) {
case HTP_OP_MDEV_GROUP: return "MDEV_GROUP";
case HTP_OP_CPY_FENCE: return "CPY+FENCE";
case HTP_OP_FENCE: return (node && node->op_params[1] == 0) ? "FENCE_WAIT" : "FENCE_SIGNAL";
case HTP_OP_ALLREDUCE: return "ALLREDUCE";
case HTP_OP_ALLREDUCE_ADD: return "ALLREDUCE+ADD";
case HTP_OP_HADAMARD: return "MUL_MAT_HADAMARD";
case HTP_OP_MUL_MAT_NX: return "MUL_MAT_NX";
case HTP_OP_MUL_MAT_ID_NX: return "MUL_MAT_ID_NX";
case HTP_OP_ADD:
case HTP_OP_SUB:
case HTP_OP_MUL:
case HTP_OP_DIV:
if (!fused.empty()) {
const auto * kp = kparams<htp_binary_kernel_params>();
if (kp->kernel_type != HTP_BINARY_KERNEL_SCALAR) {
switch (opcode) {
case HTP_OP_ADD: return "ADD_NX";
case HTP_OP_SUB: return "SUB_NX";
case HTP_OP_MUL: return "MUL_NX";
case HTP_OP_DIV: return "DIV_NX";
default: break;
}
}
}
break;
default:
break;
}
if (!node) {
return "";
}
if (htp_is_hadamard(node)) {
return "MUL_MAT_HADAMARD";
}
std::string res = ggml_op_desc(node);
for (const auto * t : fused) {
res += "+";
res += ggml_op_desc(t);
}
return res;
}
static const char * format_tensor_name(const struct ggml_tensor * t) {
return t ? t->name : "NONE";
}
static std::string format_tensor_dims(const struct ggml_tensor * t) {
if (!t) {
return "NONE";
}
char buf[64];
if (t->ne[2] == 1 && t->ne[3] == 1) {
snprintf(buf, sizeof(buf), "%d:%d", (int) t->ne[0], (int) t->ne[1]);
} else {
snprintf(buf, sizeof(buf), "%d:%d:%d:%d", (int) t->ne[0], (int) t->ne[1], (int) t->ne[2], (int) t->ne[3]);
}
return buf;
}
static const char * format_tensor_type(const struct ggml_tensor * t) {
return t ? ggml_type_name(t->type) : "NONE";
}
static std::string format_tensor_strides(const struct ggml_tensor * t) {
if (!t) {
return "NONE";
}
char buf[64];
const char * c = ggml_is_contiguous(t) ? "" : "!";
if (t->ne[2] == 1 && t->ne[3] == 1) {
snprintf(buf, sizeof(buf), "%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], c);
} else {
snprintf(buf, sizeof(buf), "%zu:%zu:%zu:%zu%s", (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], c);
}
return buf;
}
static const char * format_tensor_buff(const struct ggml_tensor * t) {
if (t && t->buffer) {
return ggml_backend_buffer_name(t->buffer);
}
return "NONE";
}
template <typename Fn>
static std::string format_op_tensors(const htp_opnode & node, Fn fn) {
std::string res;
const auto & inputs = node.get_inputs();
const auto & outputs = node.get_outputs();
if (!inputs.empty()) {
for (size_t i = 0; i < inputs.size(); i++) {
if (i > 0) res += " x ";
res += fn(inputs[i]);
}
res += " -> ";
}
if (outputs.empty()) {
res += fn(node.dst());
} else {
for (size_t i = 0; i < outputs.size(); i++) {
if (i > 0) res += " x ";
res += fn(outputs[i]);
}
}
return res;
}
static std::string format_kernel_params(const htp_opnode & node) {
char str[128];
const size_t max_size = sizeof(str);
if (node.opcode == HTP_OP_MUL_MAT || node.opcode == HTP_OP_MUL_MAT_ID ||
node.opcode == HTP_OP_MUL_MAT_NX || node.opcode == HTP_OP_MUL_MAT_ID_NX ||
node.opcode == HTP_OP_MUL_MAT_ADD) {
const auto * kparams = node.kparams<htp_mm_kernel_params>();
const char * path = "unknown";
int32_t type = kparams->kernel_type;
if (type == HTP_MM_KERNEL_HMX_2D || type == HTP_MM_KERNEL_HMX_BATCHED) {
path = (kparams->tile_size > 0) ? "hmx-tiled" : "hmx-flat";
} else if (type == HTP_MM_KERNEL_HVX_F16_F16_VTCM || type == HTP_MM_KERNEL_HVX_F32_F32_VTCM ||
type == HTP_MM_KERNEL_HVX_BF16_F32_VTCM) {
path = "hvx-flat";
} else if (type == HTP_MM_KERNEL_HVX_QUANT_ROW || type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
path = "hvx-tiled";
}
snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_FLASH_ATTN_EXT) {
const auto * kparams = node.kparams<htp_fa_kernel_params>();
const char * path = "unknown";
int32_t type = kparams->kernel_type;
if (type == HTP_FA_KERNEL_HMX) {
path = kparams->u.hmx.pipeline ? "hmx-pipe" : "hmx-seq";
} else if (type == HTP_FA_KERNEL_HVX) {
path = "hvx";
}
snprintf(str, max_size, "%s vtcm %d", path, (int) kparams->vtcm_size);
} else if (htp_op_is_unary(node.opcode)) {
const auto * kparams = node.kparams<htp_unary_kernel_params>();
snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) {
snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]);
} else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) {
snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]);
} else if (node.opcode == HTP_OP_ALLREDUCE && node.node) {
snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]);
} else if (node.opcode == HTP_OP_SSM_CONV) {
const auto * kparams = node.kparams<htp_ssm_conv_kernel_params>();
snprintf(str, max_size, "%s vtcm %d", kparams->n_t == 1 ? "decode" : "prefill", (int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_SOFTMAX) {
const auto * kparams = node.kparams<htp_softmax_kernel_params>();
snprintf(str, max_size, "k%d nth %d vtcm %d", (int) kparams->kernel_type, (int) kparams->n_threads, (int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_GATED_DELTA_NET) {
const auto * kparams = node.kparams<htp_gdn_kernel_params>();
const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";
snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",
(unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||
node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
const auto * kparams = node.kparams<htp_binary_kernel_params>();
snprintf(str, max_size, "vtcm %u", (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_ARGSORT || node.opcode == HTP_OP_TOP_K) {
const auto * kparams = node.kparams<htp_sort_kernel_params>();
snprintf(str, max_size, "%s nth %d nchk %d chk %d vtcm %d",
node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort",
(int) kparams->n_threads, (int) kparams->n_chunks,
(int) kparams->chunk_elems, (int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_GET_ROWS) {
const auto * kparams = node.kparams<htp_get_rows_kernel_params>();
const char * ktype_str = "unknown";
switch (kparams->kernel_type) {
case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break;
case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break;
case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break;
}
snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size);
} else if (htp_op_is_glu(node.opcode)) {
const auto * kparams = node.kparams<htp_glu_kernel_params>();
snprintf(str, max_size, "nth %d vtcm %u", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_PAD) {
const auto * kparams = node.kparams<htp_pad_kernel_params>();
snprintf(str, max_size, "k%d nth %d vtcm %u", (int) kparams->kernel_type, (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_CUMSUM) {
const auto * kparams = node.kparams<htp_cumsum_kernel_params>();
snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_FILL) {
const auto * kparams = node.kparams<htp_fill_kernel_params>();
snprintf(str, max_size, "%s nth %d", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (int) kparams->n_threads);
} else if (node.opcode == HTP_OP_ROLL) {
const auto * kparams = node.kparams<htp_roll_kernel_params>();
snprintf(str, max_size, "k%d nth %d", (int) kparams->kernel_type, (int) kparams->n_threads);
} else if (node.opcode == HTP_OP_DIAG) {
const auto * kparams = node.kparams<htp_diag_kernel_params>();
snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_SOLVE_TRI) {
const auto * kparams = node.kparams<htp_solve_tri_kernel_params>();
snprintf(str, max_size, "%s nth %d", kparams->batched ? "batched" : "chunked", (int) kparams->n_threads);
} else {
snprintf(str, max_size, "----");
}
return str;
}
static void append_section(std::string & res, const std::string & sec) {
if (!res.empty()) {
res += "|";
}
res += sec;
}
std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags) {
std::string res;
if (flags & HTP_OPFMT_NAMES) append_section(res, format_tensor_name(t));
if (flags & HTP_OPFMT_DIMS) append_section(res, format_tensor_dims(t));
if (flags & HTP_OPFMT_TYPES) append_section(res, format_tensor_type(t));
if (flags & HTP_OPFMT_STRIDES) append_section(res, format_tensor_strides(t));
if (flags & HTP_OPFMT_BUFFS) append_section(res, format_tensor_buff(t));
return res;
}
std::string htp_format_op(const htp_opnode & node, uint32_t flags) {
std::string res;
if (flags & HTP_OPFMT_OP) append_section(res, node.name());
if (flags & HTP_OPFMT_NAMES) append_section(res, format_op_tensors(node, format_tensor_name));
if (flags & HTP_OPFMT_DIMS) append_section(res, format_op_tensors(node, format_tensor_dims));
if (flags & HTP_OPFMT_TYPES) append_section(res, format_op_tensors(node, format_tensor_type));
if (flags & HTP_OPFMT_STRIDES) append_section(res, format_op_tensors(node, format_tensor_strides));
if (flags & HTP_OPFMT_BUFFS) append_section(res, format_op_tensors(node, format_tensor_buff));
if (flags & HTP_OPFMT_KPARAMS) append_section(res, format_kernel_params(node));
return res;
}
llama.cpp/ggml/src/ggml-hexagon/htp-opnode.h
+172
View File
@@ -0,0 +1,172 @@
#ifndef HTP_OPNODE_H
#define HTP_OPNODE_H
#include "ggml-backend-impl.h"
#include "ggml-impl.h"
#include "htp/htp-ops.h"
#include <cstring>
#include <initializer_list>
#include <memory>
#include <string>
#include <vector>
#define HTP_OP_MAX_FUSED 16
// local std::inplace_vector replacement
template <typename T, size_t MAX_N>
struct htp_opvec {
T data[MAX_N] {};
size_t n = 0;
htp_opvec() = default;
htp_opvec(std::initializer_list<T> init) {
GGML_ASSERT(init.size() <= MAX_N);
for (const auto & v : init) {
data[n++] = v;
}
}
size_t size() const { return n; }
bool empty() const { return n == 0; }
void clear() { n = 0; }
T * begin() { return data; }
T * end() { return data + n; }
const T * begin() const { return data; }
const T * end() const { return data + n; }
T & back() { return data[n - 1]; }
const T & back() const { return data[n - 1]; }
T & operator[](size_t i) { return data[i]; }
const T & operator[](size_t i) const { return data[i]; }
void push_back(T val) {
GGML_ASSERT(n < MAX_N);
data[n++] = val;
}
void insert(T * pos, T val) {
GGML_ASSERT(n < MAX_N);
for (T * p = data + n; p > pos; --p) {
*p = *(p - 1);
}
*pos = val;
n++;
}
};
static inline bool htp_is_hadamard(const struct ggml_tensor * t) {
return t && t->op == GGML_OP_MUL_MAT && ggml_get_op_params_i32(t, 1) == GGML_HINT_SRC0_IS_HADAMARD;
}
struct htp_opnode {
const ggml_tensor * node { nullptr };
htp_op_code opcode { HTP_OP_INVALID };
int32_t params[HTP_OP_MAX_PARAMS] {0};
int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS] {0};
std::vector<std::shared_ptr<ggml_tensor>> dummy;
htp_opvec<const ggml_tensor *, HTP_OP_MAX_FUSED> fused;
htp_opvec<const ggml_tensor *, HTP_OP_MAX_INPUTS> inputs;
htp_opvec<const ggml_tensor *, HTP_OP_MAX_OUTPUTS> outputs;
int n_active_src(const ggml_tensor * t) const {
if (!t) return 0;
for (int i = GGML_MAX_SRC - 1; i >= 0; i--) {
if (t->src[i]) {
return i + 1;
}
}
return 0;
}
void init(const ggml_tensor * node) {
static_assert(sizeof(this->params) == sizeof(this->node->op_params), "params size mismatch");
this->node = node;
if (!node) return;
memcpy(this->params, this->node->op_params, sizeof(this->params));
// Build inputs (preserving optional nullptrs)
if (htp_is_hadamard(this->node)) {
this->inputs.push_back(this->node->src[1]);
} else {
int n_inputs = n_active_src(this->node);
for (int i = 0; i < n_inputs; i++) {
this->inputs.push_back(this->node->src[i]);
}
}
// Build outputs
this->outputs.push_back(this->dst());
}
htp_opnode(htp_op_code opcode = HTP_OP_INVALID, const ggml_tensor * node = nullptr) : opcode(opcode) {
init(node);
}
htp_opnode(const ggml_tensor * node) : opcode(HTP_OP_INVALID) {
init(node);
}
const ggml_tensor * src0() const { return node->src[0]; }
const ggml_tensor * src1() const { return node->src[1]; }
const ggml_tensor * dst() const { return outputs.empty() ? node : outputs.back(); }
ggml_tensor * add_dummy(const ggml_tensor & t) {
dummy.push_back(std::make_shared<ggml_tensor>(t));
return dummy.back().get();
}
const htp_opvec<const ggml_tensor *, HTP_OP_MAX_INPUTS> & get_inputs() const {
return inputs;
}
const htp_opvec<const ggml_tensor *, HTP_OP_MAX_OUTPUTS> & get_outputs() const {
return outputs;
}
template <typename T>
T * kparams() {
static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS");
static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t");
return reinterpret_cast<T *>(kernel_params);
}
template <typename T>
const T * kparams() const {
static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS");
static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t");
return reinterpret_cast<const T *>(kernel_params);
}
template <typename T>
void set_kparams(const T & kp) {
static_assert(sizeof(T) <= sizeof(kernel_params), "kparams size exceeds HTP_OP_MAX_KERN_PARAMS");
static_assert(alignof(T) <= alignof(int32_t), "kparams alignment exceeds int32_t");
memcpy(kernel_params, &kp, sizeof(T));
}
std::string name() const;
};
enum htp_opformat_flags {
HTP_OPFMT_OP = (1 << 0),
HTP_OPFMT_NAMES = (1 << 1),
HTP_OPFMT_DIMS = (1 << 2),
HTP_OPFMT_TYPES = (1 << 3),
HTP_OPFMT_STRIDES = (1 << 4),
HTP_OPFMT_BUFFS = (1 << 5),
HTP_OPFMT_KPARAMS = (1 << 6),
HTP_OPFMT_EXEC = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS | HTP_OPFMT_KPARAMS,
HTP_OPFMT_SUPP = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_BUFFS,
HTP_OPFMT_PROF = HTP_OPFMT_OP | HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES | HTP_OPFMT_KPARAMS,
HTP_OPFMT_TENSOR = HTP_OPFMT_NAMES | HTP_OPFMT_DIMS | HTP_OPFMT_TYPES | HTP_OPFMT_STRIDES,
};
std::string htp_format_op(const htp_opnode & node, uint32_t flags = HTP_OPFMT_EXEC);
std::string htp_format_tensor(const ggml_tensor * t, uint32_t flags = HTP_OPFMT_TENSOR);
#endif // HTP_OPNODE_H
llama.cpp/ggml/src/ggml-hexagon/htp-repack.cpp
+1636
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp-repack.h
+69
View File
@@ -0,0 +1,69 @@
#ifndef HTP_REPACK_H
#define HTP_REPACK_H
#include <stddef.h>
#include <stdint.h>
#include "ggml.h"
#include "htp/htp-tensor.h"
static inline bool htp_is_tiled_weight_type(enum ggml_type type) {
return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 ||
type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL ||
type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q1_0 ||
type == GGML_TYPE_Q6_K ||
type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K ||
type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K ||
type == GGML_TYPE_Q2_0 ||
type == GGML_TYPE_F16 || type == GGML_TYPE_F32 ||
type == GGML_TYPE_BF16;
}
static inline bool htp_is_tiled_weight_tensor(const struct ggml_tensor * t) {
if (!ggml_is_contiguous(t) || ggml_is_view(t)) {
return false;
}
if (!htp_is_tiled_weight_type(t->type)) {
return false;
}
if (t->ne[0] < 32) {
return false;
}
if (!ggml_is_quantized(t->type)) {
return (t->ne[0] % 32 == 0) && (t->ne[1] % 32 == 0) && (t->ne[2] == 1) && (t->ne[3] == 1);
}
return t->ne[3] == 1;
}
// Size of one tiled row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins,
// so they are larger than the ggml blocks. For the other tiled types (Q2_K included) the tile has the same size.
static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) {
if (type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || type == GGML_TYPE_BF16) {
return (size_t) ne0 * sizeof(ggml_fp16_t);
}
if (type == GGML_TYPE_Q6_K) {
return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q6_K / 32);
}
if (type == GGML_TYPE_Q2_0) {
return (size_t) (ne0 / 64) * 18;
}
if (type == GGML_TYPE_Q4_K) {
return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q4_K / 32);
}
if (type == GGML_TYPE_Q5_K) {
return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q5_K / 32);
}
if (type == GGML_TYPE_Q3_K) {
return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q3_K / 32);
}
return ggml_row_size(type, ne0);
}
static inline bool htp_is_hmx_weight_type(enum ggml_type type) {
return htp_is_tiled_weight_type(type);
}
void htp_repack_tensor_tiled(struct ggml_tensor * t, const void * data, size_t size);
void htp_unpack_tensor_tiled(void * data, const struct ggml_tensor * t, size_t offset, size_t size);
#endif // HTP_REPACK_H
llama.cpp/ggml/src/ggml-hexagon/htp/CMakeLists.txt
+69
View File
@@ -0,0 +1,69 @@
cmake_minimum_required(VERSION 3.22.2)
project(ggml-htp C CXX ASM)
include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_fun.cmake)
include_directories(
${HEXAGON_SDK_ROOT}/incs
${HEXAGON_SDK_ROOT}/incs/stddef
${CMAKE_CURRENT_SOURCE_DIR}/../../../include
${CMAKE_CURRENT_SOURCE_DIR}/../..
${CMAKE_CURRENT_SOURCE_DIR}/..
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR})
set(HTP_LIB ggml-htp-${DSP_VERSION})
add_library(${HTP_LIB} SHARED
main.c
htp_iface_skel.c
work-queue.c
dma-queue.c
hmx-queue.c
htp-dcache.c
matmul-ops.c
flash-attn-ops.c
gated-delta-net-ops.c
binary-ops.c
unary-ops.c
sum-rows-ops.c
softmax-ops.c
glu-ops.c
rope-ops.c
set-rows-ops.c
get-rows-ops.c
cpy-ops.c
gather-rows.c
repeat-ops.c
ssm-conv.c
cumsum-ops.c
fill-ops.c
concat-ops.c
diag-ops.c
solve-tri-ops.c
pad-ops.c
argsort-ops.c
im2col-ops.c
roll-ops.c
pool-ops.c
allreduce-ops.c
)
target_compile_definitions(${HTP_LIB} PRIVATE
$<IF:$<BOOL:${HEXAGON_HTP_DEBUG}>,HTP_DEBUG=1,NDEBUG=1>
$<IF:$<BOOL:${HEXAGON_HTP_DEBUG}>,FARF_HIGH=1,>)
build_idl(htp_iface.idl ${HTP_LIB})
set_target_properties(${HTP_LIB} PROPERTIES EXPORT_COMPILE_COMMANDS ON)
if (NOT Python3_EXECUTABLE)
find_package(Python3 REQUIRED COMPONENTS Interpreter)
endif()
add_custom_command(TARGET ${HTP_LIB} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E echo "Adding SWIV section $<TARGET_FILE_NAME:${HTP_LIB}>"
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/htp-swiv.py -i $<TARGET_FILE:${HTP_LIB}> -o $<TARGET_FILE:${HTP_LIB}>
VERBATIM)
install(TARGETS ${HTP_LIB})
llama.cpp/ggml/src/ggml-hexagon/htp/allreduce-ops.c
+475
View File
@@ -0,0 +1,475 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic ignored "-Wunused-function"
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <stdatomic.h>
#include <math.h>
#include <string.h>
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "hvx-utils.h"
#include "htp-tensor.h"
#include "dma-queue.h"
#include "hex-profile.h"
#include "allreduce-ops.h"
#include "htp-fence.h"
struct htp_allreduce_context {
struct htp_ops_context * octx;
uint32_t n_ranks;
uint32_t n_dsts;
uint32_t nelem;
uint32_t ne0;
uint32_t ne1;
uint32_t row_size_aligned;
uint32_t rank_elem_start;
uint32_t rank_nelem;
uint32_t elems_per_thread;
uint32_t block_elems;
uint32_t vtcm_size_per_thread;
bool is_row_bcast;
uint8_t * src_spad_base[HTP_ALLREDUCE_MAX_RANKS];
uint8_t * dst_spad_base;
uint8_t * res_spad_base;
};
#define DEFINE_ALLREDUCE_THREAD_DMA_1D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD) \
static void allreduce_thread_dma_1d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \
struct htp_ops_context * octx = actx->octx; \
\
const uint32_t n_ranks = actx->n_ranks; \
const uint32_t n_dsts = actx->n_dsts; \
const uint32_t block_elems = actx->block_elems; \
\
const uint32_t dr = actx->elems_per_thread; \
const uint32_t ir0 = actx->rank_elem_start + dr * ith; \
const uint32_t ir1 = MIN(ir0 + dr, actx->rank_elem_start + actx->rank_nelem); \
if (ir0 >= ir1) return; \
\
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
dma_queue * dma_q = octx->ctx->dma[ith]; \
\
const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \
uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \
uint8_t * res_spad_base = HAS_ADD ? (actx->res_spad_base + vtcm_thread_offset) : NULL; \
\
const size_t spad_half = actx->vtcm_size_per_thread / 2; \
uint32_t ir_prefetch = ir0; \
int spad_idx = 0; \
\
for (int k = 0; k < 2 && ir_prefetch < ir1; k++) { \
uint32_t cur_elems = MIN(block_elems, ir1 - ir_prefetch); \
size_t cur_bytes = cur_elems * sizeof(TYPE); \
uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \
for (uint32_t d = 0; d < n_dsts; d++) { \
dma_addr_t d_ddr = octx->dsts[d]->data + ir_prefetch * sizeof(TYPE); \
dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 0); \
} \
for (uint32_t s = 0; s < n_ranks; s++) { \
uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \
const dma_addr_t s_ddr = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \
dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \
} \
if (HAS_ADD) { \
uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \
const dma_addr_t r_ddr = octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \
dma_queue_push(dma_q, dma_make_data(r_spad, r_ddr), cur_bytes, cur_bytes, cur_bytes, 1); \
} \
ir_prefetch += cur_elems; \
spad_idx ^= 1; \
} \
\
int comp_spad_idx = 0; \
for (uint32_t ir = ir0; ir < ir1; ) { \
uint32_t cur_elems = MIN(block_elems, ir1 - ir); \
size_t cur_bytes = cur_elems * sizeof(TYPE); \
for (uint32_t d = 0; d < n_dsts; d++) { \
dma_queue_pop(dma_q); \
} \
for (uint32_t s = 0; s < n_ranks; s++) { \
dma_queue_pop(dma_q); \
} \
if (HAS_ADD) { \
dma_queue_pop(dma_q); \
} \
uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \
uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \
uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \
uint8_t * r_spad = HAS_ADD ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \
HVX_ADD_FN(d_spad, s0_spad, s1_spad, cur_elems); \
for (uint32_t s = 2; s < n_ranks; s++) { \
uint8_t * ss_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \
HVX_ADD_FN(d_spad, d_spad, ss_spad, cur_elems); \
} \
if (HAS_ADD) { \
HVX_ADD_FN(d_spad, d_spad, r_spad, cur_elems); \
} \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir); \
for (uint32_t d = 0; d < n_dsts; d++) { \
dma_addr_t d_ddr = octx->dsts[d]->data + ir * sizeof(TYPE); \
dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), cur_bytes, cur_bytes, cur_bytes, 1); \
} \
if (ir_prefetch < ir1) { \
uint32_t next_elems = MIN(block_elems, ir1 - ir_prefetch); \
size_t next_bytes = next_elems * sizeof(TYPE); \
for (uint32_t s = 0; s < n_ranks; s++) { \
uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \
const dma_addr_t s_next = octx->src[s]->data + ir_prefetch * sizeof(TYPE); \
dma_queue_push(dma_q, dma_make_data(s_spad, s_next), next_bytes, next_bytes, next_bytes, 1); \
} \
if (HAS_ADD) { \
uint8_t * r_spad_next = res_spad_base + comp_spad_idx * spad_half; \
const dma_addr_t r_next = octx->src[2 * n_ranks]->data + ir_prefetch * sizeof(TYPE); \
dma_queue_push(dma_q, dma_make_data(r_spad_next, r_next), next_bytes, next_bytes, next_bytes, 1); \
} \
ir_prefetch += next_elems; \
} \
comp_spad_idx ^= 1; \
ir += cur_elems; \
} \
dma_queue_flush(dma_q); \
}
DEFINE_ALLREDUCE_THREAD_DMA_1D(f16, __fp16, hvx_add_f16_aaa, 0)
DEFINE_ALLREDUCE_THREAD_DMA_1D(f32, float, hvx_add_f32_aaa, 0)
DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f16, __fp16, hvx_add_f16_aaa, 1)
DEFINE_ALLREDUCE_THREAD_DMA_1D(add_f32, float, hvx_add_f32_aaa, 1)
#define DEFINE_ALLREDUCE_THREAD_DMA_2D(SUFFIX, TYPE, HVX_ADD_FN, HAS_ADD, IS_ROW_BCAST) \
static void allreduce_thread_dma_2d_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
struct htp_allreduce_context * actx = (struct htp_allreduce_context *) data; \
struct htp_ops_context * octx = actx->octx; \
\
const uint32_t n_ranks = actx->n_ranks; \
const uint32_t n_dsts = actx->n_dsts; \
const uint32_t ne0 = actx->ne0; \
const uint32_t block_rows = actx->block_elems; \
const uint32_t row_size_aligned = actx->row_size_aligned; \
const uint32_t row_bytes = ne0 * sizeof(TYPE); \
\
const uint32_t dr = actx->elems_per_thread; \
const uint32_t r0 = actx->rank_elem_start + dr * ith; \
const uint32_t r1 = MIN(r0 + dr, actx->rank_elem_start + actx->rank_nelem); \
if (r0 >= r1) return; \
\
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
dma_queue * dma_q = octx->ctx->dma[ith]; \
\
const size_t vtcm_thread_offset = ith * actx->vtcm_size_per_thread; \
uint8_t * dst_spad_base = actx->dst_spad_base + vtcm_thread_offset; \
uint8_t * res_spad_base = HAS_ADD ? (IS_ROW_BCAST ? actx->res_spad_base : (actx->res_spad_base + vtcm_thread_offset)) : NULL; \
\
const size_t spad_half = actx->vtcm_size_per_thread / 2; \
uint32_t r_prefetch = r0; \
int spad_idx = 0; \
\
for (int k = 0; k < 2 && r_prefetch < r1; k++) { \
uint32_t cur_rows = MIN(block_rows, r1 - r_prefetch); \
uint8_t * d_spad = dst_spad_base + spad_idx * spad_half; \
for (uint32_t d = 0; d < n_dsts; d++) { \
dma_addr_t d_ddr = octx->dsts[d]->data + r_prefetch * octx->dsts[d]->nb[1]; \
dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, 0); \
} \
for (uint32_t s = 0; s < n_ranks; s++) { \
uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + spad_idx * spad_half; \
const dma_addr_t s_ddr = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \
dma_queue_push(dma_q, dma_make_data(s_spad, s_ddr), row_size_aligned, octx->src[s]->nb[1], row_bytes, cur_rows); \
} \
if (HAS_ADD && !IS_ROW_BCAST) { \
uint8_t * r_spad = res_spad_base + spad_idx * spad_half; \
const dma_addr_t r_ddr = octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \
dma_queue_push(dma_q, dma_make_data(r_spad, r_ddr), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, cur_rows); \
} \
r_prefetch += cur_rows; \
spad_idx ^= 1; \
} \
\
int comp_spad_idx = 0; \
for (uint32_t r = r0; r < r1; ) { \
uint32_t cur_rows = MIN(block_rows, r1 - r); \
for (uint32_t d = 0; d < n_dsts; d++) { \
dma_queue_pop(dma_q); \
} \
for (uint32_t s = 0; s < n_ranks; s++) { \
dma_queue_pop(dma_q); \
} \
if (HAS_ADD && !IS_ROW_BCAST) { \
dma_queue_pop(dma_q); \
} \
uint8_t * d_spad = dst_spad_base + comp_spad_idx * spad_half; \
uint8_t * s0_spad = actx->src_spad_base[0] + vtcm_thread_offset + comp_spad_idx * spad_half; \
uint8_t * s1_spad = actx->src_spad_base[1] + vtcm_thread_offset + comp_spad_idx * spad_half; \
uint8_t * r_spad = (HAS_ADD && !IS_ROW_BCAST) ? (res_spad_base + comp_spad_idx * spad_half) : NULL; \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
for (uint32_t row = 0; row < cur_rows; row++) { \
uint8_t * d_row = d_spad + row * row_size_aligned; \
const uint8_t * s0_row = s0_spad + row * row_size_aligned; \
const uint8_t * s1_row = s1_spad + row * row_size_aligned; \
HVX_ADD_FN(d_row, s0_row, s1_row, ne0); \
for (uint32_t s = 2; s < n_ranks; s++) { \
const uint8_t * ss_row = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half + row * row_size_aligned; \
HVX_ADD_FN(d_row, d_row, ss_row, ne0); \
} \
if (HAS_ADD) { \
const uint8_t * res_row = IS_ROW_BCAST ? res_spad_base : (r_spad + row * row_size_aligned); \
HVX_ADD_FN(d_row, d_row, res_row, ne0); \
} \
} \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
for (uint32_t d = 0; d < n_dsts; d++) { \
dma_addr_t d_ddr = octx->dsts[d]->data + r * octx->dsts[d]->nb[1]; \
dma_queue_push(dma_q, dma_make_data(d_ddr, d_spad), octx->dsts[d]->nb[1], row_size_aligned, row_bytes, cur_rows); \
} \
if (r_prefetch < r1) { \
uint32_t next_rows = MIN(block_rows, r1 - r_prefetch); \
for (uint32_t s = 0; s < n_ranks; s++) { \
uint8_t * s_spad = actx->src_spad_base[s] + vtcm_thread_offset + comp_spad_idx * spad_half; \
const dma_addr_t s_next = octx->src[s]->data + r_prefetch * octx->src[s]->nb[1]; \
dma_queue_push(dma_q, dma_make_data(s_spad, s_next), row_size_aligned, octx->src[s]->nb[1], row_bytes, next_rows); \
} \
if (HAS_ADD && !IS_ROW_BCAST) { \
uint8_t * r_spad_next = res_spad_base + comp_spad_idx * spad_half; \
const dma_addr_t r_next = octx->src[2 * n_ranks]->data + r_prefetch * octx->src[2 * n_ranks]->nb[1]; \
dma_queue_push(dma_q, dma_make_data(r_spad_next, r_next), row_size_aligned, octx->src[2 * n_ranks]->nb[1], row_bytes, next_rows); \
} \
r_prefetch += next_rows; \
} \
comp_spad_idx ^= 1; \
r += cur_rows; \
} \
dma_queue_flush(dma_q); \
}
DEFINE_ALLREDUCE_THREAD_DMA_2D(f16, __fp16, hvx_add_f16_aaa, 0, 0)
DEFINE_ALLREDUCE_THREAD_DMA_2D(f32, float, hvx_add_f32_aaa, 0, 0)
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f16, __fp16, hvx_add_f16_aaa, 1, 0)
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f32, float, hvx_add_f32_aaa, 1, 0)
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f16, __fp16, hvx_add_f16_aaa, 1, 1)
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f32, float, hvx_add_f32_aaa, 1, 1)
static int validate_allreduce(
struct htp_ops_context * octx,
const struct htp_allreduce_kernel_params * kparams,
uint32_t n_ranks
) {
if (kparams->n_threads == 0 || (uint32_t) kparams->n_threads > octx->ctx->n_threads) {
return HTP_STATUS_INVAL_PARAMS;
}
octx->n_threads = (uint32_t) kparams->n_threads;
if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) {
return HTP_STATUS_INVAL_PARAMS;
}
const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD);
const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(
n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0);
const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread;
if (vtcm_size != (size_t) kparams->vtcm_size) {
return HTP_STATUS_INVAL_PARAMS;
}
if (vtcm_size > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
if (octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_F32) {
return HTP_STATUS_NO_SUPPORT;
}
return HTP_STATUS_OK;
}
int op_allreduce(struct htp_ops_context * octx) {
if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) {
return HTP_STATUS_OK;
}
const struct htp_allreduce_kernel_params * kparams = (const struct htp_allreduce_kernel_params *) octx->kernel_params;
const struct htp_tensor * dst = octx->dst;
const uint32_t rank = (uint32_t) kparams->rank;
const uint32_t n_ranks = (uint32_t) kparams->n_ranks;
if (n_ranks < 2 || n_ranks > HTP_ALLREDUCE_MAX_RANKS || rank >= n_ranks) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0];
const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1];
const struct htp_tensor * my_sync = octx->src[n_ranks + rank];
atomic_uint * my_fence = (atomic_uint *) (uintptr_t) my_sync->data;
const int status = validate_allreduce(octx, kparams, n_ranks);
if (status != HTP_STATUS_OK) {
if (status == HTP_STATUS_NO_SUPPORT) {
FARF(ERROR, "ggml-hex: allreduce unsupported type %d : rank %u\n", dst->type, rank);
}
htp_fence_write(my_fence, fence_seq_exit, status);
return status;
}
const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD);
const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3];
const int32_t mode = kparams->mode;
// 1. Entry Barrier: Synchronize all ranks before reading
struct htp_thread_trace * tr0 = &octx->ctx->trace[0];
htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
htp_fence_write(my_fence, fence_seq_entry, octx->status);
for (uint32_t j = 0; j < n_ranks; j++) {
if (j == rank) continue;
const struct htp_tensor * peer_sync = octx->src[n_ranks + j];
atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data;
uint64_t spins = 0;
while (1) {
uint32_t peer_seq;
uint32_t peer_status;
htp_fence_read(peer_fence, &peer_seq, &peer_status);
if ((int32_t)(peer_seq - fence_seq_entry) >= 0) {
if (peer_status > HTP_STATUS_OK) {
FARF(ERROR, "ggml-hex: allreduce entry peer %u failed with status %u\n", j, peer_status);
htp_fence_write(my_fence, fence_seq_exit, peer_status);
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
return peer_status;
}
break;
}
if (++spins > HTP_FENCE_TIMEOUT) {
FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n",
rank, j, peer_fence, fence_seq_entry, peer_seq);
htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR);
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
return HTP_STATUS_INTERNAL_ERR;
}
hex_pause();
}
}
asm volatile ("syncht" : : : "memory");
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
// 2. Multi-threaded Reduction across assigned rank chunk
if (nelem > 0) {
const uint32_t n_threads = (uint32_t) kparams->n_threads;
const uint32_t block_elems = (uint32_t) kparams->block_elems;
const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread;
const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread;
struct htp_allreduce_context actx;
actx.octx = octx;
actx.n_ranks = n_ranks;
actx.n_dsts = (uint32_t) kparams->n_dsts ? (uint32_t) kparams->n_dsts : n_ranks;
actx.nelem = nelem;
actx.ne0 = (uint32_t) kparams->ne0;
actx.ne1 = (uint32_t) kparams->ne1;
actx.row_size_aligned = (uint32_t) kparams->row_size_aligned;
actx.rank_elem_start = (uint32_t) kparams->rank_elem_start;
actx.rank_nelem = (uint32_t) kparams->rank_nelem;
actx.elems_per_thread = elems_per_thread;
actx.block_elems = block_elems;
actx.vtcm_size_per_thread = vtcm_size_per_thread;
actx.is_row_bcast = (kparams->is_row_bcast != 0);
work_queue_func_t reduce_fun = NULL;
switch (kparams->kernel_type) {
case HTP_ALLREDUCE_KERNEL_DMA_1D:
if (has_add) {
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_add_f16 : allreduce_thread_dma_1d_add_f32;
} else {
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_1d_f16 : allreduce_thread_dma_1d_f32;
}
break;
case HTP_ALLREDUCE_KERNEL_DMA_2D:
if (has_add) {
if (kparams->is_row_bcast) {
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_bcast_f16 : allreduce_thread_dma_2d_add_bcast_f32;
} else {
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_add_f16 : allreduce_thread_dma_2d_add_f32;
}
} else {
reduce_fun = (dst->type == HTP_TYPE_F16) ? allreduce_thread_dma_2d_f16 : allreduce_thread_dma_2d_f32;
}
break;
default:
FARF(ERROR, "ggml-hex: allreduce unsupported kernel %d : rank %u\n", kparams->kernel_type, rank);
htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_NO_SUPPORT);
return HTP_STATUS_NO_SUPPORT;
}
uint8_t * vtcm_ptr = (uint8_t *) octx->ctx->vtcm_base;
for (uint32_t s = 0; s < n_ranks; s++) {
actx.src_spad_base[s] = vtcm_ptr;
vtcm_ptr += n_threads * vtcm_size_per_thread;
}
actx.dst_spad_base = vtcm_ptr;
vtcm_ptr += n_threads * vtcm_size_per_thread;
if (has_add) {
actx.res_spad_base = vtcm_ptr;
vtcm_ptr += (actx.is_row_bcast ? 1 : n_threads) * vtcm_size_per_thread;
}
if (has_add && actx.is_row_bcast) {
const dma_addr_t r_ddr = octx->src[2 * n_ranks]->data;
const uint32_t row_bytes = actx.ne0 * (dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float));
dma_queue * dma_q = octx->ctx->dma[0];
dma_queue_push(dma_q, dma_make_data(actx.res_spad_base, r_ddr), actx.row_size_aligned, 0, row_bytes, 1);
dma_queue_pop(dma_q);
}
work_queue_run(octx->ctx->work_queue, reduce_fun, &actx, n_threads);
}
// 4. Exit Barrier: Synchronize all ranks after writing
htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
// drain fan-out DMA writes before exit barrier
if (mode == HTP_ALLREDUCE_SHARDED_FANOUT) {
asm volatile ("syncht" : : : "memory");
}
htp_fence_write(my_fence, fence_seq_exit, octx->status);
for (uint32_t j = 0; j < n_ranks; j++) {
if (j == rank) continue;
const struct htp_tensor * peer_sync = octx->src[n_ranks + j];
atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data;
uint64_t spins = 0;
while (1) {
uint32_t peer_seq;
uint32_t peer_status;
htp_fence_read(peer_fence, &peer_seq, &peer_status);
if ((int32_t)(peer_seq - fence_seq_exit) >= 0) {
if (peer_status > HTP_STATUS_OK) {
FARF(ERROR, "ggml-hex: allreduce exit peer %u failed with status %u\n", j, peer_status);
htp_fence_write(my_fence, fence_seq_exit, peer_status);
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
return peer_status;
}
break;
}
if (++spins > HTP_FENCE_TIMEOUT) {
FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n",
rank, j, peer_fence, fence_seq_exit, peer_seq);
htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR);
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
return HTP_STATUS_INTERNAL_ERR;
}
hex_pause();
}
}
asm volatile ("syncht" : : : "memory");
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
return octx->status;
}
llama.cpp/ggml/src/ggml-hexagon/htp/allreduce-ops.h
+63
View File
@@ -0,0 +1,63 @@
#ifndef ALLREDUCE_OPS_H
#define ALLREDUCE_OPS_H
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#define HTP_ALLREDUCE_MAX_RANKS 4
#ifdef __cplusplus
extern "C" {
#endif
enum htp_allreduce_kernel_type {
HTP_ALLREDUCE_KERNEL_UNSUPPORTED = 0,
HTP_ALLREDUCE_KERNEL_DMA_1D,
HTP_ALLREDUCE_KERNEL_DMA_2D,
};
enum htp_allreduce_mode {
HTP_ALLREDUCE_FULL = 0, // all ranks reduce full tensor
HTP_ALLREDUCE_SHARDED_FANOUT = 1, // each rank reduces 1/N shard and fans out to all buffers
};
static inline size_t htp_allreduce_vtcm_buffer_count(
uint32_t n_ranks,
uint32_t n_threads,
bool has_add,
bool is_row_bcast
) {
return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0);
}
struct htp_allreduce_kernel_params {
uint8_t kernel_type;
int32_t rank;
int32_t n_ranks;
int32_t n_threads;
int32_t block_elems; // 1D: block_elems, 2D: block_rows
int32_t elems_per_thread; // 1D: nelem_per_thread, 2D: nrows_per_thread
int32_t vtcm_size_per_thread;
int32_t vtcm_size;
int32_t ne0;
int32_t ne1;
int32_t row_size_aligned;
int32_t rank_elem_start;
int32_t rank_nelem;
int32_t n_dsts;
int32_t is_row_bcast;
int32_t mode; // enum htp_allreduce_mode: FULL or SHARDED_FANOUT
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_allreduce_kernel_params) <= 128, "htp_allreduce_kernel_params is too large for kernel_params blob");
#endif
#ifdef __cplusplus
}
#endif
#endif /* ALLREDUCE_OPS_H */
llama.cpp/ggml/src/ggml-hexagon/htp/argsort-ops.c
+1037
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp/argsort-ops.h
+163
View File
@@ -0,0 +1,163 @@
#ifndef HTP_ARGSORT_OPS_H
#define HTP_ARGSORT_OPS_H
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include "hex-fastdiv.h"
#include "htp-vtcm.h"
struct htp_sort_kernel_params {
uint32_t n_threads;
uint32_t order; // GGML_SORT_ORDER_ASC (0) or GGML_SORT_ORDER_DESC (1)
uint32_t is_top_k; // 1 if TOP_K, 0 if ARGSORT
uint32_t chunk_elems;
uint32_t n_chunks;
uint32_t vtcm_size;
uint32_t merge_elems;
uint32_t n_slots;
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne01_ne02;
};
struct htp_sort_vtcm_layout {
size_t total_bytes;
size_t phase1_slot_size;
size_t off_phase1;
size_t off_merge_values;
size_t off_merge_indices;
uint32_t chunk_elems;
uint32_t n_chunks;
uint32_t merge_elems;
uint32_t n_threads;
uint32_t n_slots;
};
static inline void htp_sort_vtcm_layout_build(
struct htp_sort_vtcm_layout * L,
uint32_t total_rows,
uint32_t chunk_elems,
uint32_t n_chunks,
uint32_t merge_elems,
uint32_t n_threads,
uint32_t n_slots) {
memset(L, 0, sizeof(*L));
L->chunk_elems = chunk_elems;
L->n_chunks = n_chunks;
L->merge_elems = merge_elems;
L->n_threads = n_threads;
L->n_slots = n_slots;
const size_t values_size = ((chunk_elems * sizeof(float)) + 127) & ~127;
const size_t indices_size = ((chunk_elems * sizeof(int32_t)) + 127) & ~127;
L->phase1_slot_size = ((values_size + indices_size) + 255) & ~255;
size_t off = 0;
if (total_rows > 1) {
VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_slots * n_threads);
} else {
VTCM_LAYOUT_ALLOC(off, off_phase1, L->phase1_slot_size * n_chunks * n_slots);
const size_t merge_values_size = ((merge_elems * sizeof(float)) + 127) & ~127;
const size_t merge_indices_size = ((merge_elems * sizeof(int32_t)) + 127) & ~127;
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_values, merge_values_size, n_chunks > 1);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_merge_indices, merge_indices_size, n_chunks > 1);
}
L->total_bytes = off;
}
static inline bool htp_sort_solve_layout(
struct htp_sort_vtcm_layout * layout,
uint32_t ne00,
uint32_t total_rows,
uint32_t k,
uint32_t n_threads_max,
size_t vtcm_budget,
bool is_top_k) {
memset(layout, 0, sizeof(*layout));
if (total_rows > 1) {
uint32_t n_threads = total_rows < n_threads_max ? total_rows : n_threads_max;
uint32_t n_vec = (ne00 + 31) / 32;
uint32_t n_vec_pow2 = 1;
while (n_vec_pow2 < n_vec) n_vec_pow2 <<= 1;
uint32_t ne00_padded = n_vec_pow2 * 32;
size_t values_size = ((ne00_padded * sizeof(float)) + 127) & ~127;
size_t indices_size = ((ne00_padded * sizeof(int32_t)) + 127) & ~127;
size_t spad_per_slot = ((values_size + indices_size) + 255) & ~255;
uint32_t n_slots = 2;
if (spad_per_slot * 2 > vtcm_budget) {
n_slots = 1;
}
size_t spad_per_thread = spad_per_slot * n_slots;
while (n_threads > 1 && (spad_per_thread * n_threads) > vtcm_budget) {
n_threads--;
}
htp_sort_vtcm_layout_build(layout, total_rows, ne00_padded, 1, 0, n_threads, n_slots);
return layout->total_bytes <= vtcm_budget;
}
uint32_t n_vec = (ne00 + 31) / 32;
uint32_t n_vec_pow2 = 1;
while (n_vec_pow2 < n_vec) n_vec_pow2 <<= 1;
uint32_t n_chunks = 1;
if (ne00 > 1024) {
while (n_chunks * 2 <= n_threads_max && n_chunks * 2 <= n_vec_pow2) {
n_chunks *= 2;
}
}
uint32_t chunk_n_vec = n_vec_pow2 / n_chunks;
uint32_t chunk_elems = chunk_n_vec * 32;
uint32_t merge_elems = 0;
if (n_chunks > 1) {
if (is_top_k) {
uint32_t local_k = k < chunk_elems ? k : chunk_elems;
uint32_t total_candidates = n_chunks * local_k;
uint32_t merge_n_vec = (total_candidates + 31) / 32;
uint32_t merge_n_vec_pow2 = 1;
while (merge_n_vec_pow2 < merge_n_vec) merge_n_vec_pow2 <<= 1;
merge_elems = merge_n_vec_pow2 * 32;
} else {
merge_elems = n_vec_pow2 * 32;
}
}
htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, n_chunks, 1);
if (layout->total_bytes > vtcm_budget && n_chunks > 1) {
n_chunks = 1;
chunk_elems = n_vec_pow2 * 32;
merge_elems = 0;
htp_sort_vtcm_layout_build(layout, 1, chunk_elems, n_chunks, merge_elems, 1, 1);
}
if (layout->total_bytes > vtcm_budget) {
return false;
}
if (n_chunks == 1 && layout->phase1_slot_size * 2 <= vtcm_budget) {
htp_sort_vtcm_layout_build(layout, 1, chunk_elems, 1, 0, 1, 2);
}
return true;
}
#if defined(__cplusplus)
static_assert(sizeof(struct htp_sort_kernel_params) <= 128, "htp_sort_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_sort_kernel_params) <= 128, "htp_sort_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_ARGSORT_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/binary-ops.c
+1390
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp/binary-ops.h
+247
View File
@@ -0,0 +1,247 @@
#ifndef HTP_BINARY_OPS_H
#define HTP_BINARY_OPS_H
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include "hex-common.h"
#include "hex-fastdiv.h"
#include "htp-ops.h"
#include "htp-vtcm.h"
enum htp_binary_kernel_type {
HTP_BINARY_KERNEL_SAME_SHAPE = 0,
HTP_BINARY_KERNEL_ROW_BCAST,
HTP_BINARY_KERNEL_SCALAR,
HTP_BINARY_KERNEL_ADD_ID,
HTP_BINARY_KERNEL_COMPLEX,
HTP_BINARY_KERNEL_REPEAT,
HTP_BINARY_KERNEL_CHUNKED,
};
struct htp_binary_kernel_params {
uint8_t kernel_type;
uint8_t split_at_ne01;
uint8_t split_at_ne02;
uint8_t src1_strided;
uint8_t n_srcs;
uint8_t is_scalar;
uint8_t pad[2];
uint32_t n_threads;
uint32_t rows_per_buffer;
uint32_t src0_row_size_aligned;
uint32_t src1_row_size_aligned;
uint32_t dst_row_size_aligned;
uint32_t src1_size;
uint32_t src2_size;
uint32_t vtcm_size;
uint32_t chunk_size;
uint32_t chunk_bytes;
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02;
struct fastdiv_values div_ne012;
struct fastdiv_values div_ne11;
struct fastdiv_values div_ne12;
struct fastdiv_values div_ne13;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob");
struct ggml_tensor;
static inline bool htp_binary_strides_safe(
const struct ggml_tensor * src0,
const struct ggml_tensor * src1,
const struct ggml_tensor * dst,
size_t elem_size
) {
if ((src0->ne[0] != 1 && src0->nb[0] != elem_size) ||
(src1->ne[0] != 1 && src1->nb[0] != elem_size) ||
(dst->ne[0] != 1 && dst->nb[0] != elem_size)) {
return false;
}
if ((src0->ne[1] > 1 && src0->nb[1] < src0->ne[0] * elem_size) ||
(src1->ne[1] > 1 && src1->nb[1] < src1->ne[0] * elem_size) ||
(dst->ne[1] > 1 && dst->nb[1] < dst->ne[0] * elem_size)) {
return false;
}
return true;
}
#else
_Static_assert(sizeof(struct htp_binary_kernel_params) <= 128, "htp_binary_kernel_params is too large for kernel_params blob");
#endif
struct htp_binary_vtcm_layout {
size_t total_bytes;
size_t off_src0;
size_t off_src1;
size_t off_src2;
size_t off_dst;
size_t src0_bytes_per_thread;
size_t src1_bytes_per_thread;
size_t dst_bytes_per_thread;
size_t src0_spad_half_size;
size_t src1_spad_half_size;
size_t dst_spad_half_size;
size_t src1_size;
size_t src2_size;
uint32_t rows_per_buffer;
};
static inline void htp_binary_vtcm_layout_build(
struct htp_binary_vtcm_layout * L,
const struct htp_binary_kernel_params * kparams,
size_t vtcm_size
) {
memset(L, 0, sizeof(*L));
const uint32_t n_threads = kparams->n_threads;
if (n_threads == 0) {
return;
}
const uint32_t n_srcs = kparams->n_srcs > 2 ? kparams->n_srcs : 2;
if (kparams->kernel_type == HTP_BINARY_KERNEL_CHUNKED) {
const size_t chunk_bytes = kparams->chunk_bytes;
if (chunk_bytes == 0) {
return;
}
L->src0_bytes_per_thread = 2 * chunk_bytes;
L->src1_bytes_per_thread = kparams->is_scalar ? 0 : (2 * (n_srcs - 1) * chunk_bytes);
L->dst_bytes_per_thread = 2 * chunk_bytes;
L->src0_spad_half_size = chunk_bytes;
L->src1_spad_half_size = kparams->is_scalar ? 0 : chunk_bytes;
L->dst_spad_half_size = chunk_bytes;
L->rows_per_buffer = 1;
L->src1_size = 0;
const size_t src0_total = n_threads * L->src0_bytes_per_thread;
const size_t src1_total = kparams->is_scalar ? 128 : (n_threads * L->src1_bytes_per_thread);
const size_t dst_total = n_threads * L->dst_bytes_per_thread;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, src0_total);
VTCM_LAYOUT_ALLOC(off, off_src1, src1_total);
VTCM_LAYOUT_ALLOC(off, off_dst, dst_total);
if (off > vtcm_size) {
return;
}
L->total_bytes = off;
return;
}
const size_t spad_row_total = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)
? 2 * (kparams->src0_row_size_aligned + (n_srcs - 1) * kparams->src1_row_size_aligned + kparams->dst_row_size_aligned)
: 2 * (kparams->src0_row_size_aligned + kparams->dst_row_size_aligned);
const size_t static_vtcm_size = kparams->src1_size + kparams->src2_size;
if (spad_row_total == 0 || vtcm_size < static_vtcm_size) {
return;
}
const size_t rows_per_buffer = (kparams->rows_per_buffer > 0)
? kparams->rows_per_buffer
: (vtcm_size - static_vtcm_size) / (n_threads * spad_row_total);
if (rows_per_buffer == 0) {
return;
}
L->rows_per_buffer = (uint32_t) rows_per_buffer;
L->src1_size = kparams->src1_size;
L->src2_size = kparams->src2_size;
L->src0_bytes_per_thread = rows_per_buffer * 2 * kparams->src0_row_size_aligned;
L->dst_bytes_per_thread = rows_per_buffer * 2 * kparams->dst_row_size_aligned;
L->src1_bytes_per_thread = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)
? rows_per_buffer * 2 * (n_srcs - 1) * kparams->src1_row_size_aligned
: 0;
L->src0_spad_half_size = L->src0_bytes_per_thread / 2;
L->src1_spad_half_size = (kparams->kernel_type == HTP_BINARY_KERNEL_SAME_SHAPE)
? rows_per_buffer * kparams->src1_row_size_aligned
: 0;
L->dst_spad_half_size = L->dst_bytes_per_thread / 2;
const size_t src0_total = n_threads * L->src0_bytes_per_thread;
const size_t src1_total = (kparams->src1_size > 0)
? kparams->src1_size
: n_threads * L->src1_bytes_per_thread;
const size_t src2_total = kparams->src2_size;
const size_t dst_total = n_threads * L->dst_bytes_per_thread;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, src0_total);
VTCM_LAYOUT_ALLOC(off, off_src1, src1_total);
VTCM_LAYOUT_ALLOC(off, off_src2, src2_total);
VTCM_LAYOUT_ALLOC(off, off_dst, dst_total);
L->total_bytes = off;
}
static inline bool htp_binary_solve_chunked(
struct htp_binary_vtcm_layout * L,
struct htp_binary_kernel_params * kparams,
uint32_t total_elems,
uint32_t elem_size,
uint32_t max_threads,
uint32_t n_srcs,
bool is_scalar,
size_t vtcm_size
) {
const size_t n_bufs = is_scalar ? 2 : (size_t) n_srcs + 1;
const size_t scalar_bytes = is_scalar ? 128 : 0;
const size_t budget_per_buf = (max_threads > 0 && vtcm_size > scalar_bytes)
? hex_align_down((vtcm_size - scalar_bytes) / (2 * max_threads * n_bufs), 128)
: 0;
const uint32_t max_chunk_bytes = (budget_per_buf > 0 && budget_per_buf < 32768) ? (uint32_t) budget_per_buf : 32768u;
const uint32_t max_chunk_elems = max_chunk_bytes / elem_size;
if (max_chunk_elems == 0) {
return false;
}
const uint32_t min_chunk_elems = 256 < max_chunk_elems ? 256 : max_chunk_elems;
const uint32_t target_chunk_elems = hex_round_up((total_elems + (2 * max_threads) - 1) / (2 * max_threads), 32);
const uint32_t chunk_size = (target_chunk_elems < min_chunk_elems) ? min_chunk_elems
: (target_chunk_elems > max_chunk_elems) ? max_chunk_elems
: target_chunk_elems;
const uint32_t chunk_bytes = hex_round_up(chunk_size * elem_size, 128);
const uint32_t total_chunks = (total_elems + chunk_size - 1) / chunk_size;
const uint32_t n_threads = (total_chunks >= 2) ? (max_threads < total_chunks ? max_threads : total_chunks) : 1;
kparams->kernel_type = HTP_BINARY_KERNEL_CHUNKED;
kparams->n_threads = n_threads;
kparams->n_srcs = (uint8_t) n_srcs;
kparams->rows_per_buffer = 1;
kparams->src1_row_size_aligned = is_scalar ? 0 : kparams->src0_row_size_aligned;
kparams->src1_size = 0;
kparams->chunk_size = chunk_size;
kparams->chunk_bytes = chunk_bytes;
kparams->is_scalar = is_scalar ? 1 : 0;
htp_binary_vtcm_layout_build(L, kparams, vtcm_size);
if (L->total_bytes == 0 || L->total_bytes > vtcm_size) {
return false;
}
kparams->vtcm_size = (uint32_t) L->total_bytes;
return true;
}
#endif
llama.cpp/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake
+155
View File
@@ -0,0 +1,155 @@
if (HEXAGON_TOOLCHAIN_INCLUDED)
return()
endif()
set(HEXAGON_TOOLCHAIN_INCLUDED true)
# Cross Compiling for Hexagon
set(HEXAGON TRUE)
set(CMAKE_SYSTEM_NAME QURT)
set(CMAKE_SYSTEM_PROCESSOR Hexagon)
set(CMAKE_SYSTEM_VERSION "1") #${HEXAGON_PLATFORM_LEVEL})
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
set(CUSTOM_RUNELF_PATH "")
if (NOT HEXAGON_SDK_ROOT)
set(HEXAGON_SDK_ROOT $ENV{HEXAGON_SDK_ROOT})
endif()
if (NOT HEXAGON_TOOLS_ROOT)
if (DEFINED ENV{HEXAGON_TOOLS_ROOT})
set(HEXAGON_TOOLS_ROOT $ENV{HEXAGON_TOOLS_ROOT})
endif()
if(NOT HEXAGON_TOOLS_ROOT)
set(HEXAGON_TOOLS_ROOT $ENV{DEFAULT_HEXAGON_TOOLS_ROOT})
endif()
endif()
file(TO_CMAKE_PATH "${HEXAGON_TOOLS_ROOT}" HEXAGON_TOOLS_ROOT)
file(TO_CMAKE_PATH "${HEXAGON_SDK_ROOT}" HEXAGON_SDK_ROOT)
if(CMAKE_HOST_SYSTEM_NAME STREQUAL Windows)
set(HEXAGON_TOOLCHAIN_SUFFIX .exe)
endif()
message(DEBUG "CMAKE_HOST_SYSTEM_NAME:${CMAKE_HOST_SYSTEM_NAME}")
include(${HEXAGON_SDK_ROOT}/build/cmake/hexagon_arch.cmake)
set(HEXAGON_TOOLCHAIN ${HEXAGON_TOOLS_ROOT})
set(HEXAGON_LIB_DIR "${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib")
set(HEXAGON_ISS_DIR ${HEXAGON_TOOLCHAIN}/Tools/lib/iss)
set(CMAKE_TRY_COMPILE_PLATFORM_VARIABLES
HEXAGON_SDK_ROOT
HEXAGON_TOOLS_ROOT
)
# QURT Related includes and linker flags
set(V_ARCH ${HEXAGON_ARCH})
set(_QURT_INSTALL_DIR "${HEXAGON_SDK_ROOT}/rtos/qurt/ADSP${V_ARCH}MP${V_ARCH_EXTN}")
set(_QURT_INSTALL_DIR "${HEXAGON_SDK_ROOT}/rtos/qurt/compute${V_ARCH}${V_ARCH_EXTN}")
if (${TREE} MATCHES PAKMAN)
set(_QURT_INSTALL_DIR "${QURT_IMAGE_DIR}/compute${V_ARCH}${V_ARCH_EXTN}")
endif()
message(DEBUG "_QURT_INSTALL_DIR:${_QURT_INSTALL_DIR}")
set(RTOS_DIR ${_QURT_INSTALL_DIR})
set(QCC_DIR "${HEXAGON_QCC_DIR}/${V_ARCH}/G0")
set(TARGET_DIR "${HEXAGON_LIB_DIR}/${V_ARCH}/G0")
include_directories(
${_QURT_INSTALL_DIR}/include
${_QURT_INSTALL_DIR}/include/qurt
${_QURT_INSTALL_DIR}/include/posix
)
set(QURT_START_LINK_LIBS)
set(QURT_START_LINK_LIBS
"${TARGET_DIR}/init.o"
"${RTOS_DIR}/lib/crt1.o"
"${RTOS_DIR}/lib/debugmon.o"
"${RTOS_DIR}/lib/libqurt.a"
"${TARGET_DIR}/libc.a"
"${TARGET_DIR}/libqcc.a"
"${TARGET_DIR}/libhexagon.a"
"${RTOS_DIR}/lib/libqurtcfs.a"
"${RTOS_DIR}/lib/libtimer_island.a"
"${RTOS_DIR}/lib/libtimer_main.a"
"${RTOS_DIR}/lib/libposix.a"
)
STRING(REPLACE ";" " " QURT_START_LINK_LIBS "${QURT_START_LINK_LIBS}")
set(QURT_END_LINK_LIBS ${TARGET_DIR}/fini.o)
# Non QURT related includes and linker flags
set(TARGET_DIR_NOOS "${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib/${HEXAGON_ARCH}")
if (NOT NO_WRAP_MEM_API)
set(WRAP_MALLOC -Wl,--wrap=malloc)
set(WRAP_CALLOC -Wl,--wrap=calloc)
set(WRAP_FREE -Wl,--wrap=free)
set(WRAP_REALLOC -Wl,--wrap=realloc)
set(WRAP_MEMALIGN -Wl,--wrap=memalign)
endif()
set(ARCH_FLAGS "-mcpu=${V_ARCH} -m${V_ARCH} -mhvx=${V_ARCH} -mhmx")
set(PIC_SHARED_LD_FLAGS
${ARCH_FLAGS}
-G0
-fpic
-Wl,-Bsymbolic
-Wl,-L${TARGET_DIR_NOOS}/G0/pic
-Wl,-L${HEXAGON_TOOLCHAIN}/Tools/target/hexagon/lib/
-Wl,--no-threads ${WRAP_MALLOC} ${WRAP_CALLOC} ${WRAP_FREE} ${WRAP_REALLOC} ${WRAP_MEMALIGN}
-shared
"-o <TARGET> <SONAME_FLAG><TARGET_SONAME>"
"<LINK_FLAGS>"
-Wl,--start-group
"<OBJECTS>"
"<LINK_LIBRARIES>"
-Wl,--end-group
-lc
)
STRING(REPLACE ";" " " PIC_SHARED_LD_FLAGS "${PIC_SHARED_LD_FLAGS}")
set(HEXAGON_PIC_SHARED_LINK_OPTIONS "${PIC_SHARED_LD_FLAGS}")
# System include paths
include_directories(SYSTEM ${HEXAGON_SDK_ROOT}/incs)
include_directories(SYSTEM ${HEXAGON_SDK_ROOT}/incs/stddef)
include_directories(SYSTEM ${HEXAGON_SDK_ROOT}/ipc/fastrpc/incs)
# LLVM toolchain setup
# Compiler paths, options and architecture
set(CMAKE_C_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang${HEXAGON_TOOLCHAIN_SUFFIX})
set(CMAKE_CXX_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang++${HEXAGON_TOOLCHAIN_SUFFIX})
set(CMAKE_AR ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-ar${HEXAGON_TOOLCHAIN_SUFFIX})
set(CMAKE_ASM_COMPILER ${HEXAGON_TOOLCHAIN}/Tools/bin/hexagon-clang++${HEXAGON_TOOLCHAIN_SUFFIX})
set(HEXAGON_LINKER ${CMAKE_C_COMPILER})
set(CMAKE_PREFIX_PATH ${HEXAGON_TOOLCHAIN}/Tools/target/hexagon)
set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "-Wl,-soname,")
set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG "-Wl,-soname,")
# Compiler Options
set(COMMON_FLAGS "${ARCH_FLAGS} -fvectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}")
set(CMAKE_CXX_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g")
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O2 -g")
set(CMAKE_CXX_FLAGS_RELEASE "${COMMON_FLAGS} -O2")
set(CMAKE_C_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g")
set(CMAKE_C_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O2 -g")
set(CMAKE_C_FLAGS_RELEASE "${COMMON_FLAGS} -O2")
set(CMAKE_ASM_FLAGS_DEBUG "${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_DEBUG}")
set(CMAKE_ASM_FLAGS_RELEASE "${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE}")
set(CMAKE_ASM_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}" )
#Linker Options
set(CMAKE_C_CREATE_SHARED_LIBRARY "${HEXAGON_LINKER} ${HEXAGON_PIC_SHARED_LINK_OPTIONS}")
set(CMAKE_CXX_CREATE_SHARED_LIBRARY "${HEXAGON_LINKER} ${HEXAGON_PIC_SHARED_LINK_OPTIONS}")
llama.cpp/ggml/src/ggml-hexagon/htp/concat-ops.c
+349
View File
@@ -0,0 +1,349 @@
#include "concat-ops.h"
#include "dma-queue.h"
#include "hex-common.h"
#include "dma-copy.h"
#include "hex-fastdiv.h"
#include "hex-profile.h"
#include "hexagon_protos.h"
#include "hexagon_types.h"
#include "htp-ctx.h"
#include "htp-fence.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
#include "hvx-utils.h"
#include "hvx_hexagon_protos.h"
#include "gather-rows.h"
#include <string.h>
struct htp_concat_context {
struct htp_ops_context * octx;
uint8_t * spad0_base;
uint8_t * spad1_base;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
uint32_t row_start;
uint32_t nrows;
uint32_t nrows_per_thread;
uint32_t nplanes;
struct fastdiv_values div_ne2;
};
static inline dma_addr_t concat_plane_addr(const struct htp_tensor * t, uint32_t p, const struct fastdiv_values * div_ne2, uint32_t ne2) {
const uint32_t i3 = fastdiv(p, div_ne2);
const uint32_t i2 = p - i3 * ne2;
return t->data + i2 * t->nb[2] + i3 * t->nb[3];
}
static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void * data) {
struct htp_concat_context * cctx = (struct htp_concat_context *) data;
struct htp_ops_context * octx = cctx->octx;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
const uint32_t src0_ne0 = src0->ne[0];
const uint32_t src1_ne0 = src1->ne[0];
const uint32_t row_end = cctx->row_start + cctx->nrows;
const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread;
const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end;
if (start_i >= end_i) return;
dma_queue * dma_q = octx->ctx->dma[ith];
uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
const uint32_t block_i = 32;
const uint32_t spad1_stride = block_i * sizeof(float);
const HVX_Vector offsets = hvx_vec_gather_offsets_w(spad1_stride);
const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 32);
const uint32_t src0_row_bytes = src0_ne0 * sizeof(float);
const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(float);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
const uint32_t ne2 = dst->ne[2];
uint32_t p = 0;
uint32_t i = start_i;
const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1];
dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(float), src1_ne0);
const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1];
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i));
dma_queue_pop(dma_q); // src1
while (p < cctx->nplanes) {
const uint32_t current_block_i = MIN(end_i - i, block_i);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
for (uint32_t j = 0; j < src1_ne0; j += 32) {
const uint8_t * src_ptr = spad1_base + j * spad1_stride;
uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(float);
hvx_transpose_32x32_w_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 32));
}
hvx_gather_sync(spad0_base + src0_row_padded);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
uint32_t np = p;
uint32_t ni = i + block_i;
if (ni >= end_i) {
ni = start_i;
np++;
}
const bool has_next = np < cctx->nplanes;
const uint32_t next_block_i = MIN(end_i - ni, block_i);
// spad1 is free after the gather sync, prefetch next src1 ahead of the dst write
if (has_next) {
const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1];
dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(float), src1_ne0);
}
dma_queue_pop(dma_q); // src0
const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1];
dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(float), current_block_i);
if (has_next) {
dma_queue_pop(dma_q); // next src1
}
dma_queue_pop(dma_q); // dst
// spad0 is free after the dst write
if (has_next) {
const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1];
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i);
}
p = np;
i = ni;
}
dma_queue_flush(dma_q);
}
static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * data) {
struct htp_concat_context * cctx = (struct htp_concat_context *) data;
struct htp_ops_context * octx = cctx->octx;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
const uint32_t src0_ne0 = src0->ne[0];
const uint32_t src1_ne0 = src1->ne[0];
const uint32_t row_end = cctx->row_start + cctx->nrows;
const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread;
const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end;
if (start_i >= end_i) return;
dma_queue * dma_q = octx->ctx->dma[ith];
uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
const uint32_t block_i = 64;
const uint32_t spad1_stride = block_i * sizeof(__fp16);
const HVX_Vector offsets = hvx_vec_gather_offsets_h(spad1_stride);
const uint32_t src1_ne0_padded = hex_round_up(src1_ne0, 64);
const uint32_t src0_row_bytes = src0_ne0 * sizeof(__fp16);
const uint32_t src0_row_padded = hex_round_up(src0_row_bytes, VLEN);
const uint32_t src0_pre = src0_row_padded - src0_row_bytes;
const uint32_t spad0_row_bytes = src0_row_padded + src1_ne0_padded * sizeof(__fp16);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
const struct fastdiv_values * div_ne2 = &cctx->div_ne2;
const uint32_t ne2 = dst->ne[2];
uint32_t p = 0;
uint32_t i = start_i;
const dma_addr_t src1_addr = concat_plane_addr(src1, p, div_ne2, ne2) + i * src1->nb[1];
dma_queue_push(dma_q, dma_make_data(spad1_base, src1_addr), spad1_stride, src1->nb[0], MIN(end_i - i, block_i) * sizeof(__fp16), src1_ne0);
const dma_addr_t src0_addr = concat_plane_addr(src0, p, div_ne2, ne2) + i * src0->nb[1];
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, src0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, MIN(end_i - i, block_i));
dma_queue_pop(dma_q); // src1
while (p < cctx->nplanes) {
const uint32_t current_block_i = MIN(end_i - i, block_i);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
for (uint32_t j = 0; j < src1_ne0; j += 64) {
const uint8_t * src_ptr = spad1_base + j * spad1_stride;
uint8_t * dst_ptr = spad0_base + src0_row_padded + j * sizeof(__fp16);
hvx_transpose_64x64_h_gather(dst_ptr, spad0_row_bytes, src_ptr, spad1_stride, offsets, current_block_i, MIN(src1_ne0 - j, 64));
}
hvx_gather_sync(spad0_base + src0_row_padded);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
uint32_t np = p;
uint32_t ni = i + block_i;
if (ni >= end_i) {
ni = start_i;
np++;
}
const bool has_next = np < cctx->nplanes;
const uint32_t next_block_i = MIN(end_i - ni, block_i);
// spad1 is free after the gather sync, prefetch next src1 ahead of the dst write
if (has_next) {
const dma_addr_t nsrc1_addr = concat_plane_addr(src1, np, div_ne2, ne2) + ni * src1->nb[1];
dma_queue_push(dma_q, dma_make_data(spad1_base, nsrc1_addr), spad1_stride, src1->nb[0], next_block_i * sizeof(__fp16), src1_ne0);
}
dma_queue_pop(dma_q); // src0
const dma_addr_t dst_addr = concat_plane_addr(dst, p, div_ne2, ne2) + i * dst->nb[1];
dma_queue_push(dma_q, dma_make_data(dst_addr, spad0_base + src0_pre), dst->nb[1], spad0_row_bytes, (src0_ne0 + src1_ne0) * sizeof(__fp16), current_block_i);
if (has_next) {
dma_queue_pop(dma_q); // next src1
}
dma_queue_pop(dma_q); // dst
// spad0 is free after the dst write
if (has_next) {
const dma_addr_t nsrc0_addr = concat_plane_addr(src0, np, div_ne2, ne2) + ni * src0->nb[1];
dma_queue_push(dma_q, dma_make_data(spad0_base + src0_pre, nsrc0_addr), spad0_row_bytes, src0->nb[1], src0_row_bytes, next_block_i);
}
p = np;
i = ni;
}
dma_queue_flush(dma_q);
}
static int concat_regular(struct htp_ops_context * octx, int dim, uint32_t type_size) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
struct htp_tensor view0 = *dst;
struct htp_tensor view1 = *dst;
for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {
view0.ne[d] = src0->ne[d];
view1.ne[d] = src1->ne[d];
}
view1.data += src0->ne[dim] * dst->nb[dim];
const uint32_t total_rows_0 = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t total_rows_1 = src1->ne[1] * src1->ne[2] * src1->ne[3];
uint32_t rstart0 = 0, nrows0 = total_rows_0;
uint32_t rstart1 = 0, nrows1 = total_rows_1;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range0 = htp_tensor_mdev_partition(
total_rows_0, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
rstart0 = range0.start;
nrows0 = range0.count;
const struct htp_tensor_mdev_range range1 = htp_tensor_mdev_partition(
total_rows_1, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
rstart1 = range1.start;
nrows1 = range1.count;
}
dma_queue * q = octx->ctx->dma[0];
dma_cpy_sametype_sameshape_range(q, &view0, src0, type_size, rstart0, nrows0);
dma_cpy_sametype_sameshape_range(q, &view1, src1, type_size, rstart1, nrows1);
dma_queue_flush(q);
return HTP_STATUS_OK;
}
static int concat_transposed(struct htp_ops_context * octx, const struct htp_concat_kernel_params * kparams, uint32_t type_size) {
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
const struct htp_tensor * dst = octx->dst;
const uint32_t total_rows = dst->ne[1];
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0 || dst->ne[2] == 0 || dst->ne[3] == 0) {
return HTP_STATUS_OK;
}
if (kparams->vtcm_size > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
const uint32_t n_threads = octx->n_threads;
struct htp_concat_transposed_vtcm_layout layout;
htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
struct htp_concat_context cctx;
cctx.octx = octx;
cctx.spad0_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad0);
cctx.spad1_base = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, layout.off_spad1);
cctx.spad0_size_per_thread = layout.src0_spad_size_per_thread;
cctx.spad1_size_per_thread = layout.src1_spad_size_per_thread;
cctx.row_start = row_start;
cctx.nrows = nrows;
cctx.nplanes = dst->ne[2] * dst->ne[3];
cctx.div_ne2 = kparams->u.transposed.div_ne2;
cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
work_queue_func_t worker_func = (type_size == 4) ? concat_2d_f32_transposed : concat_2d_f16_transposed;
work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads);
return HTP_STATUS_OK;
}
int op_concat(struct htp_ops_context * octx) {
const struct htp_concat_kernel_params * kparams = (const struct htp_concat_kernel_params *) octx->kernel_params;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
int status = HTP_STATUS_OK;
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
switch (kparams->kernel_type) {
case HTP_CONCAT_KERNEL_REGULAR:
status = concat_regular(octx, kparams->dim, type_size);
break;
case HTP_CONCAT_KERNEL_TRANSPOSED:
status = concat_transposed(octx, kparams, type_size);
break;
case HTP_CONCAT_KERNEL_GATHER_ROWS:
status = htp_gather_rows(octx, src0, src1, &kparams->u.gather, kparams->n_threads);
break;
default:
status = HTP_STATUS_NO_SUPPORT;
break;
}
htp_ops_context_set_status(octx, status);
return octx->status;
}
llama.cpp/ggml/src/ggml-hexagon/htp/concat-ops.h
+70
View File
@@ -0,0 +1,70 @@
#ifndef HTP_CONCAT_OPS_H
#define HTP_CONCAT_OPS_H
#include "hex-common.h"
#include "hex-fastdiv.h"
#include "htp-vtcm.h"
#include "gather-rows.h"
#include <stdint.h>
enum htp_concat_kernel_type {
HTP_CONCAT_KERNEL_UNSUPPORTED = 0,
HTP_CONCAT_KERNEL_REGULAR = 1,
HTP_CONCAT_KERNEL_TRANSPOSED = 2,
HTP_CONCAT_KERNEL_GATHER_ROWS = 3,
};
struct htp_concat_transposed_params {
struct fastdiv_values div_ne2;
};
struct htp_concat_kernel_params {
uint8_t kernel_type;
uint8_t dim;
uint8_t n_threads;
uint8_t pad;
uint32_t vtcm_size;
union {
struct htp_concat_transposed_params transposed;
struct htp_gather_rows_params gather;
} u;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kernel_params is too large for kernel_params blob");
#endif
struct htp_concat_transposed_vtcm_layout {
uint32_t src0_spad_size_per_thread;
uint32_t src1_spad_size_per_thread;
size_t off_spad0;
size_t off_spad1;
uint32_t total_bytes;
};
static inline void htp_concat_transposed_vtcm_layout_build(
struct htp_concat_transposed_vtcm_layout * L,
uint32_t src0_ne0,
uint32_t src1_ne0,
uint32_t type_size,
uint32_t n_threads) {
uint32_t block_i = (type_size == 4) ? 32 : 64;
uint32_t spad1_stride = block_i * type_size;
uint32_t src1_ne0_padded = hex_round_up(src1_ne0, block_i);
uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + src1_ne0_padded * type_size;
L->src0_spad_size_per_thread = block_i * spad0_row_bytes;
L->src1_spad_size_per_thread = src1_ne0_padded * spad1_stride;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_spad0, (size_t) n_threads * L->src0_spad_size_per_thread);
VTCM_LAYOUT_ALLOC(off, off_spad1, (size_t) n_threads * L->src1_spad_size_per_thread);
L->total_bytes = (uint32_t) off;
}
#endif // HTP_CONCAT_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/cpy-ops.c
+488
View File
@@ -0,0 +1,488 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic ignored "-Wunused-function"
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <qurt_memory.h>
#include <math.h>
#include <string.h>
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "cpy-ops.h"
#include "dma-copy.h"
#include "htp-ctx.h"
#include "htp-fence.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-utils.h"
#include "gather-rows.h"
struct htp_copy_context {
struct htp_ops_context * octx;
const struct htp_copy_kernel_params * kparams;
uint32_t row_start;
uint32_t nrows;
uint32_t src0_nrows_per_thread;
uint32_t elem_start;
uint32_t nelem;
uint32_t elem_per_thread;
struct htp_copy_convert_vtcm_layout layout;
uint8_t * vtcm_src0;
uint8_t * vtcm_dst;
};
#define cpy_preamble \
const struct htp_tensor *src0 = octx->src[0]; \
const struct htp_tensor *dst = octx->dst; \
\
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
const uint32_t ne02 = src0->ne[2]; \
const uint32_t ne03 = src0->ne[3]; \
\
const uint32_t nb00 = src0->nb[0]; \
const uint32_t nb01 = src0->nb[1]; \
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb0 = dst->nb[0]; \
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3];
#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
const uint32_t th_nelem = ct->elem_per_thread; \
const uint32_t th_start = ct->elem_start + ith * th_nelem; \
const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \
if (th_start >= th_end) return; \
\
dma_queue * dma_q = octx->ctx->dma[ith]; \
\
const uint32_t ne01_ne00 = ne01 * ne00; \
const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \
const uint32_t ne1_ne0 = ne1 * ne0; \
const uint32_t ne2_ne1_ne0 = ne2 * ne1_ne0; \
\
const struct htp_copy_reshape_params * rsh = &ct->kparams->u.reshape; \
uint32_t e = th_start; \
uint32_t i13 = fastdiv(e, &rsh->div_ne2_ne1_ne0); \
uint32_t rem = e - i13 * ne2_ne1_ne0; \
uint32_t i12 = fastdiv(rem, &rsh->div_ne1_ne0); \
uint32_t rem2 = rem - i12 * ne1_ne0; \
uint32_t i11 = fastdiv(rem2, &rsh->div_ne0); \
uint32_t i10 = rem2 - i11 * ne0; \
\
uint32_t i03 = fastdiv(e, &rsh->div_ne02_ne01_ne00); \
uint32_t rem_s = e - i03 * ne02_ne01_ne00; \
uint32_t i02 = fastdiv(rem_s, &rsh->div_ne01_ne00); \
uint32_t rem2_s = rem_s - i02 * ne01_ne00; \
uint32_t i01 = fastdiv(rem2_s, &rsh->div_ne00); \
uint32_t i00 = rem2_s - i01 * ne00; \
\
dma_addr_t dst_addr = dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \
dma_addr_t src0_addr = src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \
\
const bool rows_contig = (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \
\
while (e < th_end) { \
const uint32_t run = MIN(MIN(ne00 - i00, ne0 - i10), th_end - e); \
if (rows_contig) { \
dma_cpy_sametype_reshape_contig(dma_q, dst_addr, src0_addr, run * ELEM_SIZE); \
} else { \
dma_cpy_push_2d_chunked(dma_q, dst_addr, src0_addr, nb0, nb00, ELEM_SIZE, run); \
} \
e += run; \
\
dst_addr += run * nb0; \
i10 += run; \
if (i10 == ne0) { \
i10 = 0; \
if (++i11 == ne1) { \
i11 = 0; \
if (++i12 == ne2) { \
i12 = 0; \
i13++; \
} \
} \
dst_addr = dst->data + i11*nb1 + i12*nb2 + i13*nb3; \
} \
\
src0_addr += run * nb00; \
i00 += run; \
if (i00 == ne00) { \
i00 = 0; \
if (++i01 == ne01) { \
i01 = 0; \
if (++i02 == ne02) { \
i02 = 0; \
i03++; \
} \
} \
src0_addr = src0->data + i01*nb01 + i02*nb02 + i03*nb03; \
} \
} \
dma_queue_flush(dma_q); \
}
DEFINE_CPY_RESHAPE(f32, float, 4)
DEFINE_CPY_RESHAPE(f16, __fp16, 2)
DEFINE_CPY_RESHAPE(i32, int32_t, 4)
struct cpy_row_cursor {
uint32_t r;
uint32_t i01;
uint32_t i02;
uint32_t i03;
};
static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uint32_t ne01, uint32_t ne02) {
c->r += n;
c->i01 += n;
if (c->i01 == ne01) {
c->i01 = 0;
if (++c->i02 == ne02) {
c->i02 = 0;
c->i03++;
}
}
}
#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
\
const uint32_t dr = ct->src0_nrows_per_thread; \
const uint32_t ir0 = ct->row_start + dr * ith; \
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \
if (ir0 >= ir1) return; \
\
dma_queue * dma_q = octx->ctx->dma[ith]; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \
const struct htp_copy_convert_vtcm_layout * L = &ct->layout; \
const uint32_t src0_buf_size = L->src0_buf_size; \
const uint32_t dst_buf_size = L->dst_buf_size; \
const uint32_t src0_row_stride = L->src0_row_stride; \
const uint32_t dst_row_stride = L->dst_row_stride; \
const uint32_t blk_rows = L->blk_rows; \
uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * L->spad0_size_per_thread; \
uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * L->spad1_size_per_thread; \
const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \
const uint32_t dst_row_size = ne00 * ct->kparams->dst_type_size; \
\
const uint32_t ne02_ne01 = ne02 * ne01; \
uint32_t i03 = fastdiv(ir0, &cvt->div_ne02_ne01); \
uint32_t rem = ir0 - i03 * ne02_ne01; \
uint32_t i02 = fastdiv(rem, &cvt->div_ne01); \
uint32_t i01 = rem - i02 * ne01; \
\
struct cpy_row_cursor f = { ir0, i01, i02, i03 }; \
struct cpy_row_cursor c = f; \
\
for (uint32_t b = 0; b < 2 && f.r < ir1; ++b) { \
uint8_t * src_spad = vtcm_src0_base + b * src0_buf_size; \
uint8_t * dst_spad = vtcm_dst_base + b * dst_buf_size; \
const uint32_t nr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \
dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \
dma_queue_push(dma_q, dma_make_data(dst->data, dst_spad), \
dst_row_size, dst_buf_size, dst_row_size, 0); \
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
src0_row_stride, nb01, src0_row_size, nr); \
cpy_cursor_advance(&f, nr, ne01, ne02); \
} \
\
for (uint32_t blk = 0; c.r < ir1; ++blk) { \
const uint32_t nr = MIN(blk_rows, MIN(ne01 - c.i01, ir1 - c.r)); \
uint8_t * dst_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; \
uint8_t * src_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \
for (uint32_t r = 0; r < nr; ++r) { \
CONV_FUNC(dst_spad + r * dst_row_stride, src_spad + r * src0_row_stride, ne00); \
} \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) blk); \
\
dma_addr_t c_dst_addr = dst->data + c.i01*nb1 + c.i02*nb2 + c.i03*nb3; \
dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \
nb1, dst_row_stride, dst_row_size, nr); \
cpy_cursor_advance(&c, nr, ne01, ne02); \
\
if (f.r < ir1) { \
const uint32_t fnr = MIN(blk_rows, MIN(ne01 - f.i01, ir1 - f.r)); \
dma_addr_t f_src0_addr = src0->data + f.i01*nb01 + f.i02*nb02 + f.i03*nb03; \
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
src0_row_stride, nb01, src0_row_size, fnr); \
cpy_cursor_advance(&f, fnr, ne01, ne02); \
} \
} \
dma_queue_flush(dma_q); \
}
DEFINE_CPY_CONVERT_SAMESHAPE(f16_f32, hvx_copy_f16_f32_aa)
DEFINE_CPY_CONVERT_SAMESHAPE(f32_f16, hvx_copy_f32_f16_aa)
DEFINE_CPY_CONVERT_SAMESHAPE(i32_f32, hvx_copy_i32_f32_aa)
DEFINE_CPY_CONVERT_SAMESHAPE(f32_i32, hvx_copy_f32_i32_aa)
static int cpy_scalar(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) {
return HTP_STATUS_OK;
}
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
if (src0->type == dst->type) {
dma_cpy_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, kparams->src0_type_size);
dma_queue_flush(octx->ctx->dma[0]);
return HTP_STATUS_OK;
}
dma_queue * dma_q = octx->ctx->dma[0];
uint8_t * s_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, 0);
uint8_t * d_ptr = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, VLEN);
dma_addr_t s_vtcm = (dma_addr_t)(uintptr_t) s_ptr;
dma_addr_t d_vtcm = (dma_addr_t)(uintptr_t) d_ptr;
const uint32_t s_size = kparams->src0_type_size;
const uint32_t d_size = kparams->dst_type_size;
dma_queue_push(dma_q, dma_make_data(s_vtcm, src0->data), s_size, s_size, s_size, 1);
dma_queue_pop(dma_q);
const HVX_Vector v_src = hvx_vmem(s_ptr);
HVX_Vector v_dst;
if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_I32) {
v_dst = Q6_Vw_equals_Vsf(v_src);
} else if (src0->type == HTP_TYPE_I32 && dst->type == HTP_TYPE_F32) {
v_dst = Q6_Vsf_equals_Vw(v_src);
} else if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_F16) {
v_dst = hvx_vec_f32_to_f16(v_src, v_src);
} else if (src0->type == HTP_TYPE_F16 && dst->type == HTP_TYPE_F32) {
v_dst = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src));
} else {
return HTP_STATUS_NO_SUPPORT;
}
hvx_vmem(d_ptr) = v_dst;
dma_queue_push(dma_q, dma_make_data(dst->data, d_vtcm), d_size, d_size, d_size, 1);
dma_queue_flush(dma_q);
return HTP_STATUS_OK;
}
static int cpy_1d_contig(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
uint32_t elem_start = 0;
uint32_t nelem = kparams->total_elems;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
kparams->total_elems, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
elem_start = range.start;
nelem = range.count;
}
if (nelem > 0) {
dma_queue * q = octx->ctx->dma[0];
const uint32_t type_size = kparams->src0_type_size;
dma_addr_t dst_addr = dst->data + elem_start * type_size;
dma_addr_t src0_addr = src0->data + elem_start * type_size;
dma_cpy_sametype_reshape_contig(q, dst_addr, src0_addr, nelem * type_size);
dma_queue_flush(q);
}
return HTP_STATUS_OK;
}
static int cpy_sameshape_sametype(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
uint32_t row_start = 0;
uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
kparams->total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows > 0) {
dma_queue * q = octx->ctx->dma[0];
dma_cpy_sametype_sameshape_range(q, dst, src0, kparams->src0_type_size, row_start, nrows);
dma_queue_flush(q);
}
return HTP_STATUS_OK;
}
static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
uint32_t row_start = 0;
uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
kparams->total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
if (kparams->vtcm_size > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
const uint32_t n_threads = octx->n_threads;
const struct htp_copy_convert_params * cvt = &kparams->u.convert;
struct htp_copy_context ct;
htp_copy_convert_vtcm_layout_build(&ct.layout, src0->ne[0], kparams->src0_type_size, kparams->dst_type_size, cvt->blk_rows, n_threads);
ct.octx = octx;
ct.kparams = kparams;
ct.row_start = row_start;
ct.nrows = nrows;
ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
ct.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_src0);
ct.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, ct.layout.off_dst);
work_queue_func_t copy_fun = NULL;
if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) {
copy_fun = cpy_thread_f16_f32_sameshape;
} else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) {
copy_fun = cpy_thread_f32_f16_sameshape;
} else if (dst->type == HTP_TYPE_I32 && src0->type == HTP_TYPE_F32) {
copy_fun = cpy_thread_i32_f32_sameshape;
} else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_I32) {
copy_fun = cpy_thread_f32_i32_sameshape;
} else {
return HTP_STATUS_NO_SUPPORT;
}
work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
return HTP_STATUS_OK;
}
static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
uint32_t elem_start = 0;
uint32_t nelem = kparams->total_elems;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
kparams->total_elems, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
elem_start = range.start;
nelem = range.count;
}
if (nelem == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
struct htp_copy_context ct;
ct.octx = octx;
ct.kparams = kparams;
ct.elem_start = elem_start;
ct.nelem = nelem;
ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div);
work_queue_func_t copy_fun = NULL;
switch (src0->type) {
case HTP_TYPE_F32: copy_fun = cpy_thread_f32_reshape; break;
case HTP_TYPE_F16: copy_fun = cpy_thread_f16_reshape; break;
case HTP_TYPE_I32: copy_fun = cpy_thread_i32_reshape; break;
default: return HTP_STATUS_NO_SUPPORT;
}
work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
return HTP_STATUS_OK;
}
int op_cpy(struct htp_ops_context * octx) {
const struct htp_copy_kernel_params * kparams = (const struct htp_copy_kernel_params *) octx->kernel_params;
int status = HTP_STATUS_OK;
switch (kparams->kernel_type) {
case HTP_COPY_KERNEL_SCALAR:
status = cpy_scalar(octx, kparams);
break;
case HTP_COPY_KERNEL_1D_CONTIG:
status = cpy_1d_contig(octx, kparams);
break;
case HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE:
status = cpy_sameshape_sametype(octx, kparams);
break;
case HTP_COPY_KERNEL_SAMESHAPE_CONVERT:
status = cpy_sameshape_convert(octx, kparams);
break;
case HTP_COPY_KERNEL_RESHAPE:
status = cpy_reshape(octx, kparams);
break;
case HTP_COPY_KERNEL_GATHER_ROWS:
status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads);
break;
default:
status = HTP_STATUS_NO_SUPPORT;
break;
}
htp_ops_context_set_status(octx, status);
if (octx->op == HTP_OP_CPY_FENCE) {
if (octx->ctx->mdev.count > 1) {
htp_mdev_group_barrier(octx);
}
if (octx->ctx->mdev.idx == 0) {
const struct htp_tensor * sync = octx->src[1];
const uint32_t seq = (uint32_t) octx->op_params[0];
atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data;
htp_fence_write(sync_fence, seq, octx->status);
FARF(HIGH, "ggml-hex: sync-release : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status);
}
}
return octx->status;
}
llama.cpp/ggml/src/ggml-hexagon/htp/cpy-ops.h
+125
View File
@@ -0,0 +1,125 @@
#ifndef HTP_CPY_OPS_H
#define HTP_CPY_OPS_H
#include "hex-common.h"
#include "hex-fastdiv.h"
#include "htp-vtcm.h"
#include "gather-rows.h"
#include <stdint.h>
enum htp_copy_kernel_type {
HTP_COPY_KERNEL_UNSUPPORTED = 0,
HTP_COPY_KERNEL_1D_CONTIG = 1,
HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE = 2,
HTP_COPY_KERNEL_SAMESHAPE_CONVERT = 3,
HTP_COPY_KERNEL_RESHAPE = 4,
HTP_COPY_KERNEL_SCALAR = 5,
HTP_COPY_KERNEL_GATHER_ROWS = 6,
};
struct htp_copy_convert_params {
uint32_t blk_rows; // rows moved per DMA descriptor
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02_ne01;
};
struct htp_copy_reshape_params {
struct fastdiv_values div_ne0;
struct fastdiv_values div_ne1_ne0;
struct fastdiv_values div_ne2_ne1_ne0;
struct fastdiv_values div_ne00;
struct fastdiv_values div_ne01_ne00;
struct fastdiv_values div_ne02_ne01_ne00;
};
struct htp_copy_kernel_params {
uint8_t kernel_type;
uint8_t src0_type_size;
uint8_t dst_type_size;
uint8_t n_threads;
uint32_t total_elems;
uint32_t total_rows;
uint32_t vtcm_size;
union {
struct htp_copy_convert_params convert;
struct htp_copy_reshape_params reshape;
struct htp_gather_rows_params gather;
} u;
};
// Size of one VTCM staging buffer: big enough that a DMA descriptor moves many short
// rows at once. The copy kernels keep four per thread (two in flight each way), about
// 1 MB of VTCM with 8 threads.
#define HTP_COPY_VTCM_BLK_BYTES (32 * 1024)
struct htp_copy_convert_vtcm_layout {
size_t total_bytes;
size_t off_src0;
size_t off_dst;
uint32_t src0_row_stride;
uint32_t dst_row_stride;
uint32_t blk_rows;
uint32_t src0_buf_size;
uint32_t dst_buf_size;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
};
static inline void htp_copy_convert_vtcm_layout_build(
struct htp_copy_convert_vtcm_layout * L,
uint32_t ne00,
uint32_t src_type_size,
uint32_t dst_type_size,
uint32_t blk_rows,
uint32_t n_threads) {
const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
L->src0_row_stride = src0_row_stride;
L->dst_row_stride = dst_row_stride;
L->blk_rows = blk_rows;
L->src0_buf_size = blk_rows * src0_row_stride;
L->dst_buf_size = blk_rows * dst_row_stride;
L->spad0_size_per_thread = 2 * L->src0_buf_size;
L->spad1_size_per_thread = 2 * L->dst_buf_size;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, (size_t) n_threads * L->spad0_size_per_thread);
VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) n_threads * L->spad1_size_per_thread);
L->total_bytes = off;
}
// ne01 bounds a block: rows of one block share the dim-1 stride.
static inline void htp_copy_convert_solve_layout(
struct htp_copy_convert_vtcm_layout * layout,
uint32_t ne00,
uint32_t ne01,
uint32_t src_type_size,
uint32_t dst_type_size,
uint32_t n_threads) {
const uint32_t src0_row_stride = hex_round_up(ne00 * src_type_size, 256);
const uint32_t dst_row_stride = hex_round_up(ne00 * dst_type_size, 256);
const uint32_t max_row = src0_row_stride > dst_row_stride ? src0_row_stride : dst_row_stride;
uint32_t blk_rows = HTP_COPY_VTCM_BLK_BYTES / max_row;
if (blk_rows > ne01) {
blk_rows = ne01;
}
if (blk_rows == 0) {
blk_rows = 1;
}
htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads);
}
#if defined(__cplusplus)
static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_CPY_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/cumsum-ops.c
+290
View File
@@ -0,0 +1,290 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic ignored "-Wunused-function"
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
#include "hvx-types.h"
#include "hvx-utils.h"
#include "dma-queue.h"
#include "cumsum-ops.h"
#define htp_cumsum_tensors_preamble \
const struct htp_tensor * restrict src0 = octx->src[0]; \
const struct htp_tensor * restrict dst = octx->dst; \
\
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
const uint32_t ne02 = src0->ne[2]; \
const uint32_t ne03 = src0->ne[3]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb00 = src0->nb[0]; \
const uint32_t nb01 = src0->nb[1]; \
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t nb0 = dst->nb[0]; \
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3];
struct htp_cumsum_context {
struct htp_ops_context * octx;
size_t src_row_size;
size_t dst_row_size;
size_t src_row_size_aligned;
size_t dst_row_size_aligned;
uint32_t rows_per_thread;
uint32_t total_rows;
uint32_t row_start;
};
#define htp_cumsum_preamble \
struct htp_cumsum_context * cctx = (struct htp_cumsum_context *) data; \
struct htp_ops_context * octx = cctx->octx; \
htp_cumsum_tensors_preamble; \
dma_queue * dma_q = octx->ctx->dma[ith];
// ---------------------------------------------------------------------------
// HVX prefix scan helpers
// ---------------------------------------------------------------------------
#if __HVX_ARCH__ > 75
static inline HVX_Vector hvx_cumsum_vadd(HVX_Vector a, HVX_Vector b) {
return Q6_Vsf_vadd_VsfVsf(a, b);
}
#else
static inline HVX_Vector hvx_cumsum_vadd(HVX_Vector a, HVX_Vector b) {
return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b));
}
#endif // __HVX_ARCH__ > 75
static inline HVX_Vector hvx_prefix_scan_f32(HVX_Vector v, HVX_Vector carry_in) {
const HVX_Vector zero = Q6_V_vsplat_R(0);
v = hvx_cumsum_vadd(v, Q6_V_vlalign_VVR(v, zero, 4));
v = hvx_cumsum_vadd(v, Q6_V_vlalign_VVR(v, zero, 8));
v = hvx_cumsum_vadd(v, Q6_V_vlalign_VVR(v, zero, 16));
v = hvx_cumsum_vadd(v, Q6_V_vlalign_VVR(v, zero, 32));
v = hvx_cumsum_vadd(v, Q6_V_vlalign_VVR(v, zero, 64));
v = hvx_cumsum_vadd(v, carry_in);
return v;
}
static inline HVX_Vector hvx_splat_last_f32(HVX_Vector v) {
return hvx_vec_repl4(Q6_V_vror_VR(v, 124));
}
static inline void hvx_cumsum_row_f32(const float * restrict src, float * restrict dst, uint32_t n) {
const uint32_t nvec = n >> 5;
const uint32_t nloe = n & (VLEN_FP32 - 1);
HVX_Vector carry = Q6_V_vsplat_R(0);
for (uint32_t i = 0; i < nvec; i++) {
HVX_Vector v = *((const HVX_UVector *) (src + i * VLEN_FP32));
v = hvx_prefix_scan_f32(v, carry);
hvx_vec_store_u(dst + i * VLEN_FP32, VLEN, v);
carry = hvx_splat_last_f32(v);
}
if (nloe) {
float acc = hvx_vec_get_f32(carry);
const float * src_tail = src + nvec * VLEN_FP32;
float * dst_tail = dst + nvec * VLEN_FP32;
for (uint32_t i = 0; i < nloe; i++) {
acc += src_tail[i];
dst_tail[i] = acc;
}
}
}
// ---------------------------------------------------------------------------
// Per thread worker: Double-buffered DMA
// ---------------------------------------------------------------------------
static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) {
htp_cumsum_preamble;
const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows);
if (ir0 >= ir1) {
return;
}
const size_t src_row_size = cctx->src_row_size;
const size_t dst_row_size = cctx->dst_row_size;
const size_t src_row_size_aligned = cctx->src_row_size_aligned;
const size_t dst_row_size_aligned = cctx->dst_row_size_aligned;
const dma_addr_t src_data = src0->data;
const dma_addr_t dst_data = dst->data;
uint8_t * src_spad = octx->src0_spad.data + (ith * src_row_size_aligned * 2);
uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned * 2);
for (uint32_t ir = ir0, spad_idx = 0; ir < ir1 && spad_idx < 2; ir++, spad_idx++) {
// Dummy dst writeback to establish queue ordering
dma_queue_push(dma_q,
dma_make_data(dst_data, dst_spad + (spad_idx * dst_row_size_aligned)),
dst_row_size, dst_row_size_aligned, dst_row_size, 0);
dma_queue_push(dma_q,
dma_make_data(src_spad + (spad_idx * src_row_size_aligned),
src_data + (ir * src_row_size)),
src_row_size_aligned, src_row_size, src_row_size, 1);
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = ir0; ir < ir1; ir++) {
float * dst_spad_row = (float *) dma_queue_pop(dma_q).src;
float * src_spad_row = (float *) dma_queue_pop(dma_q).dst;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
hvx_cumsum_row_f32(src_spad_row, dst_spad_row, ne00);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
dma_queue_push(dma_q,
dma_make_data(dst_data + (ir * dst_row_size), dst_spad_row),
dst_row_size, dst_row_size_aligned, dst_row_size, 1);
const uint32_t next_row = ir + 2;
if (next_row < ir1) {
dma_queue_push(dma_q,
dma_make_data(src_spad_row, src_data + (next_row * src_row_size)),
src_row_size_aligned, src_row_size, src_row_size, 1);
}
}
dma_queue_flush(dma_q);
FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
// ---------------------------------------------------------------------------
// Per thread worker: Direct HVX (no DMA)
// ---------------------------------------------------------------------------
static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) {
htp_cumsum_preamble;
const uint8_t * src_data = (const uint8_t *) src0->data;
uint8_t * dst_data = (uint8_t *) dst->data;
const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
for (uint32_t ir = ir0; ir < ir1; ir++) {
const float * restrict src_row = (const float *) (src_data + ir * cctx->src_row_size);
float * restrict dst_row = (float *) (dst_data + ir * cctx->dst_row_size);
hvx_cumsum_row_f32(src_row, dst_row, ne00);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
int op_cumsum_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
const struct htp_cumsum_kernel_params * kparams =
(const struct htp_cumsum_kernel_params *) octx->kernel_params;
const uint32_t total_rows = kparams->total_rows;
const size_t src_row_size = src0->nb[1];
const size_t dst_row_size = dst->nb[1];
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t n_threads = octx->n_threads;
if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
return HTP_STATUS_NO_SUPPORT;
}
struct htp_cumsum_vtcm_layout layout = { 0 };
if (kparams->use_dma) {
htp_cumsum_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads);
octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;
octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;
octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;
octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;
octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);
octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);
octx->src0_spad.src = NULL;
octx->dst_spad.src = NULL;
}
struct htp_cumsum_context cctx = {
.octx = octx,
.src_row_size = src_row_size,
.dst_row_size = dst_row_size,
.src_row_size_aligned = layout.src_row_size_aligned,
.dst_row_size_aligned = layout.dst_row_size_aligned,
.rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_rows = nrows,
.row_start = row_start,
};
if (kparams->use_dma) {
work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads);
} else {
work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads);
}
return HTP_STATUS_OK;
}
int op_cumsum(struct htp_ops_context * octx) {
const struct htp_tensor * dst = octx->dst;
switch (dst->type) {
case HTP_TYPE_F32:
return op_cumsum_f32(octx);
default:
return HTP_STATUS_NO_SUPPORT;
}
}
llama.cpp/ggml/src/ggml-hexagon/htp/cumsum-ops.h
+49
View File
@@ -0,0 +1,49 @@
#ifndef HTP_CUMSUM_OPS_H
#define HTP_CUMSUM_OPS_H
#include "hex-common.h"
#include "htp-vtcm.h"
#include <stdint.h>
struct htp_cumsum_vtcm_layout {
size_t total_bytes;
size_t off_src0;
size_t off_dst;
size_t src_row_size_aligned;
size_t dst_row_size_aligned;
size_t src0_bytes_per_thread;
size_t dst_bytes_per_thread;
};
static inline void htp_cumsum_vtcm_layout_build(
struct htp_cumsum_vtcm_layout * L,
size_t src_row_size,
size_t dst_row_size,
uint32_t n_threads
) {
L->src_row_size_aligned = hex_round_up(src_row_size, 128);
L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);
L->src0_bytes_per_thread = 2 * L->src_row_size_aligned;
L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
L->total_bytes = off;
}
struct htp_cumsum_kernel_params {
uint32_t n_threads;
uint32_t total_rows;
uint32_t use_dma;
uint32_t vtcm_size;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_cumsum_kernel_params) <= 128, "htp_cumsum_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_CUMSUM_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/diag-ops.c
+243
View File
@@ -0,0 +1,243 @@
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
#include "hvx-types.h"
#include "hex-utils.h"
#include "hvx-copy.h"
#include "dma-queue.h"
#include "diag-ops.h"
#define htp_diag_tensors_preamble \
const struct htp_tensor * restrict src0 = octx->src[0]; \
const struct htp_tensor * restrict dst = octx->dst; \
\
const uint32_t ne02 = src0->ne[2]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
\
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3];
struct htp_diag_context {
struct htp_ops_context * octx;
const struct htp_diag_kernel_params * kparams;
size_t src_batch_size;
size_t dst_row_size;
size_t src_batch_size_aligned;
size_t dst_row_size_aligned;
uint32_t batches_per_thread;
uint32_t total_batches;
uint32_t batch_start;
};
#define htp_diag_preamble \
struct htp_diag_context * dctx = (struct htp_diag_context *) data; \
struct htp_ops_context * octx = dctx->octx; \
htp_diag_tensors_preamble;
static inline void hvx_diag_row_f32(const float * restrict src, float * restrict dst,
uint32_t row_idx, uint32_t n) {
hvx_splat_f32_a((uint8_t *) dst, 0.0f, n);
dst[row_idx] = src[row_idx];
}
// ---------------------------------------------------------------------------
// Per thread worker: DMA src fetch, compute in VTCM, DMA dst writeback
// ---------------------------------------------------------------------------
static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) {
htp_diag_preamble;
dma_queue * dma_q = octx->ctx->dma[ith];
const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith;
const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches);
if (ib0 >= ib1) {
return;
}
const size_t src_batch_size = dctx->src_batch_size;
const size_t dst_row_size = dctx->dst_row_size;
const size_t src_batch_size_aligned = dctx->src_batch_size_aligned;
const size_t dst_row_size_aligned = dctx->dst_row_size_aligned;
const dma_addr_t src_data = src0->data;
const dma_addr_t dst_data = dst->data;
// 1 src buffer + 1 dst row buffer per thread in VTCM
uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned);
uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ib = ib0; ib < ib1; ib++) {
const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02);
const uint32_t i2 = ib - i3 * ne02;
const dma_addr_t src_batch = src_data + i3 * nb03 + i2 * nb02;
// Fetch source vector into VTCM
dma_queue_push(dma_q,
dma_make_data(src_spad, src_batch),
src_batch_size_aligned, src_batch_size, src_batch_size, 1);
dma_queue_flush(dma_q);
const float * src_spad_f32 = (const float *) src_spad;
float * dst_spad_f32 = (float *) dst_spad;
for (uint32_t i1 = 0; i1 < ne1; i1++) {
// Compute row in VTCM
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1));
hvx_diag_row_f32(src_spad_f32, dst_spad_f32, i1, ne0);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1));
// Write completed row back to DDR
const dma_addr_t dst_row = dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1;
dma_queue_push(dma_q,
dma_make_data(dst_row, dst_spad),
dst_row_size, dst_row_size_aligned, dst_row_size, 1);
dma_queue_flush(dma_q);
}
}
FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
// ---------------------------------------------------------------------------
// Per thread worker: Direct HVX (no DMA)
// ---------------------------------------------------------------------------
static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) {
htp_diag_preamble;
const uint8_t * src_data = (const uint8_t *) src0->data;
uint8_t * dst_data = (uint8_t *) dst->data;
const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith;
const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0);
for (uint32_t ib = ib0; ib < ib1; ib++) {
const uint32_t i3 = fastdiv(ib, &dctx->kparams->div_ne02);
const uint32_t i2 = ib - i3 * ne02;
const float * restrict src_batch = (const float *)(src_data + i3 * nb03 + i2 * nb02);
for (uint32_t i1 = 0; i1 < ne1; i1++) {
float * restrict dst_row = (float *)(dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1);
hvx_diag_row_f32(src_batch, dst_row, i1, ne0);
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0);
FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
int op_diag_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
const struct htp_diag_kernel_params * kparams =
(const struct htp_diag_kernel_params *) octx->kernel_params;
const uint32_t total_batches = kparams->total_batches;
uint32_t batch_start = 0;
uint32_t nbatches = total_batches;
if (octx->ctx->mdev.count > 1) {
const bool can_split = (kparams->can_split != 0) && htp_tensor_mdev_data_aligned(dst);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? kparams->batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
batch_start = range.start;
nbatches = range.count;
}
if (nbatches == 0) {
return HTP_STATUS_OK;
}
if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nbatches))) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t n_threads = octx->n_threads;
if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
return HTP_STATUS_NO_SUPPORT;
}
const size_t src_batch_size = src0->ne[0] * sizeof(float);
const size_t dst_row_size = dst->ne[0] * sizeof(float);
struct htp_diag_vtcm_layout layout = { 0 };
if (kparams->use_dma) {
htp_diag_vtcm_layout_build(&layout, src_batch_size, dst_row_size, n_threads);
octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;
octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;
octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;
octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;
octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);
octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);
octx->src0_spad.src = NULL;
octx->dst_spad.src = NULL;
}
struct htp_diag_context dctx = {
.octx = octx,
.kparams = kparams,
.src_batch_size = src_batch_size,
.dst_row_size = dst_row_size,
.src_batch_size_aligned = layout.src_batch_size_aligned,
.dst_row_size_aligned = layout.dst_row_size_aligned,
.batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div),
.total_batches = nbatches,
.batch_start = batch_start,
};
if (kparams->use_dma) {
work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads);
} else {
work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads);
}
return HTP_STATUS_OK;
}
int op_diag(struct htp_ops_context * octx) {
const struct htp_tensor * dst = octx->dst;
int err = HTP_STATUS_OK;
switch (dst->type) {
case HTP_TYPE_F32:
err = op_diag_f32(octx);
break;
default:
err = HTP_STATUS_NO_SUPPORT;
break;
}
return err;
}
llama.cpp/ggml/src/ggml-hexagon/htp/diag-ops.h
+54
View File
@@ -0,0 +1,54 @@
#ifndef HTP_DIAG_OPS_H
#define HTP_DIAG_OPS_H
#include "hex-common.h"
#include "hex-fastdiv.h"
#include "htp-vtcm.h"
#include <stdint.h>
struct htp_diag_vtcm_layout {
size_t total_bytes;
size_t off_src0;
size_t off_dst;
size_t src_batch_size_aligned;
size_t dst_row_size_aligned;
size_t src0_bytes_per_thread;
size_t dst_bytes_per_thread;
};
static inline void htp_diag_vtcm_layout_build(
struct htp_diag_vtcm_layout * L,
size_t src_batch_size,
size_t dst_row_size,
uint32_t n_threads
) {
L->src_batch_size_aligned = hex_round_up(src_batch_size, 128);
L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);
L->src0_bytes_per_thread = L->src_batch_size_aligned;
L->dst_bytes_per_thread = L->dst_row_size_aligned;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
L->total_bytes = off;
}
struct htp_diag_kernel_params {
uint32_t n_threads;
uint32_t total_batches;
uint32_t use_dma;
uint32_t can_split;
uint32_t batches_per_chunk;
uint32_t vtcm_size;
struct fastdiv_values div_ne02;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_diag_kernel_params) <= 128, "htp_diag_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_DIAG_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/dma-copy.h
+169
View File
@@ -0,0 +1,169 @@
#ifndef HTP_DMA_COPY_H
#define HTP_DMA_COPY_H
// DDR<->DDR DMA copies of same-type, same-shape tensors with arbitrary strides.
// Used by CPY for the copy itself and by CONCAT, which is two such copies into
// two views of its destination. Every helper only pushes descriptors; the
// caller flushes the queue when it needs the data.
#include "dma-queue.h"
#include "hex-common.h"
#include "htp-tensor.h"
#include <stddef.h>
#include <stdint.h>
// Contiguous byte run, as 1d transfers of at most DMA_SAFE_CHUNK_SIZE each.
static inline void dma_cpy_sametype_reshape_contig(dma_queue * dma_q,
dma_addr_t dst,
dma_addr_t src0,
uint32_t total_bytes) {
if (total_bytes == 0) {
return;
}
const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE;
while (total_bytes > 0) {
const uint32_t chunk = MIN(total_bytes, max_chunk);
if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1)) {
dma_queue_flush(dma_q);
dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1);
}
dst += chunk;
src0 += chunk;
total_bytes -= chunk;
}
}
// One 2d transfer, split at the 16-bit nrows field.
static inline void dma_cpy_push_2d_chunked(dma_queue * dma_q,
dma_addr_t dst,
dma_addr_t src,
size_t dst_stride,
size_t src_stride,
size_t row_size,
uint32_t nrows) {
if (row_size == 0 || nrows == 0) {
return;
}
while (nrows > 0) {
const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS);
if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) {
dma_queue_flush(dma_q);
dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows);
}
dst += cur_rows * dst_stride;
src += cur_rows * src_stride;
nrows -= cur_rows;
}
}
// Copy a range of rows [row_start, row_start + nrows) from src0 into dst:
// same type, same ne[], any nb[] above dim 0, dim 0 dense on both sides (nb[0] == elem_size).
static inline void dma_cpy_sametype_sameshape_range(dma_queue * dma_q,
const struct htp_tensor * dst,
const struct htp_tensor * src0,
uint32_t elem_size,
uint32_t row_start,
uint32_t nrows) {
if (nrows == 0) {
return;
}
const uint32_t ne00 = src0->ne[0];
const uint32_t ne01 = src0->ne[1];
const uint32_t ne02 = src0->ne[2];
const uint32_t ne03 = src0->ne[3];
if (ne00 == 0 || ne01 == 0 || ne02 == 0 || ne03 == 0) {
return;
}
const uint32_t nb01 = src0->nb[1];
const uint32_t nb02 = src0->nb[2];
const uint32_t nb03 = src0->nb[3];
const uint32_t nb1 = dst->nb[1];
const uint32_t nb2 = dst->nb[2];
const uint32_t nb3 = dst->nb[3];
const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);
if (contiguous) {
dma_cpy_sametype_reshape_contig(dma_q,
dst->data + (dma_addr_t) row_start * ne00 * elem_size,
src0->data + (dma_addr_t) row_start * ne00 * elem_size,
nrows * ne00 * elem_size);
return;
}
// The single-descriptor path flattens (i01,i02,i03) into one row index, so every
// row must sit at a constant stride: nb01 on the source, nb1 on the destination.
// Walk the outer dims and require each to continue that progression. A dim of
// extent 1 spans no rows, so it is skipped -- but its own stride must NOT then be
// used to justify the next dim's stride, which is what comparing nb03 against
// ne02*nb02 did: ggml leaves the stride of an extent-1 dim meaningless, so a view
// could pass the check while its rows were nowhere near that stride.
uint32_t exp_src = ne01 * nb01;
uint32_t exp_dst = ne01 * nb1;
bool contiguous_outer = true;
if (ne02 != 1) {
contiguous_outer = contiguous_outer && (nb02 == exp_src) && (nb2 == exp_dst);
}
exp_src *= ne02;
exp_dst *= ne02;
if (ne03 != 1) {
contiguous_outer = contiguous_outer && (nb03 == exp_src) && (nb3 == exp_dst);
}
if (contiguous_outer) {
dma_cpy_push_2d_chunked(dma_q,
dst->data + (dma_addr_t) row_start * nb1,
src0->data + (dma_addr_t) row_start * nb01,
nb1, nb01, ne00 * elem_size, nrows);
return;
}
const uint32_t ne02_ne01 = ne02 * ne01;
uint32_t i03 = row_start / ne02_ne01;
uint32_t rem = row_start - i03 * ne02_ne01;
uint32_t i02 = rem / ne01;
uint32_t i01 = rem - i02 * ne01;
dma_addr_t cur_dst = dst->data + (dma_addr_t) i01 * nb1 + (dma_addr_t) i02 * nb2 + (dma_addr_t) i03 * nb3;
dma_addr_t cur_src0 = src0->data + (dma_addr_t) i01 * nb01 + (dma_addr_t) i02 * nb02 + (dma_addr_t) i03 * nb03;
uint32_t r = row_start;
const uint32_t row_end = row_start + nrows;
while (r < row_end) {
uint32_t cur_rows = MIN(row_end - r, ne01 - i01);
dma_cpy_push_2d_chunked(dma_q, cur_dst, cur_src0, nb1, nb01, ne00 * elem_size, cur_rows);
r += cur_rows;
i01 += cur_rows;
if (i01 == ne01) {
i01 = 0;
if (++i02 == ne02) {
i02 = 0;
i03++;
}
cur_dst = dst->data + (dma_addr_t) i02 * nb2 + (dma_addr_t) i03 * nb3;
cur_src0 = src0->data + (dma_addr_t) i02 * nb02 + (dma_addr_t) i03 * nb03;
} else {
cur_dst += cur_rows * nb1;
cur_src0 += cur_rows * nb01;
}
}
}
// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on
// both sides (nb[0] == elem_size).
static inline void dma_cpy_sametype_sameshape(dma_queue * dma_q,
const struct htp_tensor * dst,
const struct htp_tensor * src0,
uint32_t elem_size) {
const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
dma_cpy_sametype_sameshape_range(dma_q, dst, src0, elem_size, 0, total_rows);
}
#endif /* HTP_DMA_COPY_H */
llama.cpp/ggml/src/ggml-hexagon/htp/dma-queue.c
+212
View File
@@ -0,0 +1,212 @@
#include "dma-queue.h"
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#pragma clang diagnostic ignored "-Wunused-function"
static inline uint32_t pow2_ceil(uint32_t x) {
if (x <= 1) {
return 1;
}
int p = 2;
x--;
while (x >>= 1) {
p <<= 1;
}
return p;
}
static inline uintptr_t align_up(uintptr_t addr, size_t align) {
return (addr + align - 1) & ~(align - 1);
}
static inline size_t dma_ring_sizeof(size_t capacity) {
capacity = pow2_ceil(capacity);
size_t size_r = sizeof(dma_ring);
size_t offset_desc = align_up(size_r, HEX_L2_LINE_SIZE);
size_t size_desc = capacity * sizeof(dma_descriptor_2d);
size_t offset_data = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);
size_t size_data = capacity * sizeof(dma_data);
return offset_data + size_data;
}
static inline dma_ring * dma_ring_init(void * ptr, size_t capacity, struct htp_thread_trace * trace) {
capacity = pow2_ceil(capacity);
size_t size_r = sizeof(dma_ring);
size_t offset_desc = align_up(size_r, HEX_L2_LINE_SIZE);
size_t size_desc = capacity * sizeof(dma_descriptor_2d);
size_t offset_data = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);
dma_ring * r = (dma_ring *) ptr;
r->trace = trace;
r->capacity = capacity;
r->idx_mask = capacity - 1;
r->push_idx = 0;
r->pop_idx = 0;
r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc);
r->data = (dma_data *) ((uintptr_t) ptr + offset_data);
r->tail = &r->desc[capacity - 1];
return r;
}
size_t dma_queue_sizeof(size_t capacity) {
size_t size_q = sizeof(dma_queue);
size_t offset_r0 = align_up(size_q, HEX_L2_LINE_SIZE);
size_t size_r0 = dma_ring_sizeof(capacity);
size_t offset_r1 = align_up(offset_r0 + size_r0, HEX_L2_LINE_SIZE);
size_t size_r1 = dma_ring_sizeof(DMA_FALLBACK_CAPACITY);
return offset_r1 + size_r1;
}
size_t dma_queue_alignof(void) {
return HEX_L2_LINE_SIZE;
}
dma_queue_t dma_queue_init(void * ptr, size_t capacity, struct htp_thread_trace * trace) {
size_t total_size = dma_queue_sizeof(capacity);
memset(ptr, 0, total_size);
dma_queue * q = (dma_queue *) ptr;
size_t size_q = sizeof(dma_queue);
size_t offset_r0 = align_up(size_q, HEX_L2_LINE_SIZE);
size_t size_r0 = dma_ring_sizeof(capacity);
size_t offset_r1 = align_up(offset_r0 + size_r0, HEX_L2_LINE_SIZE);
q->ring0 = dma_ring_init((void *) ((uintptr_t) ptr + offset_r0), capacity, trace);
q->ring1 = dma_ring_init((void *) ((uintptr_t) ptr + offset_r1), DMA_FALLBACK_CAPACITY, trace);
q->alias = false;
FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", (unsigned) capacity, total_size);
return q;
}
void dma_queue_free(dma_queue_t q) {
(void) q;
}
size_t dma_queue_alias_sizeof(void) {
return sizeof(dma_queue);
}
dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q) {
dma_queue * q = (dma_queue *) ptr;
memset(q, 0, sizeof(dma_queue));
q->ring0 = main_q->ring0;
q->ring1 = main_q->ring1;
q->alias = true;
return q;
}
void dma_queue_alias_free(dma_queue_t q) {
(void) q;
}
bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
dma_ring * r0 = q->ring0;
dma_ring * r1 = q->ring1;
if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) {
return false;
}
r1->tail = r0->tail;
size_t rem_rows = nrows;
dma_addr_t cur_dst = ddata.dst;
dma_addr_t cur_src = ddata.src;
while (rem_rows > 0) {
const uint32_t cur_rows = MIN(rem_rows, DMA_MAX_NROWS);
dma_data cur_data = dma_make_data(cur_dst, cur_src);
if (!dma_ring_push_single_2d(r1, cur_data, dst_stride, src_stride, row_size, cur_rows)) {
dma_ring_flush(r1);
dma_ring_push_single_2d(r1, cur_data, dst_stride, src_stride, row_size, cur_rows);
}
cur_dst += cur_rows * dst_stride;
cur_src += cur_rows * src_stride;
rem_rows -= cur_rows;
}
dma_ring_flush(r1);
r0->tail = r1->tail;
return dma_ring_push_single_2d(r0, ddata, 0, 0, 0, /*nrows=*/ 0);
}
bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total) {
dma_ring * r0 = q->ring0;
dma_ring * r1 = q->ring1;
if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) {
return false;
}
r1->tail = r0->tail;
size_t rem_bytes = total;
dma_addr_t cur_dst = ddata.dst;
dma_addr_t cur_src = ddata.src;
while (rem_bytes > 0) {
const uint32_t cur_bytes = MIN(rem_bytes, DMA_SAFE_CHUNK_SIZE);
dma_data cur_data = dma_make_data(cur_dst, cur_src);
if (!dma_ring_push_single_contig(r1, cur_data, cur_bytes)) {
dma_ring_flush(r1);
dma_ring_push_single_contig(r1, cur_data, cur_bytes);
}
cur_dst += cur_bytes;
cur_src += cur_bytes;
rem_bytes -= cur_bytes;
}
dma_ring_flush(r1);
r0->tail = r1->tail;
return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0);
}
#if __HVX_ARCH__ < 75
bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
dma_ring * r0 = q->ring0;
dma_ring * r1 = q->ring1;
if (((r0->push_idx + 1) & r0->idx_mask) == r0->pop_idx) {
return false;
}
r1->tail = r0->tail;
size_t rem_rows = nrows;
dma_addr_t cur_dst = ddata.dst;
dma_addr_t cur_src = ddata.src;
while (rem_rows > 0) {
dma_data cur_data = dma_make_data(cur_dst, cur_src);
if (!dma_ring_push_single_1d(r1, cur_data, row_size)) {
dma_ring_flush(r1);
dma_ring_push_single_1d(r1, cur_data, row_size);
}
cur_dst += dst_stride;
cur_src += src_stride;
rem_rows -= 1;
}
dma_ring_flush(r1);
r0->tail = r1->tail;
return dma_ring_push_single_1d(r0, ddata, /*size=*/ 0);
}
#endif
llama.cpp/ggml/src/ggml-hexagon/htp/dma-queue.h
+525
View File
@@ -0,0 +1,525 @@
#ifndef HTP_DMA_H
#define HTP_DMA_H
#include <HAP_farf.h>
#include <hexagon_types.h>
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include "hex-utils.h"
#include "hex-profile.h"
#ifdef __cplusplus
extern "C" {
#endif
// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date
typedef struct dma_descriptor_1d_s {
void * next;
uint32_t size:24;
uint32_t desc_size:2;
uint32_t dst_comp:1;
uint32_t src_comp:1;
uint32_t dst_bypass:1;
uint32_t src_bypass:1;
uint32_t order:1;
uint32_t done:1;
uint32_t src;
uint32_t dst;
} dma_descriptor_1d;
#if __HVX_ARCH__ < 75
typedef struct dma_descriptor_2d_s {
void * next;
uint32_t reserved0:24;
uint32_t desc_size:2;
uint32_t dst_comp:1;
uint32_t src_comp:1;
uint32_t dst_bypass:1;
uint32_t src_bypass:1;
uint32_t order:1;
uint32_t done:1;
uint32_t src;
uint32_t dst;
uint32_t desc_type:8;
uint32_t reserved1:24;
uint32_t row_size:16;
uint32_t nrows:16;
uint32_t src_stride:16;
uint32_t dst_stride:16;
uint32_t src_offset:16;
uint32_t dst_offset:16;
} dma_descriptor_2d;
#else
typedef struct dma_descriptor_2d_s {
void * next;
uint32_t dst_stride:24;
uint32_t desc_size:2;
uint32_t dst_comp:1;
uint32_t src_comp:1;
uint32_t dst_bypass:1;
uint32_t src_bypass:1;
uint32_t order:1;
uint32_t done:1;
uint32_t src;
uint32_t dst;
uint32_t desc_type:8;
#if __HVX_ARCH__ > 79
uint32_t src_upper:8;
uint32_t dst_upper:8;
uint32_t allocation:2;
uint32_t reserved0:2;
uint32_t transform:4;
#else
uint32_t reserved0:24;
#endif
uint32_t row_size:24;
uint32_t nrows_lo:8;
uint32_t nrows_hi:8;
uint32_t src_stride:24;
uint32_t offset:24;
uint32_t reserved1:8;
} dma_descriptor_2d;
#endif
#if __HVX_ARCH__ > 79
typedef uint64_t dma_addr_t;
#else
typedef uint32_t dma_addr_t;
#endif
typedef struct {
dma_addr_t dst;
dma_addr_t src;
} dma_data;
// Hardware descriptor field limits
#define DMA_MAX_NROWS 0xFFFFu // 16-bit HW descriptor limit (65535)
#define DMA_MAX_SIZE_16B 0xFFFFu // 16-bit HW descriptor limit for row_size (65535)
#define DMA_MAX_STRIDE_16B 0xFFFFu // 16-bit HW descriptor limit for strides (65535)
#define DMA_MAX_SIZE_24B 0x00FFFFFFu // 24-bit HW descriptor limit for row_size / 1D size (16MB - 1)
#define DMA_MAX_STRIDE_24B 0x00FFFFFFu // 24-bit HW descriptor limit for strides (16MB - 1)
#define DMA_SAFE_CHUNK_SIZE 0x00F00000u // ~15MB safe contiguous chunk size
#if __HVX_ARCH__ < 75
#define DMA_MAX_2D_ROW_SIZE DMA_MAX_SIZE_16B
#define DMA_MAX_2D_STRIDE DMA_MAX_STRIDE_16B
#else
#define DMA_MAX_2D_ROW_SIZE DMA_MAX_SIZE_24B
#define DMA_MAX_2D_STRIDE DMA_MAX_STRIDE_24B
#endif
#define DMA_FALLBACK_CAPACITY 16u // descriptors in secondary fallback ring
typedef struct dma_ring_s dma_ring;
struct dma_ring_s {
dma_descriptor_2d * desc; // descriptor pointers
dma_descriptor_2d * tail; // tail pointer
dma_data * data; // dst/src data
uint32_t push_idx;
uint32_t pop_idx;
uint32_t capacity;
uint32_t idx_mask;
struct htp_thread_trace * trace;
};
typedef struct dma_queue_s dma_queue;
typedef dma_queue * dma_queue_t;
struct dma_queue_s {
dma_ring * ring0; // Main descriptor ring state
dma_ring * ring1; // Secondary fallback descriptor ring state
bool alias; // When set, dma_queue_delete will not free the ring
};
size_t dma_queue_sizeof(size_t capacity);
size_t dma_queue_alignof(void);
dma_queue_t dma_queue_init(void * ptr, size_t capacity, struct htp_thread_trace * trace);
void dma_queue_free(dma_queue_t q);
size_t dma_queue_alias_sizeof(void);
dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q);
void dma_queue_alias_free(dma_queue_t q);
bool dma_queue_push_fallback_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows);
bool dma_queue_push_fallback_contig(dma_queue * q, dma_data ddata, size_t total);
#if __HVX_ARCH__ < 75
bool dma_queue_push_fallback_1d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows);
#endif
// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead
// but those do not seem to always compiler properly.
static inline void dmstart(void * next) {
asm volatile(" release(%0):at" : : "r"(next));
asm volatile(" dmstart(%0)" : : "r"(next));
}
static inline void dmlink(void * cur, void * next) {
asm volatile(" release(%0):at" : : "r"(next));
asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next));
}
static inline unsigned int dmpoll(void) {
unsigned int ret = 0;
asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory");
return ret;
}
static inline unsigned int dmwait(void) {
unsigned int ret = 0;
asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory");
return ret;
}
static inline dma_data dma_make_data_impl(dma_addr_t dst, dma_addr_t src)
{
dma_data d = { dst, src };
return d;
}
#define dma_make_data(dst, src) dma_make_data_impl((dma_addr_t) (dst), (dma_addr_t) (src))
static inline bool dma_ring_push_single_1d(dma_ring * r, dma_data ddata, size_t size) {
#if __HVX_ARCH__ > 79
assert(!((ddata.src | ddata.dst) >> 32) || size == 0);
#endif
if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) {
return false;
}
dma_descriptor_1d * desc = (dma_descriptor_1d *) &r->desc[r->push_idx];
desc->src = (uint32_t) ddata.src;
desc->dst = (uint32_t) ddata.dst;
desc->size = size;
r->data[r->push_idx] = ddata;
htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx);
if (size) {
desc->next = NULL;
desc->desc_size = 0; // 1D mode
desc->src_bypass = 1;
desc->dst_bypass = 1;
desc->order = 0;
desc->done = 0;
dmlink(r->tail, desc);
r->tail = (dma_descriptor_2d *) desc;
} else {
desc->desc_size = 0;
desc->done = 1;
}
r->push_idx = (r->push_idx + 1) & r->idx_mask;
return true;
}
static inline bool dma_ring_push_single_2d(dma_ring * r, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
#if __HVX_ARCH__ > 79
assert(!((ddata.src | ddata.dst) >> 40) || nrows == 0);
const uint32_t src_hi = (uint32_t) (ddata.src >> 32);
const uint32_t dst_hi = (uint32_t) (ddata.dst >> 32);
const bool is_ext = (src_hi | dst_hi) != 0;
#endif
if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) {
return false;
}
dma_descriptor_2d * desc = &r->desc[r->push_idx];
desc->next = NULL;
desc->reserved1 = 0;
desc->desc_size = 1; // 2d mode
desc->src_bypass = 1;
desc->dst_bypass = 1;
desc->src_comp = 0;
desc->dst_comp = 0;
desc->order = 0;
desc->done = 0;
desc->src_stride = src_stride;
desc->dst_stride = dst_stride;
desc->src = (uint32_t) ddata.src;
desc->dst = (uint32_t) ddata.dst;
desc->row_size = row_size;
#if __HVX_ARCH__ < 75
desc->reserved0 = 0;
desc->desc_type = 0; // 2d (16-bit) mode
desc->nrows = nrows;
desc->src_offset = 0;
desc->dst_offset = 0;
#else
#if __HVX_ARCH__ > 79
desc->src_upper = src_hi;
desc->dst_upper = dst_hi;
desc->allocation = 0;
desc->reserved0 = 0;
desc->transform = 0;
desc->desc_type = is_ext ? 10 : 9; // 2d 40-bit or 24-bit mode
#else
desc->reserved0 = 0;
desc->desc_type = 9; // 2d (24-bit) mode
#endif
desc->nrows_lo = (nrows & 0xff);
desc->nrows_hi = (nrows >> 8);
desc->offset = 0;
#endif
r->data[r->push_idx] = ddata;
htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx);
if (nrows) {
dmlink(r->tail, desc);
r->tail = desc;
} else {
desc->done = 1;
}
r->push_idx = (r->push_idx + 1) & r->idx_mask;
return true;
}
#if __HVX_ARCH__ < 75
static inline bool dma_ring_push_single_contig(dma_ring * r, dma_data ddata, size_t size) {
return dma_ring_push_single_1d(r, ddata, size);
}
#else
static inline bool dma_ring_push_single_contig(dma_ring * r, dma_data ddata, size_t size) {
return dma_ring_push_single_2d(r, ddata, size, size, size, 1);
}
#endif
static inline dma_data dma_ring_pop(dma_ring * r) {
dma_data ddata = { 0 };
if (r->push_idx == r->pop_idx) {
return ddata;
}
ddata = r->data[r->pop_idx];
volatile dma_descriptor_2d * desc = &r->desc[r->pop_idx];
// Wait for desc to complete
if (!desc->done) {
// FARF(ALWAYS, "dma-poll: idx %u dst %p src %p", r->pop_idx, ddata.dst, ddata.src);
while (!desc->done) {
dmpoll();
}
}
htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx);
r->pop_idx = (r->pop_idx + 1) & r->idx_mask;
return ddata;
}
static inline dma_data dma_ring_pop_nowait(dma_ring * r) {
dma_data ddata = { 0 };
if (r->push_idx == r->pop_idx) {
return ddata;
}
ddata = r->data[r->pop_idx];
htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx);
r->pop_idx = (r->pop_idx + 1) & r->idx_mask;
return ddata;
}
static inline bool dma_ring_empty(dma_ring * r) {
return r->push_idx == r->pop_idx;
}
static inline void dma_ring_flush(dma_ring * r) {
while (!dma_ring_empty(r)) {
dma_ring_pop(r);
}
}
static inline uint32_t dma_ring_depth(dma_ring * r) {
return (r->push_idx - r->pop_idx) & r->idx_mask;
}
static inline uint32_t dma_ring_capacity(dma_ring * r) {
return r->capacity;
}
static inline bool dma_queue_push_single_1d(dma_queue * q, dma_data ddata, size_t size) {
return dma_ring_push_single_1d(q->ring0, ddata, size);
}
static inline bool dma_queue_push_single_2d(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
return dma_ring_push_single_2d(q->ring0, ddata, dst_stride, src_stride, row_size, nrows);
}
static inline dma_data dma_queue_pop(dma_queue * q) {
return dma_ring_pop(q->ring0);
}
static inline dma_data dma_queue_pop_nowait(dma_queue * q) {
return dma_ring_pop_nowait(q->ring0);
}
static inline bool dma_queue_empty(dma_queue * q) {
return dma_ring_empty(q->ring0);
}
static inline void dma_queue_flush(dma_queue * q) {
dma_ring_flush(q->ring0);
}
static inline uint32_t dma_queue_depth(dma_queue * q) {
return dma_ring_depth(q->ring0);
}
static inline uint32_t dma_queue_capacity(dma_queue * q) {
return dma_ring_capacity(q->ring0);
}
static inline bool dma_queue_push(dma_queue * q, dma_data ddata, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
if (__builtin_expect(nrows == 0, 0)) {
return dma_ring_push_single_1d(q->ring0, ddata, 0);
}
// 1. Hot path: Contiguous or single-row (80-90% of calls)
if (nrows == 1 || !((row_size ^ src_stride) | (row_size ^ dst_stride))) {
const size_t total = row_size * nrows;
if (__builtin_expect(total <= DMA_MAX_SIZE_24B, 1)) {
return dma_ring_push_single_contig(q->ring0, ddata, total);
}
return dma_queue_push_fallback_contig(q, ddata, total);
}
// 2. Hot path: Standard strided 2D (10-20% of calls)
if (__builtin_expect(nrows <= DMA_MAX_NROWS &&
(row_size | src_stride | dst_stride) <= DMA_MAX_2D_ROW_SIZE, 1)) {
return dma_ring_push_single_2d(q->ring0, ddata, dst_stride, src_stride, row_size, nrows);
}
// 3. Cold path: Descriptor chunking fallbacks (< 0.1%)
#if __HVX_ARCH__ < 75
if (row_size <= DMA_MAX_SIZE_16B && (src_stride | dst_stride) <= DMA_MAX_STRIDE_16B) {
return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows);
}
return dma_queue_push_fallback_1d(q, ddata, dst_stride, src_stride, row_size, nrows);
#else
return dma_queue_push_fallback_2d(q, ddata, dst_stride, src_stride, row_size, nrows);
#endif
}
static inline void dma_sync_read(dma_queue * dma_q, void * dst, dma_addr_t src, size_t bytes) {
const uint32_t b = (uint32_t) bytes;
if (b > 0) {
dma_queue_push(dma_q, dma_make_data(dst, src), b, b, b, 1);
dma_queue_pop(dma_q);
}
}
static inline void dma_sync_write(dma_queue * dma_q, dma_addr_t dst, const void * src, size_t bytes) {
const uint32_t b = (uint32_t) bytes;
if (b > 0) {
dma_queue_push(dma_q, dma_make_data(dst, src), b, b, b, 1);
dma_queue_pop(dma_q);
}
}
#define DMA_CACHE_MAX_SIZE 128U
// Fully assoc LRU cache
typedef struct __attribute__((aligned(128))) {
uint8_t *base;
uint32_t line_size;
uint32_t capacity;
dma_addr_t src[DMA_CACHE_MAX_SIZE];
uint16_t age[DMA_CACHE_MAX_SIZE];
} dma_cache_fa;
static inline void dma_cache_fa_init(dma_cache_fa *c, uint8_t *base, uint32_t line_size, uint32_t capacity)
{
c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;
c->base = base;
c->line_size = line_size;
for (unsigned i=0; i < c->capacity; i++) {
c->src[i] = 0;
c->age[i] = 0;
}
}
static inline bool dma_cache_fa_push(dma_queue *q, dma_cache_fa *c, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)
{
uint32_t o_idx = 0;
uint16_t o_age = 0;
uint8_t * dst = 0;
for (unsigned i=0; i < c->capacity; i++) {
if (c->src[i] == src_addr) {
c->age[i] = 0;
dst = c->base + (i * c->line_size); nrows = 0; // dummy dma
} else {
c->age[i]++;
if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; }
}
}
if (!dst) {
c->age[o_idx] = 0;
c->src[o_idx] = src_addr;
dst = c->base + o_idx * c->line_size; // normal nrows dma
return dma_queue_push(q, dma_make_data(dst, src_addr), dst_stride, src_stride, row_size, nrows);
}
return dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0);
}
// Direct mapped cache
typedef struct __attribute__((aligned(128))) {
uint8_t *base;
uint32_t line_size;
uint32_t capacity;
uint32_t idx_mask;
dma_addr_t src[DMA_CACHE_MAX_SIZE];
} dma_cache_dm;
static inline void dma_cache_dm_init(dma_cache_dm *c, uint8_t *base, uint32_t line_size, uint32_t capacity)
{
c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;
c->idx_mask = c->capacity - 1;
c->base = base;
c->line_size = line_size;
for (unsigned i=0; i < c->capacity; i++) {
c->src[i] = 0;
}
}
static inline bool dma_cache_dm_push(dma_queue *q, dma_cache_dm *c, uint32_t slot, dma_addr_t src_addr, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)
{
const uint32_t i = slot & c->idx_mask;
uint8_t * dst = c->base + (i * c->line_size);
if (c->src[i] == src_addr) {
return dma_queue_push_single_1d(q, dma_make_data(dst, src_addr), 0); // dummy dma
}
c->src[i] = src_addr;
return dma_queue_push(q, dma_make_data(dst, src_addr), dst_stride, src_stride, row_size, nrows);
}
#ifdef __cplusplus
} // extern "C"
#endif
#endif /* HTP_DMA_H */
llama.cpp/ggml/src/ggml-hexagon/htp/fill-ops.c
+162
View File
@@ -0,0 +1,162 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic ignored "-Wunused-function"
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <string.h>
#include "hex-common.h"
#include "hex-profile.h"
#include "hvx-copy.h"
#include "hvx-utils.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "fill-ops.h"
// ggml op_params layout for FILL:
// op_params[0] (as float) - the scalar fill value
#define fill_preamble \
const struct htp_tensor * dst = octx->dst; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3];
struct htp_fill_context {
struct htp_ops_context * octx;
const struct htp_fill_kernel_params * kparams;
uint32_t nrows_per_thread;
uint32_t total_rows; // ne1 * ne2 * ne3
uint32_t row_start;
HVX_Vector splat_vec;
uint32_t elem_size;
};
static void fill_thread_contiguous(unsigned int nth, unsigned int ith, void * data) {
const struct htp_fill_context * fctx = (const struct htp_fill_context *) data;
struct htp_ops_context * octx = fctx->octx;
fill_preamble;
// Parallelise over the flat row index spanning ne1*ne2*ne3
const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows);
if (ir0 >= ir1) {
return;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
const uint32_t elem_start = ir0 * ne0;
const uint32_t elem_end = ir1 * ne0;
uint8_t * dst_ptr = (uint8_t *) dst->data + elem_start * fctx->elem_size;
hvx_splat_u(dst_ptr, fctx->splat_vec, elem_end - elem_start, fctx->elem_size);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1);
FARF(HIGH, "fill-contig %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);
}
static void fill_thread_strided(unsigned int nth, unsigned int ith, void * data) {
const struct htp_fill_context * fctx = (const struct htp_fill_context *) data;
struct htp_ops_context * octx = fctx->octx;
const struct htp_fill_kernel_params * kparams = fctx->kparams;
fill_preamble;
// Parallelise over the flat row index spanning ne1*ne2*ne3
const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows);
if (ir0 >= ir1) {
return;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
for (uint32_t ir = ir0; ir < ir1; ++ir) {
const uint32_t i3 = fastdiv(ir, &kparams->div_ne1_ne2);
const uint32_t rem = ir - i3 * (ne1 * ne2);
const uint32_t i2 = fastdiv(rem, &kparams->div_ne1);
const uint32_t i1 = rem - i2 * ne1;
uint8_t * dst_ptr = (uint8_t *) dst->data + i1*nb1 + i2*nb2 + i3*nb3;
hvx_splat_u(dst_ptr, fctx->splat_vec, ne0, fctx->elem_size);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1);
FARF(HIGH, "fill-strided %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);
}
int op_fill(struct htp_ops_context * octx) {
const struct htp_tensor * dst = octx->dst;
const struct htp_fill_kernel_params * kparams =
(const struct htp_fill_kernel_params *) octx->kernel_params;
if (htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
uint32_t row_start = 0;
uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t n_threads = octx->n_threads;
FARF(HIGH, "fill: (%ux%ux%ux%u) type=%u ktype=%u\n",
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->type, kparams->kernel_type);
float val_f32 = 0.f;
memcpy(&val_f32, &octx->op_params[0], sizeof(float));
struct htp_fill_context fctx = {
.octx = octx,
.kparams = kparams,
.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_rows = nrows,
.row_start = row_start,
.elem_size = kparams->elem_size,
};
switch (dst->type) {
case HTP_TYPE_F32:
fctx.splat_vec = hvx_vec_splat_f32(val_f32);
break;
case HTP_TYPE_F16:
fctx.splat_vec = hvx_vec_splat_f16((_Float16) val_f32);
break;
default:
return HTP_STATUS_NO_SUPPORT;
}
work_queue_func_t worker = (kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS)
? fill_thread_contiguous
: fill_thread_strided;
work_queue_run(octx->ctx->work_queue, worker, &fctx, n_threads);
return HTP_STATUS_OK;
}
llama.cpp/ggml/src/ggml-hexagon/htp/fill-ops.h
+29
View File
@@ -0,0 +1,29 @@
#ifndef HTP_FILL_OPS_H
#define HTP_FILL_OPS_H
#include "hex-fastdiv.h"
#include <stdint.h>
enum htp_fill_kernel_type {
HTP_FILL_KERNEL_CONTIGUOUS = 0,
HTP_FILL_KERNEL_STRIDED = 1,
};
struct htp_fill_kernel_params {
uint8_t kernel_type;
uint32_t n_threads;
uint32_t total_rows;
uint32_t elem_size;
uint32_t rows_per_chunk;
struct fastdiv_values div_ne1;
struct fastdiv_values div_ne1_ne2;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_fill_kernel_params) <= 128, "htp_fill_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_FILL_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/flash-attn-ops.c
+2828
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp/flash-attn-ops.h
+410
View File
@@ -0,0 +1,410 @@
#ifndef HTP_FLASH_ATTN_OPS_H
#define HTP_FLASH_ATTN_OPS_H
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include "hex-fastdiv.h"
#include "hex-common.h"
#include "htp-vtcm.h"
#ifdef __cplusplus
extern "C" {
#endif
// Tile constants (mirrored from hmx-utils.h for use on host side if needed)
#define HTP_FA_HMX_TILE_SIZE 2048
#define HMX_FP16_TILE_SIZE 2048
#define HMX_FP16_TILE_N_ROWS 32
#define HMX_FP16_TILE_N_COLS 32
#define HMX_FP16_TILE_N_ELMS 1024
#define HVX_FA_DMA_CACHE_SIZE 128
#define HMX_FA_DMA_CACHE_SIZE 4
#define HTP_FA_M_INITIAL_VAL -10000.0f
enum htp_fa_kernel_type {
HTP_FA_KERNEL_UNSUPPORTED = 0,
HTP_FA_KERNEL_HVX,
HTP_FA_KERNEL_HMX
};
// Multicore: gcd(n_kv_heads, N) teams own the KV heads, team members share the Q-blocks
#define HTP_FA_FLAG_TEAM_SPLIT 0x01
static inline void htp_fa_mdev_teams(uint32_t n_kv_heads, uint32_t mdev_count, bool head_split,
uint32_t * n_teams, uint32_t * team_size) {
const uint32_t t = (head_split && n_kv_heads > 0 && mdev_count > 1) ? hex_gcd_u32(n_kv_heads, mdev_count) : 1;
*n_teams = t;
*team_size = mdev_count > 1 ? mdev_count / t : 1;
}
struct htp_fa_kernel_params {
uint8_t kernel_type; // enum htp_fa_kernel_type
uint8_t head_split; // 1 = partition by KV heads in multicore, 0 = token partition
uint8_t flags; // HTP_FA_FLAG_*
uint8_t n_threads; // Number of threads to run
// Common parameters
uint16_t Br;
uint16_t Bc;
uint16_t n_kv_blocks; // also HVX's n_blocks
uint16_t G; // GQA factor (n_heads / n_kv_heads)
float scale;
float max_bias;
float logit_softcap;
uint32_t vtcm_size;
float m0;
float m1;
uint32_t n_head_log2;
struct fastdiv_values src3_div2;
struct fastdiv_values src3_div3;
struct fastdiv_values broadcast_rk2;
struct fastdiv_values broadcast_rk3;
struct fastdiv_values broadcast_rv2;
struct fastdiv_values broadcast_rv3;
union {
struct {
uint32_t g_br;
int32_t mask_broadcast;
int32_t pipeline;
struct fastdiv_values div_G;
} hmx;
struct {
uint32_t size_q_row_padded;
uint32_t size_k_row_padded;
uint32_t size_v_row_padded;
struct fastdiv_values src0_div21;
struct fastdiv_values src0_div1;
} hvx;
} u;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_fa_kernel_params) <= 128, "htp_fa_kernel_params is too large for kernel_params blob");
#endif
// VTCM region layout for the HMX flash-attention kernel.
//
// Single source of truth for both the host (which needs the total size to pick a
// (Br, Bc) tiling that fits the VTCM budget) and the device (which needs the actual
// byte offsets to place each scratch buffer). Building the layout once and reading
// offsets/total from it makes host estimate and device allocation impossible to
// desync -- previously they were duplicated formulas in two files and drifted.
//
// All fields are byte offsets / byte sizes -- no HVX_Vector type is named here so the
// header stays host-includable. The device casts (base + off_*) to the proper type.
// An offset of 0 marks a region that is not allocated for this configuration (only
// off_v_tiles[1], which exists only when pipelining); the device sets such pointers NULL.
struct hmx_fa_vtcm_layout {
// Byte offsets from vtcm_base for each region.
size_t off_q_tiles;
size_t off_q_dma;
size_t off_o_tiles[2];
size_t off_k_fp16[2];
size_t off_v_fp16[2];
size_t off_k_tiles[2];
size_t off_v_tiles[2];
size_t off_s_tiles[2];
size_t off_p_tiles[2];
size_t off_d_tiles[2];
size_t off_d_inv_l;
size_t off_m_vec;
size_t off_l_vec;
size_t off_s_rowmax;
size_t off_p_rowsum;
size_t off_row_bufs;
size_t off_hmx_scales_id;
size_t off_hmx_scales_qk;
size_t off_mask_buf;
size_t off_slopes;
size_t off_sinks;
// Region byte sizes reused by the device at runtime (not just for allocation).
size_t q_tile_bytes;
size_t o_tile_bytes;
size_t k_tile_bytes;
size_t s_tile_bytes; // S and P tiles (same size)
size_t d_tile_bytes; // d_tiles[0..1] + d_inv_l, allocated back to back
size_t m_line_bytes; // one mask row
size_t m_buf_slot_bytes; // one dma_cache slot = align_up(Br * m_line_bytes, 4096)
size_t col_vec_bytes;
size_t sinks_bytes;
// Derived strides.
size_t row_buf_stride; // HVX vectors (128B) per row buffer
size_t mask_buf_row_stride; // __fp16 elements per row in the mask buffer
bool pipeline;
size_t total_bytes;
};
#define FA_MAX_KV_BLOCKS 1024
// Build the VTCM layout.
static inline void hmx_fa_vtcm_layout_build(struct hmx_fa_vtcm_layout * L,
size_t gqa_factor, size_t DK, size_t DV,
size_t Br, size_t Bc, size_t n_threads,
bool pipeline, bool is_q_fp32,
bool has_sinks, size_t n_heads) {
const size_t g_br = hex_align_up(gqa_factor * Br, HMX_FP16_TILE_N_ROWS);
const size_t q_tile_size = hex_align_up(g_br * DK * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE);
const size_t o_tile_size = hex_align_up(g_br * DV * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE);
const size_t k_tile_size = hex_align_up(Bc * DK * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE);
const size_t v_tile_size = hex_align_up(Bc * DV * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE);
const size_t s_tile_size = hex_align_up(g_br * Bc * sizeof(__fp16), HTP_FA_HMX_TILE_SIZE);
// The rescale matrices are diagonal: the HMX kernels only ever load the g_br/32
// tiles that sit on the diagonal, so store just those, packed back to back with
// a stride of one tile. The old [g_br, g_br] square layout allocated g_br/32
// times more than it used, which is also why a second D buffer was unaffordable.
const size_t d_tile_size = (g_br / HMX_FP16_TILE_N_ROWS) * HTP_FA_HMX_TILE_SIZE;
const size_t q_dma_size = hex_align_up(g_br * DK * (is_q_fp32 ? sizeof(float) : sizeof(__fp16)), 128);
const size_t k_dma_size = hex_align_up(Bc * hex_round_up(DK * sizeof(__fp16), 128), 128);
const size_t v_dma_size = hex_align_up(Bc * hex_round_up(DV * sizeof(__fp16), 128), 128);
const size_t col_vec_size = hex_align_up(g_br * sizeof(float), 256);
const size_t row_vec_size = hex_align_up(Bc * sizeof(__fp16), 256);
const size_t m_line_size = hex_align_up(Bc * sizeof(__fp16), 128);
const size_t m_buf_slot = hex_align_up(Br * m_line_size, 256);
const size_t m_buf_size = m_buf_slot * HMX_FA_DMA_CACHE_SIZE;
const size_t slopes_size = hex_align_up(g_br * sizeof(__fp16), 128);
const size_t sinks_size = hex_round_up(n_heads * sizeof(float), 128);
size_t off = 0;
// Group A (Part 1 - HMX Tiled buffers)
VTCM_LAYOUT_ALLOC(off, off_q_tiles, q_tile_size);
VTCM_LAYOUT_ALLOC(off, off_o_tiles[0], o_tile_size);
VTCM_LAYOUT_ALLOC(off, off_o_tiles[1], o_tile_size);
VTCM_LAYOUT_ALLOC(off, off_d_tiles[0], d_tile_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_d_tiles[1], d_tile_size, pipeline);
VTCM_LAYOUT_ALLOC(off, off_d_inv_l, d_tile_size);
// Group B & C share start offset (Group B tiles must be 2KB aligned)
size_t off_group_b_c = hex_align_up(off, HTP_FA_HMX_TILE_SIZE);
// Group B: Compute-only buffers
size_t off_group_b = off_group_b_c;
VTCM_LAYOUT_ALLOC(off_group_b, off_k_tiles[0], k_tile_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_k_tiles[1], k_tile_size, pipeline);
VTCM_LAYOUT_ALLOC(off_group_b, off_v_tiles[0], v_tile_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_v_tiles[1], v_tile_size, pipeline);
VTCM_LAYOUT_ALLOC(off_group_b, off_s_tiles[0], s_tile_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_s_tiles[1], s_tile_size, pipeline);
VTCM_LAYOUT_ALLOC(off_group_b, off_p_tiles[0], s_tile_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_p_tiles[1], s_tile_size, pipeline);
VTCM_LAYOUT_ALLOC(off_group_b, off_s_rowmax, col_vec_size);
VTCM_LAYOUT_ALLOC(off_group_b, off_p_rowsum, col_vec_size);
VTCM_LAYOUT_ALLOC(off_group_b, off_row_bufs, row_vec_size * 2 * n_threads);
const size_t group_b_size = off_group_b - off_group_b_c;
// Group C: Q fetch DMA buffer
size_t off_group_c = off_group_b_c;
VTCM_LAYOUT_ALLOC(off_group_c, off_q_dma, q_dma_size);
const size_t group_c_size = off_group_c - off_group_b_c;
off = off_group_b_c + hex_smax(group_b_size, group_c_size);
// Group A (Part 2 - remaining non-HMX buffers)
VTCM_LAYOUT_ALLOC(off, off_k_fp16[0], k_dma_size);
VTCM_LAYOUT_ALLOC(off, off_k_fp16[1], k_dma_size);
VTCM_LAYOUT_ALLOC(off, off_v_fp16[0], v_dma_size);
VTCM_LAYOUT_ALLOC(off, off_v_fp16[1], v_dma_size);
VTCM_LAYOUT_ALLOC(off, off_m_vec, col_vec_size);
VTCM_LAYOUT_ALLOC(off, off_l_vec, col_vec_size);
VTCM_LAYOUT_ALLOC(off, off_hmx_scales_id, 256);
VTCM_LAYOUT_ALLOC(off, off_hmx_scales_qk, 256);
VTCM_LAYOUT_ALLOC(off, off_mask_buf, m_buf_size);
VTCM_LAYOUT_ALLOC(off, off_slopes, slopes_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, sinks_size, has_sinks);
L->q_tile_bytes = q_tile_size;
L->o_tile_bytes = o_tile_size;
L->k_tile_bytes = k_tile_size;
L->col_vec_bytes = col_vec_size;
L->s_tile_bytes = s_tile_size;
// Measured from the actual offsets rather than assumed to be N * d_tile_size, so
// that inserting a region between them (or adding padding to VTCM_LAYOUT_ALLOC)
// cannot silently leave the tail of the run unzeroed.
L->d_tile_bytes = (L->off_d_inv_l + d_tile_size) - L->off_d_tiles[0];
L->m_line_bytes = m_line_size;
L->m_buf_slot_bytes = m_buf_slot;
L->row_buf_stride = row_vec_size / 128;
L->mask_buf_row_stride = m_line_size / sizeof(__fp16);
L->sinks_bytes = has_sinks ? sinks_size : 0;
L->pipeline = pipeline;
L->total_bytes = off;
}
// Exact VTCM usage for a given (gqa_factor, DK, DV, Br, Bc) configuration.
static inline size_t hmx_fa_compute_vtcm_usage(size_t gqa_factor, size_t DK, size_t DV, size_t Br, size_t Bc, size_t n_threads, bool pipeline, bool is_q_fp32, bool has_sinks, size_t n_heads) {
struct hmx_fa_vtcm_layout L;
hmx_fa_vtcm_layout_build(&L, gqa_factor, DK, DV, Br, Bc, n_threads, pipeline, is_q_fp32, has_sinks, n_heads);
return L.total_bytes;
}
#define FA_HVX_BLOCK_SIZE 64
#define FA_HVX_G_MAX 8
struct hvx_fa_vtcm_layout {
size_t off_q;
size_t off_k;
size_t off_v;
size_t off_m;
size_t off_a;
size_t off_sinks;
size_t size_q_block;
size_t size_k_block;
size_t size_v_block;
size_t size_m_block;
size_t size_vkq_acc;
size_t size_sinks;
size_t total_bytes;
};
static inline void hvx_fa_vtcm_layout_build(struct hvx_fa_vtcm_layout * L,
size_t DK, size_t DV,
bool is_q_fp32, bool has_mask,
bool has_sinks, size_t n_heads,
size_t n_threads) {
const size_t size_q_row_padded = hex_round_up(DK * (is_q_fp32 ? 4 : 2), 128);
const size_t size_k_row_padded = hex_round_up(DK * sizeof(__fp16), 128);
const size_t size_v_row_padded = hex_round_up(DV * sizeof(__fp16), 128);
const size_t size_q_block = size_q_row_padded * FA_HVX_G_MAX;
const size_t size_k_block = size_k_row_padded * FA_HVX_BLOCK_SIZE;
const size_t size_v_block = size_v_row_padded * FA_HVX_BLOCK_SIZE;
const size_t size_m_block = hex_round_up(FA_HVX_BLOCK_SIZE * sizeof(__fp16), 128);
const size_t size_vkq_acc = hex_round_up(DV * sizeof(float), 128) * FA_HVX_G_MAX;
const size_t size_sinks = hex_round_up(n_heads * sizeof(float), 128);
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_q, size_q_block * n_threads);
VTCM_LAYOUT_ALLOC(off, off_k, size_k_block * 2 * n_threads);
VTCM_LAYOUT_ALLOC(off, off_v, size_v_block * 2 * n_threads);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_m, size_m_block * HVX_FA_DMA_CACHE_SIZE * n_threads, has_mask);
VTCM_LAYOUT_ALLOC(off, off_a, size_vkq_acc * n_threads);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_sinks, size_sinks, has_sinks);
L->size_q_block = size_q_block;
L->size_k_block = size_k_block;
L->size_v_block = size_v_block;
L->size_m_block = size_m_block;
L->size_vkq_acc = size_vkq_acc;
L->size_sinks = has_sinks ? size_sinks : 0;
L->total_bytes = off;
}
static inline size_t hvx_fa_compute_vtcm_usage(size_t DK, size_t DV, bool is_q_fp32, bool has_mask, bool has_sinks, size_t n_heads, size_t n_threads) {
struct hvx_fa_vtcm_layout L;
hvx_fa_vtcm_layout_build(&L, DK, DV, is_q_fp32, has_mask, has_sinks, n_heads, n_threads);
return L.total_bytes;
}
#define FA_MIN_KV_BLOCKS 3
// Cost-based (Br, Bc) search for flash attention with pipeline constraint.
static inline int hmx_fa_find_chunk_size(size_t * Br_out,
size_t * Bc_out,
size_t gqa_factor,
size_t DK,
size_t DV,
size_t qo_len,
size_t kv_len,
size_t vtcm_budget,
size_t n_threads,
bool is_q_fp32,
bool has_sinks,
size_t n_heads,
size_t q_split) {
const size_t T = HMX_FP16_TILE_N_ROWS; // 32
const size_t br_unit = hmx_ceil_div(T, gqa_factor);
const size_t bc_unit = HMX_FP16_TILE_N_COLS * 2; // 64
const bool can_pipeline = (kv_len >= FA_MIN_KV_BLOCKS * bc_unit && n_threads >= 2);
// Br_max: largest Br aligned to br_unit that does not exceed qo_len.
const size_t Br_max = qo_len >= br_unit ? hex_align_down(qo_len, br_unit) : br_unit;
// Pipeline constraint: cap Bc so n_kv_blocks >= FA_MIN_KV_BLOCKS.
// Only relax when kv_len is too short to form enough blocks.
size_t Bc_limit = can_pipeline ? hex_align_down(kv_len / FA_MIN_KV_BLOCKS, bc_unit) :
(kv_len >= bc_unit ? hex_align_down(kv_len, bc_unit) : bc_unit);
// For small-Q SWA-sized KV caches (qo_len <= 16, kv_len <= 2048), cap Bc at 256 for block-skipping granularity
if (qo_len <= 16 && kv_len <= 2048 && Bc_limit > 256) {
Bc_limit = 256;
}
// Cost coefficients calibrated from profiling
const size_t c_q_fixed = 800; // per-Q-block: q_load + epilogue o_update + o_norm + o_store
const size_t c_iter_base = 200; // per-KV-iter base (HMX dot/update + DMA)
const size_t c_softmax = 600; // per 64-row vector chunk on HVX
size_t best_cost = SIZE_MAX, best_mn = 0;
size_t best_Br = 0, best_Bc = 0;
for (size_t Br = Br_max; Br >= br_unit; Br -= br_unit) {
// Try all Bc candidates from Bc_limit down to bc_unit
for (size_t Bc = Bc_limit; Bc >= bc_unit; Bc -= bc_unit) {
size_t vtcm_needed = hmx_fa_compute_vtcm_usage(gqa_factor, DK, DV, Br, Bc, n_threads, can_pipeline, is_q_fp32, has_sinks, n_heads);
if (vtcm_needed <= vtcm_budget) {
// This Bc fits for this Br!
const size_t q_blocks = (qo_len + Br - 1) / Br;
const size_t kv_blocks = (kv_len + Bc - 1) / Bc;
const size_t actual_threads = (kv_blocks >= 3 && n_threads >= 2) ? n_threads : 1;
const size_t n_rows_g = Br * gqa_factor;
const size_t n_row_vec_cnt = (n_rows_g + 63) / 64;
const size_t n_use = n_row_vec_cnt < actual_threads ? n_row_vec_cnt : actual_threads;
const size_t vecs_per_t = n_use > 0 ? (n_row_vec_cnt + n_use - 1) / n_use : 1;
const size_t c_iter_actual = c_iter_base + c_softmax * vecs_per_t;
const size_t q_blocks_nsp = (q_blocks + q_split - 1) / (q_split ? q_split : 1);
const size_t cost = q_blocks_nsp * (c_q_fixed + kv_blocks * c_iter_actual);
const size_t mn = Br * Bc;
if (cost < best_cost || (cost == best_cost && mn > best_mn)) {
best_cost = cost;
best_mn = mn;
best_Br = Br;
best_Bc = Bc;
}
// Since we iterate Bc from largest to smallest, this is the largest Bc that fits
// for this Br. We can break to the next Br.
break;
}
}
if (Br == br_unit) {
break;
}
}
if (best_Br == 0 || best_Bc == 0) {
return -1;
}
*Br_out = best_Br;
*Bc_out = best_Bc;
return 0;
}
#ifdef __cplusplus
}
#endif
#endif /* HTP_FLASH_ATTN_OPS_H */
llama.cpp/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
+2357
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp/gated-delta-net-ops.h
+297
View File
@@ -0,0 +1,297 @@
#ifndef HTP_GATED_DELTA_NET_OPS_H
#define HTP_GATED_DELTA_NET_OPS_H
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include "hex-fastdiv.h"
#include "hex-common.h"
#include "htp-vtcm.h"
#define HTP_GDN_MAX_SV 128
#define HTP_GDN_CHUNK_SIZE 64
#define HTP_GDN_MIN_TOKENS 8
#ifndef HMX_FP16_TILE_SIZE
#define HMX_FP16_TILE_SIZE 2048
#endif
enum htp_gdn_kernel_type {
HTP_GDN_KERNEL_HVX_RECURRENT = 0,
HTP_GDN_KERNEL_HMX_CHUNKED = 1,
};
struct htp_gdn_kernel_params {
uint8_t kernel_type;
uint8_t pipeline;
uint16_t chunk_size;
uint16_t n_chunks;
uint16_t n_heads_batch;
uint32_t n_threads;
uint32_t S_v;
uint32_t H;
uint32_t n_tokens;
uint32_t n_seqs;
uint32_t K;
uint32_t total_rows;
uint32_t kda;
uint32_t vtcm_size;
uint32_t state_seq_stride;
uint32_t state_size_per_snap;
float scale;
struct fastdiv_values div_H;
struct fastdiv_values div_q1;
struct fastdiv_values div_k1;
struct fastdiv_values div_rq3;
struct fastdiv_values div_rk3;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_gdn_kernel_params) <= 128, "htp_gdn_kernel_params is too large for kernel_params blob");
#endif
struct htp_gdn_vtcm_layout {
size_t state_aligned;
size_t bytes_per_thread;
size_t off_state;
size_t total_bytes;
};
static inline void htp_gdn_vtcm_layout_build(
struct htp_gdn_vtcm_layout * L,
uint32_t S_v,
uint32_t n_threads
) {
size_t state_bytes = (size_t) S_v * S_v * sizeof(float);
L->state_aligned = hex_round_up(state_bytes, 128);
L->bytes_per_thread = 2 * L->state_aligned;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_state, L->bytes_per_thread * n_threads);
L->total_bytes = off;
}
struct htp_gdn_hmx_vtcm_layout {
size_t off_s_state;
size_t off_s_f16;
size_t off_s_col_tiles;
size_t off_s_update_f32;
size_t off_s_update_tiles;
size_t off_q_f32[2];
size_t off_k_f32[2];
size_t off_v_f32[2];
size_t off_g_f32[2];
size_t off_b_f32[2];
size_t off_g_raw[2];
size_t off_b_raw[2];
size_t off_o_f32[2];
size_t off_v_inter_f32;
size_t off_o_inter_f32;
size_t off_o_intra_f32;
size_t off_k_f16;
size_t off_v_prime_f16;
size_t off_delta_f16;
size_t off_d_f16;
size_t off_q_row_tiles;
size_t off_q_prime_row_tiles;
size_t off_k_row_tiles;
size_t off_k_col_tiles;
size_t off_k_prime_row_tiles;
size_t off_k_col_tiles_64x128;
size_t off_kk_tiles;
size_t off_qk_tiles;
size_t off_v_inter_tiles;
size_t off_o_inter_tiles;
size_t off_inv_row_tiles;
size_t off_a_row_tiles;
size_t off_v_prime_col_tiles;
size_t off_delta_tiles;
size_t off_delta_col_tiles;
size_t off_o_intra_tiles;
size_t off_d_row_tiles;
size_t off_gamma;
size_t off_lambda_init;
size_t off_decay_m;
size_t off_decay_a;
size_t off_rows_kk;
size_t off_rows_qk;
size_t off_rows_inv;
size_t off_rows_a;
size_t off_thread_scratch;
size_t off_scales_1;
size_t state_f32_bytes;
size_t state_f16_bytes;
size_t state_tiles_bytes;
size_t dma_chunk_bytes;
size_t act_f16_bytes;
size_t tile_64xSv_bytes;
size_t tile_64x64_bytes;
uint32_t n_heads_batch;
uint32_t n_threads;
bool pipeline;
size_t total_bytes;
};
static inline void htp_gdn_hmx_vtcm_layout_build(
struct htp_gdn_hmx_vtcm_layout * L,
uint32_t S_v,
uint32_t chunk_size,
uint32_t n_heads_batch,
uint32_t n_threads,
bool pipeline
) {
memset(L, 0, sizeof(*L));
L->n_heads_batch = n_heads_batch;
L->n_threads = n_threads;
L->pipeline = pipeline;
const size_t bh = (size_t) n_heads_batch;
const size_t nth = (size_t) (n_threads > 0 ? n_threads : 1);
const size_t state_f32_sz = hex_round_up(S_v * S_v * sizeof(float), 2048);
const size_t state_f16_sz = hex_round_up(S_v * S_v * sizeof(__fp16), 2048);
const size_t n_sv_tiles = S_v / 32;
const size_t state_tiles_sz = n_sv_tiles * n_sv_tiles * 2048;
const size_t dma_chunk_sz = hex_round_up(chunk_size * S_v * sizeof(float), 2048);
const size_t dma_scalar_sz = hex_round_up(chunk_size * sizeof(float), 128);
const size_t act_f16_sz = hex_round_up(chunk_size * S_v * sizeof(__fp16), 2048);
const size_t tile_64xSv_sz = 2 * n_sv_tiles * 2048;
const size_t tile_64x64_sz = 4 * 2048;
const size_t decay_sz = 64 * 64 * sizeof(__fp16);
const size_t row_vecs_sz = 64 * 128;
L->state_f32_bytes = state_f32_sz;
L->state_f16_bytes = state_f16_sz;
L->state_tiles_bytes = state_tiles_sz;
L->dma_chunk_bytes = dma_chunk_sz;
L->act_f16_bytes = act_f16_sz;
L->tile_64xSv_bytes = tile_64xSv_sz;
L->tile_64x64_bytes = tile_64x64_sz;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_s_state, bh * state_f32_sz);
VTCM_LAYOUT_ALLOC(off, off_s_f16, bh * state_f16_sz);
off = hex_align_up(off, HMX_FP16_TILE_SIZE);
VTCM_LAYOUT_ALLOC(off, off_s_col_tiles, bh * state_tiles_sz);
VTCM_LAYOUT_ALLOC(off, off_s_update_f32, bh * state_f32_sz);
off = hex_align_up(off, HMX_FP16_TILE_SIZE);
VTCM_LAYOUT_ALLOC(off, off_s_update_tiles, bh * state_tiles_sz);
VTCM_LAYOUT_ALLOC(off, off_q_f32[0], bh * dma_chunk_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_q_f32[1], bh * dma_chunk_sz, pipeline);
VTCM_LAYOUT_ALLOC(off, off_k_f32[0], bh * dma_chunk_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_k_f32[1], bh * dma_chunk_sz, pipeline);
VTCM_LAYOUT_ALLOC(off, off_v_f32[0], bh * dma_chunk_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_v_f32[1], bh * dma_chunk_sz, pipeline);
VTCM_LAYOUT_ALLOC(off, off_g_f32[0], bh * dma_scalar_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_f32[1], bh * dma_scalar_sz, pipeline);
VTCM_LAYOUT_ALLOC(off, off_b_f32[0], bh * dma_scalar_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_b_f32[1], bh * dma_scalar_sz, pipeline);
const size_t raw_gb_sz = hex_round_up(bh * chunk_size * sizeof(float), 128);
VTCM_LAYOUT_ALLOC(off, off_g_raw[0], raw_gb_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_g_raw[1], raw_gb_sz, pipeline);
VTCM_LAYOUT_ALLOC(off, off_b_raw[0], raw_gb_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_b_raw[1], raw_gb_sz, pipeline);
VTCM_LAYOUT_ALLOC(off, off_o_f32[0], bh * dma_chunk_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_o_f32[1], bh * dma_chunk_sz, pipeline);
VTCM_LAYOUT_ALLOC(off, off_v_inter_f32, bh * dma_chunk_sz);
VTCM_LAYOUT_ALLOC(off, off_o_inter_f32, bh * dma_chunk_sz);
VTCM_LAYOUT_ALLOC(off, off_o_intra_f32, bh * dma_chunk_sz);
VTCM_LAYOUT_ALLOC(off, off_k_f16, bh * act_f16_sz);
VTCM_LAYOUT_ALLOC(off, off_v_prime_f16, bh * act_f16_sz);
VTCM_LAYOUT_ALLOC(off, off_delta_f16, bh * act_f16_sz);
VTCM_LAYOUT_ALLOC(off, off_d_f16, bh * act_f16_sz);
off = hex_align_up(off, HMX_FP16_TILE_SIZE);
VTCM_LAYOUT_ALLOC(off, off_q_row_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_q_prime_row_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_k_row_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_k_col_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_k_prime_row_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_k_col_tiles_64x128, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_kk_tiles, bh * tile_64x64_sz);
VTCM_LAYOUT_ALLOC(off, off_qk_tiles, bh * tile_64x64_sz);
VTCM_LAYOUT_ALLOC(off, off_v_inter_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_o_inter_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_inv_row_tiles, bh * tile_64x64_sz);
VTCM_LAYOUT_ALLOC(off, off_a_row_tiles, bh * tile_64x64_sz);
VTCM_LAYOUT_ALLOC(off, off_v_prime_col_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_delta_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_delta_col_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_o_intra_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_d_row_tiles, bh * tile_64xSv_sz);
VTCM_LAYOUT_ALLOC(off, off_gamma, bh * hex_round_up(chunk_size * sizeof(float), 128));
VTCM_LAYOUT_ALLOC(off, off_lambda_init, bh * hex_round_up(chunk_size * sizeof(float), 128));
VTCM_LAYOUT_ALLOC(off, off_decay_m, bh * decay_sz);
VTCM_LAYOUT_ALLOC(off, off_decay_a, bh * decay_sz);
VTCM_LAYOUT_ALLOC(off, off_rows_kk, bh * row_vecs_sz);
VTCM_LAYOUT_ALLOC(off, off_rows_qk, bh * row_vecs_sz);
VTCM_LAYOUT_ALLOC(off, off_rows_inv, bh * row_vecs_sz);
VTCM_LAYOUT_ALLOC(off, off_rows_a, bh * row_vecs_sz);
const size_t thread_scratch_sz = 64 * 128;
off = hex_align_up(off, HMX_FP16_TILE_SIZE);
VTCM_LAYOUT_ALLOC(off, off_thread_scratch, nth * thread_scratch_sz);
off = hex_align_up(off, HMX_FP16_TILE_SIZE);
VTCM_LAYOUT_ALLOC(off, off_scales_1, HMX_FP16_TILE_SIZE);
L->total_bytes = off;
}
static inline bool htp_gdn_hmx_solve_layout(
struct htp_gdn_hmx_vtcm_layout * layout_out,
uint32_t S_v,
uint32_t chunk_size,
uint32_t total_rows,
size_t vtcm_budget,
uint32_t n_threads,
bool pipeline,
uint32_t * n_heads_batch_out
) {
uint32_t max_batch = 8;
if (max_batch > total_rows) {
max_batch = total_rows;
}
if (max_batch > n_threads) {
max_batch = n_threads;
}
static const uint32_t candidates[] = { 8, 6, 4, 2, 1 };
for (size_t i = 0; i < sizeof(candidates) / sizeof(candidates[0]); ++i) {
uint32_t bh = candidates[i];
if (bh > max_batch) {
continue;
}
struct htp_gdn_hmx_vtcm_layout L;
htp_gdn_hmx_vtcm_layout_build(&L, S_v, chunk_size, bh, n_threads, pipeline);
if (L.total_bytes <= vtcm_budget) {
*layout_out = L;
*n_heads_batch_out = bh;
return true;
}
}
if (pipeline) {
return htp_gdn_hmx_solve_layout(layout_out, S_v, chunk_size, total_rows, vtcm_budget, n_threads, false, n_heads_batch_out);
}
return false;
}
#endif // HTP_GATED_DELTA_NET_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/gather-rows.c
+403
View File
@@ -0,0 +1,403 @@
#include "gather-rows.h"
#include <assert.h>
#include "hex-common.h"
#include "hex-profile.h"
#include "hvx-utils.h"
#include "dma-queue.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "work-queue.h"
typedef void (*gather_rows_compute_fn_t)(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur);
struct gather_rows_task {
struct htp_ops_context * octx;
const struct htp_tensor * a;
const struct htp_tensor * b;
dma_addr_t dst;
uint32_t r_start;
uint32_t rows;
uint32_t rows_per_thread;
const struct htp_gather_rows_params * kparams;
gather_rows_compute_fn_t compute;
};
static inline __attribute__((always_inline)) void gather_rows_dma_in(
dma_queue * q, const struct gather_rows_task * task,
uint8_t * region, uint32_t r, uint32_t nr, uint32_t elem_size) {
const struct htp_tensor * a = task->a;
const struct htp_tensor * b = task->b;
const struct htp_gather_rows_params * kparams = task->kparams;
const size_t a_size = (nr - 1) * a->nb[1] + kparams->na * elem_size;
dma_queue_push(q, dma_make_data(region, a->data + r * a->nb[1]), a_size, a_size, a_size, 1);
if (b) {
if (kparams->b_dense) {
const size_t b_size = (nr - 1) * b->nb[1] + kparams->nb * elem_size;
dma_queue_push(q, dma_make_data(region + kparams->v1, b->data + r * b->nb[1]), b_size, b_size, b_size, 1);
} else {
const size_t b_size = (nr - 1) * b->nb[1] + elem_size;
for (uint32_t k = 0; k < kparams->nb; k++) {
dma_queue_push(q, dma_make_data(region + kparams->v1 + k * kparams->span1,
b->data + k * b->nb[0] + r * b->nb[1]), b_size, b_size, b_size, 1);
}
}
}
}
static inline void gather_rows_dma_out(dma_queue * q, const struct gather_rows_task * task,
uint8_t * out, uint32_t r, uint32_t nr, uint32_t elem_size) {
const struct htp_gather_rows_params * kparams = task->kparams;
const size_t out_size = nr * kparams->ne * elem_size;
dma_queue_push(q, dma_make_data(task->dst + r * kparams->ne * elem_size, out), out_size, out_size, out_size, 1);
}
static __attribute__((noinline)) void gather_rows_init_tables_f32(
HVX_Vector * tab, HVX_Vector * inc,
const struct htp_gather_rows_params * kparams,
const struct htp_tensor * a, const struct htp_tensor * b) {
const int32_t s_a = kparams->rg * a->nb[1];
const int32_t s_b = b ? kparams->rg * b->nb[1] : 0;
const uint32_t b_nb1 = b ? b->nb[1] : 0;
uint32_t row = 0;
uint32_t col = 0;
int32_t t_buf[32] __attribute__((aligned(128)));
int32_t s_buf[32] __attribute__((aligned(128)));
for (uint32_t k = 0; k < kparams->vg; k++) {
for (uint32_t lane = 0; lane < 32; lane++) {
if (col < kparams->na) {
t_buf[lane] = row * a->nb[1] + col * 4;
s_buf[lane] = s_a;
} else {
t_buf[lane] = kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1;
s_buf[lane] = s_b;
}
if (++col == kparams->ne) {
col = 0;
row++;
}
}
tab[k] = *(const HVX_Vector *) t_buf;
inc[k] = *(const HVX_Vector *) s_buf;
}
}
static __attribute__((noinline)) void gather_rows_init_tables_f16(
HVX_Vector * tab, HVX_Vector * inc,
const struct htp_gather_rows_params * kparams,
const struct htp_tensor * a, const struct htp_tensor * b) {
const int16_t s_a = (int16_t) (kparams->rg * a->nb[1]);
const int16_t s_b = (int16_t) (b ? kparams->rg * b->nb[1] : 0);
const uint32_t b_nb1 = b ? b->nb[1] : 0;
uint32_t row = 0;
uint32_t col = 0;
int16_t t_buf[64] __attribute__((aligned(128)));
int16_t s_buf[64] __attribute__((aligned(128)));
for (uint32_t k = 0; k < kparams->vg; k++) {
for (uint32_t lane = 0; lane < 64; lane++) {
if (col < kparams->na) {
t_buf[lane] = (int16_t) (row * a->nb[1] + col * 2);
s_buf[lane] = s_a;
} else {
t_buf[lane] = (int16_t) (kparams->v1 + (col - kparams->na) * kparams->span1 + row * b_nb1);
s_buf[lane] = s_b;
}
if (++col == kparams->ne) {
col = 0;
row++;
}
}
tab[k] = *(const HVX_Vector *) t_buf;
inc[k] = *(const HVX_Vector *) s_buf;
}
}
static void gather_rows_compute_f32_vg1(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
(void) cur;
const uint32_t region_size = kparams->region_size;
HVX_Vector vcur = tab[0];
const HVX_Vector vinc = inc[0];
#pragma unroll(4)
for (uint32_t v = 0; v < nvec; v++) {
Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);
vcur = Q6_Vw_vadd_VwVw(vcur, vinc);
}
}
static void gather_rows_compute_f32_vgn(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
const uint32_t vg = kparams->vg;
const uint32_t region_size = kparams->region_size;
for (uint32_t k = 0; k < vg; k++) {
cur[k] = tab[k];
}
uint32_t k = 0;
#pragma unroll(4)
for (uint32_t v = 0; v < nvec; v++) {
Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);
cur[k] = Q6_Vw_vadd_VwVw(cur[k], inc[k]);
if (++k == vg) {
k = 0;
}
}
}
static void gather_rows_compute_f16_vg1(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
(void) cur;
const uint32_t region_size = kparams->region_size;
HVX_Vector vcur = tab[0];
const HVX_Vector vinc = inc[0];
#pragma unroll(4)
for (uint32_t v = 0; v < nvec; v++) {
Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, vcur);
vcur = Q6_Vh_vadd_VhVh(vcur, vinc);
}
}
static void gather_rows_compute_f16_vgn(
uint8_t * restrict out, const uint8_t * restrict region, uint32_t nvec,
const struct htp_gather_rows_params * restrict kparams,
const HVX_Vector * restrict tab, const HVX_Vector * restrict inc, HVX_Vector * restrict cur) {
const uint32_t vg = kparams->vg;
const uint32_t region_size = kparams->region_size;
for (uint32_t k = 0; k < vg; k++) {
cur[k] = tab[k];
}
uint32_t k = 0;
#pragma unroll(4)
for (uint32_t v = 0; v < nvec; v++) {
Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) region, region_size, cur[k]);
cur[k] = Q6_Vh_vadd_VhVh(cur[k], inc[k]);
if (++k == vg) {
k = 0;
}
}
}
static void gather_rows_thread_f32(unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct gather_rows_task * task = (const struct gather_rows_task *) data;
const struct htp_gather_rows_params * kparams = task->kparams;
const struct htp_tensor * a = task->a;
const struct htp_tensor * b = task->b;
const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
const uint32_t r_beg = task->r_start + r_offset;
const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
if (r_beg >= r_end) {
return;
}
struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
struct htp_gather_rows_vtcm_layout layout;
htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 4);
uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice);
HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab);
HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc);
HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur);
uint8_t * region[2] = {
VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0),
VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1),
};
uint8_t * out[2] = {
VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0),
VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1),
};
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
gather_rows_init_tables_f32(tab, inc, kparams, a, b);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
dma_queue * q = task->octx->ctx->dma[ith];
const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
const uint32_t nr_max = kparams->nr_max;
const uint32_t nr0 = MIN(nr_max, r_end - r_beg);
dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out
gather_rows_dma_in(q, task, region[0], r_beg, nr0, 4);
const uint32_t r1 = r_beg + nr_max;
if (r1 < r_end) {
const uint32_t nr1 = MIN(nr_max, r_end - r1);
dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out
gather_rows_dma_in(q, task, region[1], r1, nr1, 4);
}
uint32_t buf = 0;
for (uint32_t r = r_beg; r < r_end; r += nr_max) {
const uint32_t nr = MIN(nr_max, r_end - r);
const uint32_t next_buf = buf ^ 1;
dma_queue_pop(q); // complete out
for (uint32_t i = 0; i < n_in; i++) {
dma_queue_pop(q);
}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
const uint32_t nvec = (nr * kparams->ne + 31) >> 5;
task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);
hvx_gather_sync(out[buf]);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
gather_rows_dma_out(q, task, out[buf], r, nr, 4);
const uint32_t r_prefetch = r + 2 * nr_max;
if (r_prefetch < r_end) {
const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);
gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 4);
}
buf = next_buf;
}
dma_queue_flush(q);
}
static void gather_rows_thread_f16(unsigned int nth, unsigned int ith, void * data) {
(void) nth;
const struct gather_rows_task * task = (const struct gather_rows_task *) data;
const struct htp_gather_rows_params * kparams = task->kparams;
const struct htp_tensor * a = task->a;
const struct htp_tensor * b = task->b;
const uint32_t r_offset = MIN(ith * task->rows_per_thread, task->rows);
const uint32_t r_beg = task->r_start + r_offset;
const uint32_t r_end = task->r_start + MIN(r_offset + task->rows_per_thread, task->rows);
if (r_beg >= r_end) {
return;
}
struct htp_thread_trace * tr = &task->octx->ctx->trace[ith];
struct htp_gather_rows_vtcm_layout layout;
htp_gather_rows_vtcm_layout_build(&layout, kparams->vg, kparams->region_size, kparams->nr_max, kparams->ne, 2);
uint8_t * base = VTCM_LAYOUT_PTR(uint8_t, task->octx->ctx->vtcm_base, (size_t) ith * kparams->slice);
HVX_Vector * tab = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_tab);
HVX_Vector * inc = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_inc);
HVX_Vector * cur = VTCM_LAYOUT_PTR(HVX_Vector, base, layout.off_cur);
uint8_t * region[2] = {
VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region0),
VTCM_LAYOUT_PTR(uint8_t, base, layout.off_region1),
};
uint8_t * out[2] = {
VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out0),
VTCM_LAYOUT_PTR(uint8_t, base, layout.off_out1),
};
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
gather_rows_init_tables_f16(tab, inc, kparams, a, b);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, (uint16_t) r_beg);
dma_queue * q = task->octx->ctx->dma[ith];
const uint32_t n_in = 1 + (task->b ? (kparams->b_dense ? 1 : kparams->nb) : 0);
const uint32_t nr_max = kparams->nr_max;
const uint32_t nr0 = MIN(nr_max, r_end - r_beg);
dma_queue_push(q, dma_make_data(task->dst, out[0]), 0, 0, 0, 0); // dummy out
gather_rows_dma_in(q, task, region[0], r_beg, nr0, 2);
const uint32_t r1 = r_beg + nr_max;
if (r1 < r_end) {
const uint32_t nr1 = MIN(nr_max, r_end - r1);
dma_queue_push(q, dma_make_data(task->dst, out[1]), 0, 0, 0, 0); // dummy out
gather_rows_dma_in(q, task, region[1], r1, nr1, 2);
}
uint32_t buf = 0;
for (uint32_t r = r_beg; r < r_end; r += nr_max) {
const uint32_t nr = MIN(nr_max, r_end - r);
const uint32_t next_buf = buf ^ 1;
dma_queue_pop(q); // complete out
for (uint32_t i = 0; i < n_in; i++) {
dma_queue_pop(q);
}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
const uint32_t nvec = (nr * kparams->ne + 63) >> 6;
task->compute(out[buf], region[buf], nvec, kparams, tab, inc, cur);
hvx_gather_sync(out[buf]);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r);
gather_rows_dma_out(q, task, out[buf], r, nr, 2);
const uint32_t r_prefetch = r + 2 * nr_max;
if (r_prefetch < r_end) {
const uint32_t nr_prefetch = MIN(nr_max, r_end - r_prefetch);
gather_rows_dma_in(q, task, region[buf], r_prefetch, nr_prefetch, 2);
}
buf = next_buf;
}
dma_queue_flush(q);
}
int htp_gather_rows(
struct htp_ops_context * octx,
const struct htp_tensor * a,
const struct htp_tensor * b,
const struct htp_gather_rows_params * kparams,
uint32_t n_threads) {
if (!htp_ops_context_set_n_threads(octx, n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
assert(kparams->slice * octx->n_threads <= octx->ctx->vtcm_size);
const uint32_t total_rows = kparams->total_rows;
uint32_t r_start = 0;
uint32_t r_count = total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
total_rows, kparams->rg, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
r_start = range.start;
r_count = range.count;
}
if (r_count == 0) {
return HTP_STATUS_OK;
}
const uint32_t elem_size = (a->type == HTP_TYPE_F16) ? 2 : 4;
struct gather_rows_task task;
task.octx = octx;
task.a = a;
task.b = b;
task.dst = octx->dst->data;
task.r_start = r_start;
task.rows = r_count;
task.kparams = kparams;
task.rows_per_thread = kparams->rows_per_thread;
if (elem_size == 2) {
task.compute = (kparams->vg == 1) ? gather_rows_compute_f16_vg1 : gather_rows_compute_f16_vgn;
} else {
task.compute = (kparams->vg == 1) ? gather_rows_compute_f32_vg1 : gather_rows_compute_f32_vgn;
}
work_queue_func_t worker_func = (elem_size == 2) ? gather_rows_thread_f16 : gather_rows_thread_f32;
work_queue_run(octx->ctx->work_queue, worker_func, &task, octx->n_threads);
return HTP_STATUS_OK;
}
llama.cpp/ggml/src/ggml-hexagon/htp/gather-rows.h
+189
View File
@@ -0,0 +1,189 @@
#ifndef HTP_GATHER_ROWS_H
#define HTP_GATHER_ROWS_H
#include "hex-common.h"
#include "htp-vtcm.h"
#include <stdint.h>
#include <stdbool.h>
#ifndef GGML_MAX_DIMS
#define GGML_MAX_DIMS 4
#endif
struct htp_gather_rows_params {
uint32_t na;
uint32_t nb;
uint32_t ne;
uint32_t rg;
uint32_t vg;
uint32_t nr_max;
uint32_t v1;
uint32_t span1;
uint32_t region_size;
uint32_t slice;
uint32_t total_rows;
uint32_t rows_per_thread;
uint8_t b_dense;
uint8_t pad[3];
};
struct htp_gather_rows_vtcm_layout {
size_t off_tab;
size_t off_inc;
size_t off_cur;
size_t off_region0;
size_t off_out0;
size_t off_region1;
size_t off_out1;
size_t total_per_thread;
};
static inline void htp_gather_rows_vtcm_layout_build(
struct htp_gather_rows_vtcm_layout * L,
uint32_t vg,
uint32_t region_size,
uint32_t nr_max,
uint32_t ne,
uint32_t elem_size) {
const size_t vec_bytes = (size_t) vg * 128;
const size_t region_aligned = hex_round_up(region_size, 128);
const size_t out_aligned = hex_round_up(region_aligned + (size_t) nr_max * ne * elem_size, 128) - region_aligned;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_tab, vec_bytes);
VTCM_LAYOUT_ALLOC(off, off_inc, vec_bytes);
VTCM_LAYOUT_ALLOC(off, off_cur, vec_bytes);
VTCM_LAYOUT_ALLOC(off, off_region0, region_aligned);
VTCM_LAYOUT_ALLOC(off, off_out0, out_aligned);
VTCM_LAYOUT_ALLOC(off, off_region1, region_aligned);
VTCM_LAYOUT_ALLOC(off, off_out1, out_aligned);
L->total_per_thread = off;
}
static inline bool htp_gather_rows_solve_layout(
struct htp_gather_rows_params * kparams,
uint32_t na, uint32_t a_nb1,
uint32_t nb, uint32_t b_nb1, bool b_dense,
uint32_t elem_size,
uint32_t total_rows,
uint32_t n_threads,
uint32_t mdev_count,
uint32_t vtcm_size) {
const uint32_t max_ne0 = 128 / elem_size;
const uint32_t ne = na + nb;
if (ne == 0 || ne > max_ne0) {
return false;
}
if (elem_size != 4 && elem_size != 2) {
return false;
}
const uint32_t v_lanes = 128 / elem_size;
const uint32_t g = hex_gcd_u32(ne, v_lanes);
const uint32_t rg = v_lanes / g;
const uint32_t vg = ne / g;
const uint32_t slice = (vtcm_size / n_threads) & ~127u;
const uint32_t total_chunks = total_rows / rg;
const uint32_t chunks_per_dev = (total_chunks + mdev_count - 1) / mdev_count;
const uint32_t max_rows_per_dev = (mdev_count > 1 && total_chunks >= mdev_count)
? (chunks_per_dev * rg + (total_rows % rg))
: total_rows;
const uint32_t rows_per_thread = hex_round_up((max_rows_per_dev + n_threads - 1) / n_threads, rg);
const uint32_t fixed = 3 * vg * 128;
const uint32_t per_row = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0) + ne * elem_size;
if (slice <= fixed + 2 * per_row * rg) {
return false;
}
const uint32_t avail_per_buf = (slice - fixed) / 2;
uint32_t nr_max = MIN(avail_per_buf / per_row, rows_per_thread);
if (elem_size == 2) {
const uint32_t per_row_region = a_nb1 + (nb > 0 ? (b_dense ? b_nb1 : nb * b_nb1) : 0);
if (per_row_region > 0) {
nr_max = MIN(nr_max, (32768u - 256u) / per_row_region);
}
}
nr_max = (nr_max / rg) * rg;
uint32_t v1 = 0;
uint32_t span1 = 0;
uint32_t region_size = 0;
while (nr_max > 0) {
v1 = hex_round_up((nr_max - 1) * a_nb1 + na * elem_size, 128);
if (nb > 0) {
if (b_dense) {
span1 = elem_size;
region_size = v1 + (nr_max - 1) * b_nb1 + nb * elem_size;
} else {
span1 = (nr_max - 1) * b_nb1 + elem_size;
region_size = v1 + nb * span1;
}
} else {
span1 = 0;
region_size = v1;
}
struct htp_gather_rows_vtcm_layout layout;
htp_gather_rows_vtcm_layout_build(&layout, vg, region_size, nr_max, ne, elem_size);
if ((elem_size != 2 || region_size <= 32768) && layout.total_per_thread <= slice) {
break;
}
nr_max -= rg;
}
if (nr_max == 0) {
return false;
}
kparams->na = na;
kparams->nb = nb;
kparams->ne = ne;
kparams->rg = rg;
kparams->vg = vg;
kparams->nr_max = nr_max;
kparams->v1 = v1;
kparams->span1 = span1;
kparams->region_size = region_size;
kparams->slice = slice;
kparams->total_rows = total_rows;
kparams->rows_per_thread = rows_per_thread;
kparams->b_dense = b_dense ? 1 : 0;
kparams->pad[0] = 0;
kparams->pad[1] = 0;
kparams->pad[2] = 0;
return true;
}
#ifdef __cplusplus
struct ggml_tensor;
static inline bool htp_gather_rows_tensor_outer_contiguous(const struct ggml_tensor * t) {
size_t next_nb = ((const size_t *) t->nb)[1] * ((const int64_t *) t->ne)[1];
for (int i = 2; i < GGML_MAX_DIMS; i++) {
if (((const int64_t *) t->ne)[i] != 1 && ((const size_t *) t->nb)[i] != next_nb) {
return false;
}
next_nb *= ((const int64_t *) t->ne)[i];
}
return true;
}
#else
struct htp_ops_context;
struct htp_tensor;
int htp_gather_rows(
struct htp_ops_context * octx,
const struct htp_tensor * a,
const struct htp_tensor * b,
const struct htp_gather_rows_params * kparams,
uint32_t n_threads);
#endif
#endif // HTP_GATHER_ROWS_H
llama.cpp/ggml/src/ggml-hexagon/htp/get-rows-ops.c
+510
View File
@@ -0,0 +1,510 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic ignored "-Wunused-function"
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <math.h>
#include <string.h>
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "dma-queue.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-utils.h"
#include "hvx-quant.h"
#include "get-rows-ops.h"
#include "work-queue.h"
struct get_rows_context;
typedef uint32_t (*get_rows_read_index_fn_t)(const void * src1, uint32_t idx, int idx_type);
typedef void (*get_rows_flat_compute_fn_t)(float * dst_spad, const void * src_spad, uint32_t cur_elems);
typedef void (*get_rows_tiled_compute_fn_t)(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row);
struct get_rows_context {
struct htp_ops_context * octx;
const struct htp_get_rows_kernel_params * kparams;
struct htp_get_rows_vtcm_layout vtcm_layout;
uint8_t * vtcm_base;
const void * src1_data;
get_rows_read_index_fn_t read_index;
get_rows_flat_compute_fn_t compute_flat;
get_rows_tiled_compute_fn_t compute_tiled;
uint32_t task_start;
uint32_t tasks;
uint32_t tasks_per_thread;
uint32_t tile_size;
uint32_t tile_stride;
uint32_t src0_row_size;
uint32_t dst_row_size;
int idx_type;
};
#define get_rows_preamble \
const uint32_t ne00 = octx->src[0]->ne[0]; \
const uint32_t ne01 = octx->src[0]->ne[1]; \
const uint32_t ne02 = octx->src[0]->ne[2]; \
const uint32_t ne03 = octx->src[0]->ne[3]; \
\
const uint32_t ne10 = octx->src[1]->ne[0]; \
const uint32_t ne11 = octx->src[1]->ne[1]; \
const uint32_t ne12 = octx->src[1]->ne[2]; \
const uint32_t ne13 = octx->src[1]->ne[3]; \
\
const uint32_t ne0 = octx->dst->ne[0]; \
const uint32_t ne1 = octx->dst->ne[1]; \
const uint32_t ne2 = octx->dst->ne[2]; \
const uint32_t ne3 = octx->dst->ne[3]; \
\
const uint32_t nb01 = octx->src[0]->nb[1]; \
const uint32_t nb02 = octx->src[0]->nb[2]; \
const uint32_t nb03 = octx->src[0]->nb[3]; \
\
const uint32_t nb10 = octx->src[1]->nb[0]; \
const uint32_t nb11 = octx->src[1]->nb[1]; \
const uint32_t nb12 = octx->src[1]->nb[2]; \
\
const uint32_t nb1 = octx->dst->nb[1]; \
const uint32_t nb2 = octx->dst->nb[2]; \
const uint32_t nb3 = octx->dst->nb[3]; \
\
const uint32_t nr = ne10 * ne11 * ne12;
static uint32_t get_rows_read_index_ddr(
const void * src1,
uint32_t idx,
int idx_type
) {
const uint32_t off = idx * ((idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t));
return *(const uint32_t *) ((const uint8_t *) src1 + off);
}
static __attribute__((noinline)) uint32_t get_rows_read_index_vtcm(
const void * src1,
uint32_t idx,
int idx_type
) {
const uint32_t off = idx * ((idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t));
const HVX_Vector v = hvx_vmem((const uint8_t *) src1 + off); // off & ~127u in HW
// vextract is normally slow, but here it avoids an HVX->L2 store and L1D miss
return (uint32_t) Q6_R_vextract_VR(v, off);
}
static void get_rows_thread_sametype(unsigned int nth, unsigned int ith, void * data) {
struct get_rows_context * grctx = (struct get_rows_context *) data;
struct htp_ops_context * octx = grctx->octx;
const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
get_rows_preamble;
const uint32_t dr = grctx->tasks_per_thread;
const uint32_t ir0 = grctx->task_start + dr * ith;
if (ir0 >= grctx->task_start + grctx->tasks) {
return;
}
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
const uint32_t row_size_bytes = grctx->src0_row_size;
dma_queue * dma_q = octx->ctx->dma[ith];
for (uint32_t i = ir0; i < ir1; ++i) {
const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);
const uint32_t rem = i - i12 * ne11 * ne10;
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);
const uint32_t i10 = rem - i11 * ne10;
const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type);
assert(i01 < ne01);
const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);
const uint32_t i02 = i11 - q02 * ne02;
const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);
const uint32_t i03 = i12 - q03 * ne03;
const dma_addr_t src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03;
const dma_addr_t dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;
while (!dma_queue_push(dma_q, dma_make_data(dst_data, src0_data), nb1, nb01, row_size_bytes, 1)) {
dma_queue_pop(dma_q);
}
}
dma_queue_flush(dma_q);
}
struct get_rows_flat_task {
dma_addr_t src0_data;
dma_addr_t dst_data;
};
static inline struct get_rows_flat_task get_rows_flat_calc_task(
const struct htp_ops_context * octx,
const struct get_rows_context * grctx,
uint32_t i,
bool need_src0
) {
const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
get_rows_preamble;
const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);
const uint32_t rem = i - i12 * ne11 * ne10;
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);
const uint32_t i10 = rem - i11 * ne10;
struct get_rows_flat_task task;
task.src0_data = 0;
task.dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;
if (need_src0) {
const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type);
assert(i01 < ne01);
const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);
const uint32_t i02 = i11 - q02 * ne02;
const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);
const uint32_t i03 = i12 - q03 * ne03;
task.src0_data = octx->src[0]->data + i01*nb01 + i02*nb02 + i03*nb03;
}
return task;
}
static void get_rows_thread_flat(unsigned int nth, unsigned int ith, void * data) {
struct get_rows_context * grctx = (struct get_rows_context *) data;
struct htp_ops_context * octx = grctx->octx;
get_rows_preamble;
const uint32_t dr = grctx->tasks_per_thread;
const uint32_t ir0 = grctx->task_start + dr * ith;
if (ir0 >= grctx->task_start + grctx->tasks) {
return;
}
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
dma_queue * dma_q = octx->ctx->dma[ith];
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
uint8_t * vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread;
uint8_t * vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread;
const uint32_t src0_bytes = grctx->src0_row_size;
const uint32_t dst_bytes = grctx->dst_row_size;
get_rows_flat_compute_fn_t compute = grctx->compute_flat;
for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, spad_idx++) {
const uint32_t i = ir0 + step;
struct get_rows_flat_task task = get_rows_flat_calc_task(octx, grctx, i, true);
dma_queue_push(dma_q,
dma_make_data(octx->dst->data,
vtcm_dst + spad_idx * vtcm_layout->dst_spad_half_size),
dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0);
dma_queue_push(dma_q,
dma_make_data(vtcm_src0 + spad_idx * vtcm_layout->src0_spad_half_size, task.src0_data),
vtcm_layout->src0_spad_half_size, src0_bytes, src0_bytes, 1);
}
for (uint32_t step = 0; step < ir1 - ir0; ++step) {
const uint32_t i = ir0 + step;
void * dst_spad = (void *) dma_queue_pop(dma_q).src;
void * src_spad = (void *) dma_queue_pop(dma_q).dst;
struct get_rows_flat_task task = get_rows_flat_calc_task(octx, grctx, i, false);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
compute((float *) dst_spad, src_spad, ne00);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
dma_queue_push(dma_q,
dma_make_data(task.dst_data, dst_spad),
dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1);
const uint32_t next_step = step + 2;
if (next_step < ir1 - ir0) {
const uint32_t pi = ir0 + next_step;
struct get_rows_flat_task next_task = get_rows_flat_calc_task(octx, grctx, pi, true);
dma_queue_push(dma_q,
dma_make_data(src_spad, next_task.src0_data),
vtcm_layout->src0_spad_half_size, src0_bytes, src0_bytes, 1);
}
}
dma_queue_flush(dma_q);
}
struct get_rows_tiled_task {
dma_addr_t tile_src_base;
dma_addr_t dst_data;
uint32_t row;
};
static inline struct get_rows_tiled_task get_rows_tiled_calc_task(
const struct htp_ops_context * octx,
const struct get_rows_context * grctx,
uint32_t i,
uint32_t n_k_tiles,
uint32_t tile_size
) {
const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
get_rows_preamble;
const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);
const uint32_t rem = i - i12 * ne11 * ne10;
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);
const uint32_t i10 = rem - i11 * ne10;
const uint32_t i01 = grctx->read_index(grctx->src1_data, i, grctx->idx_type);
assert(i01 < ne01);
const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);
const uint32_t i02 = i11 - q02 * ne02;
const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);
const uint32_t i03 = i12 - q03 * ne03;
const uint32_t column_tile = i01 / HTP_TENSOR_TILE_N_ROWS;
const uint32_t row = i01 % HTP_TENSOR_TILE_N_ROWS;
const dma_addr_t matrix = octx->src[0]->data + i02*nb02 + i03*nb03;
struct get_rows_tiled_task task;
task.tile_src_base = matrix + (column_tile * n_k_tiles) * tile_size;
task.dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;
task.row = row;
return task;
}
static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) {
struct get_rows_context * grctx = (struct get_rows_context *) data;
struct htp_ops_context * octx = grctx->octx;
get_rows_preamble;
const uint32_t dr = grctx->tasks_per_thread;
const uint32_t ir0 = grctx->task_start + dr * ith;
if (ir0 >= grctx->task_start + grctx->tasks) {
return;
}
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(octx->src[0]->type);
const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
uint8_t * src_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_src0) + ith * vtcm_layout->src0_bytes_per_thread;
uint8_t * dst_spad_base = VTCM_LAYOUT_PTR(uint8_t, grctx->vtcm_base, vtcm_layout->off_dst) + ith * vtcm_layout->dst_bytes_per_thread;
dma_queue * dma_q = octx->ctx->dma[ith];
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
const uint32_t tile_size = grctx->tile_size;
const uint32_t tile_stride = grctx->tile_stride;
const uint32_t dst_bytes = grctx->dst_row_size;
get_rows_tiled_compute_fn_t compute_tiled = grctx->compute_tiled;
for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) {
const uint32_t i = ir0 + step;
struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size);
// Dummy writeback to prime the queue with dst descriptor
dma_queue_push(dma_q,
dma_make_data(task.dst_data, dst_spad_base + spad_idx * vtcm_layout->dst_spad_half_size),
dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0);
// Prefetch row tiles
dma_queue_push(dma_q,
dma_make_data(src_spad_base + spad_idx * vtcm_layout->src0_spad_half_size, task.tile_src_base),
tile_stride, tile_size, tile_size, n_k_tiles);
}
for (uint32_t step = 0; step < ir1 - ir0; ++step) {
const uint32_t i = ir0 + step;
float * dst_spad = (float *) dma_queue_pop(dma_q).src;
uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_q).dst;
struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
compute_tiled(dst_spad, src_spad, n_k_tiles, tile_stride, task.row);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
// Real writeback of dst_spad
dma_queue_push(dma_q,
dma_make_data(task.dst_data, dst_spad),
dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1);
const uint32_t next_step = step + 2;
if (next_step < ir1 - ir0) {
const uint32_t ni = ir0 + next_step;
struct get_rows_tiled_task next_task = get_rows_tiled_calc_task(octx, grctx, ni, n_k_tiles, tile_size);
dma_queue_push(dma_q,
dma_make_data(src_spad, next_task.tile_src_base),
tile_stride, tile_size, tile_size, n_k_tiles);
}
}
dma_queue_flush(dma_q);
}
static __attribute__((noinline)) void get_rows_dma_src1(
struct htp_ops_context * octx,
uint8_t * vtcm_src1,
uint32_t elem_size
) {
const uint32_t ne10 = octx->src[1]->ne[0];
const uint32_t ne11 = octx->src[1]->ne[1];
const uint32_t ne12 = octx->src[1]->ne[2];
const uint32_t ne13 = octx->src[1]->ne[3];
const uint32_t nb11 = octx->src[1]->nb[1];
const uint32_t nb12 = octx->src[1]->nb[2];
const uint32_t nb13 = octx->src[1]->nb[3];
dma_queue * dma_q = octx->ctx->dma[0];
uint8_t * cur_dst = vtcm_src1;
for (uint32_t i13 = 0; i13 < ne13; ++i13) {
for (uint32_t i12 = 0; i12 < ne12; ++i12) {
const dma_addr_t src_plane = octx->src[1]->data + i12 * nb12 + i13 * nb13;
while (!dma_queue_push(dma_q, dma_make_data(cur_dst, src_plane),
ne10 * elem_size, nb11, ne10 * elem_size, ne11)) {
dma_queue_pop(dma_q);
}
cur_dst += ne10 * ne11 * elem_size;
}
}
dma_queue_flush(dma_q);
}
int op_get_rows(struct htp_ops_context * octx) {
const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;
const uint32_t ne00 = octx->src[0]->ne[0];
const uint32_t src0_row_size = htp_tensor_get_row_size(octx->src[0]->type, ne00);
if (src0_row_size == 0) {
return HTP_STATUS_NO_SUPPORT;
}
if (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) {
if (octx->src[0]->type != octx->dst->type) {
return HTP_STATUS_NO_SUPPORT;
}
} else {
if (octx->dst->type != HTP_TYPE_F32) {
return HTP_STATUS_NO_SUPPORT;
}
}
if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) {
return HTP_STATUS_NO_SUPPORT;
}
const struct htp_tensor * dst = octx->dst;
const uint32_t total_tasks = kparams->total_tasks;
const uint32_t dst_row_size = (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) ? src0_row_size : ne00 * sizeof(float);
uint32_t task_start = 0;
uint32_t tasks = total_tasks;
if (octx->ctx->mdev.count > 1) {
uint32_t tasks_per_chunk = 1;
htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) dst->nb[0], dst_row_size, &tasks_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
task_start = range.start;
tasks = range.count;
}
if (tasks == 0) {
return HTP_STATUS_OK;
}
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t n_threads = octx->n_threads;
struct get_rows_context grctx;
grctx.octx = octx;
grctx.kparams = kparams;
grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
grctx.compute_flat = NULL;
grctx.compute_tiled = NULL;
grctx.task_start = task_start;
grctx.tasks = tasks;
grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
grctx.tile_size = 0;
grctx.tile_stride = 0;
grctx.src0_row_size = src0_row_size;
grctx.dst_row_size = dst_row_size;
grctx.idx_type = octx->src[1]->type;
htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, total_tasks, grctx.idx_type, n_threads);
if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) {
FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n",
octx->ctx->vtcm_size, grctx.vtcm_layout.total_bytes);
return HTP_STATUS_INVAL_PARAMS;
}
work_queue_func_t q_func = NULL;
switch (kparams->kernel_type) {
case HTP_GET_ROWS_KERNEL_SAMETYPE:
q_func = get_rows_thread_sametype;
break;
case HTP_GET_ROWS_KERNEL_TILED:
switch (octx->src[0]->type) {
case HTP_TYPE_F16: grctx.compute_tiled = hvx_dequantize_row_tiled_f16_f32; break;
case HTP_TYPE_Q1_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q1_0_f32; break;
case HTP_TYPE_Q2_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q2_0_f32; break;
case HTP_TYPE_Q8_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q8_0_f32; break;
case HTP_TYPE_Q4_0: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_0_f32; break;
case HTP_TYPE_Q4_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q4_k_f32; break;
case HTP_TYPE_Q5_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q5_k_f32; break;
case HTP_TYPE_Q6_K: grctx.compute_tiled = hvx_dequantize_row_tiled_q6_k_f32; break;
default: return HTP_STATUS_NO_SUPPORT;
}
grctx.tile_size = htp_tensor_get_tile_size(octx->src[0]->type);
grctx.tile_stride = htp_tensor_get_tile_padded_size(octx->src[0]->type);
q_func = get_rows_thread_tiled;
break;
case HTP_GET_ROWS_KERNEL_FLAT:
switch (octx->src[0]->type) {
case HTP_TYPE_F16: grctx.compute_flat = hvx_dequantize_row_flat_f16_f32; break;
case HTP_TYPE_BF16: grctx.compute_flat = hvx_dequantize_row_flat_bf16_f32; break;
case HTP_TYPE_Q1_0: grctx.compute_flat = hvx_dequantize_row_flat_q1_0_f32; break;
case HTP_TYPE_Q2_0: grctx.compute_flat = hvx_dequantize_row_flat_q2_0_f32; break;
case HTP_TYPE_Q8_0: grctx.compute_flat = hvx_dequantize_row_flat_q8_0_f32; break;
case HTP_TYPE_Q4_0: grctx.compute_flat = hvx_dequantize_row_flat_q4_0_f32; break;
case HTP_TYPE_Q4_K: grctx.compute_flat = hvx_dequantize_row_flat_q4_k_f32; break;
case HTP_TYPE_Q5_K: grctx.compute_flat = hvx_dequantize_row_flat_q5_k_f32; break;
case HTP_TYPE_Q6_K: grctx.compute_flat = hvx_dequantize_row_flat_q6_k_f32; break;
default: return HTP_STATUS_NO_SUPPORT;
}
q_func = get_rows_thread_flat;
break;
default:
return HTP_STATUS_NO_SUPPORT;
}
const uint32_t elem_size = (grctx.idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t);
const size_t src1_bytes = (size_t) total_tasks * elem_size;
if (!htp_tensor_is_extended(octx->src[1]) && src1_bytes <= 128 &&
htp_tensor_is_contiguous(octx->src[1], elem_size)) {
grctx.src1_data = (const void *) (uintptr_t) octx->src[1]->data;
grctx.read_index = get_rows_read_index_ddr;
} else {
uint8_t * vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, grctx.vtcm_base, grctx.vtcm_layout.off_src1);
get_rows_dma_src1(octx, vtcm_src1, elem_size);
grctx.src1_data = vtcm_src1;
grctx.read_index = get_rows_read_index_vtcm;
}
FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n",
octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3],
grctx.vtcm_layout.src0_bytes_per_thread * n_threads,
grctx.vtcm_layout.dst_bytes_per_thread * n_threads,
kparams->kernel_type, n_threads);
work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads);
return HTP_STATUS_OK;
}
llama.cpp/ggml/src/ggml-hexagon/htp/get-rows-ops.h
+91
View File
@@ -0,0 +1,91 @@
#ifndef HTP_GET_ROWS_OPS_H
#define HTP_GET_ROWS_OPS_H
#include <stdbool.h>
#include <string.h>
#include "hex-fastdiv.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
enum htp_get_rows_kernel_type {
HTP_GET_ROWS_KERNEL_SAMETYPE = 0,
HTP_GET_ROWS_KERNEL_TILED,
HTP_GET_ROWS_KERNEL_FLAT,
};
struct htp_get_rows_kernel_params {
uint8_t kernel_type;
uint32_t n_threads;
uint32_t total_tasks;
uint32_t vtcm_size;
// Fastdiv helpers
struct fastdiv_values div_ne10;
struct fastdiv_values div_ne10_ne11;
struct fastdiv_values div_ne02;
struct fastdiv_values div_ne03;
};
struct htp_get_rows_vtcm_layout {
size_t total_bytes;
size_t off_src1;
size_t src1_bytes;
size_t off_src0;
size_t off_dst;
size_t src0_bytes_per_thread;
size_t dst_bytes_per_thread;
size_t src0_spad_half_size;
size_t dst_spad_half_size;
};
static inline void htp_get_rows_vtcm_layout_build(
struct htp_get_rows_vtcm_layout * L,
int kernel_type,
int type,
uint32_t ne00,
uint32_t total_tasks,
int idx_type,
uint32_t n_threads) {
memset(L, 0, sizeof(*L));
const size_t elem_size = (idx_type == HTP_TYPE_I64) ? sizeof(int64_t) : sizeof(int32_t);
const size_t src1_raw_bytes = total_tasks * elem_size;
const size_t src1_bytes = ((src1_raw_bytes + 127) & ~127) + 128;
L->src1_bytes = src1_bytes;
const size_t src1_alloc_bytes = (src1_bytes + 255) & ~255;
if (kernel_type != HTP_GET_ROWS_KERNEL_SAMETYPE) {
size_t src0_row_size = 0;
if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {
const size_t tile_stride = htp_tensor_get_tile_padded_size(type);
const uint32_t n_k_tiles = ne00 / htp_tensor_get_tile_k(type);
src0_row_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;
} else {
src0_row_size = htp_tensor_get_row_size(type, ne00);
}
L->src0_spad_half_size = (src0_row_size + 255) & ~255;
L->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255;
L->src0_bytes_per_thread = 2 * L->src0_spad_half_size;
L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
}
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src1, src1_alloc_bytes);
VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
L->total_bytes = off;
}
#if defined(__cplusplus)
static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_get_rows_kernel_params) <= 128, "htp_get_rows_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_GET_ROWS_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/glu-ops.c
+1219
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp/glu-ops.h
+76
View File
@@ -0,0 +1,76 @@
#ifndef HTP_GLU_OPS_H
#define HTP_GLU_OPS_H
#include "hex-common.h"
#include "htp-ops.h"
#include "htp-vtcm.h"
#include <stdint.h>
struct htp_glu_kernel_params {
uint32_t n_threads;
uint32_t vtcm_row_per_thread;
uint32_t vtcm_size;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_glu_kernel_params) <= 128, "htp_glu_kernel_params is too large for kernel_params blob");
#endif
static inline bool htp_op_is_glu(uint32_t opcode) {
switch (opcode) {
case HTP_OP_GLU_SWIGLU:
case HTP_OP_GLU_SWIGLU_OAI:
case HTP_OP_GLU_SWIGLU_CLAMP:
case HTP_OP_GLU_GEGLU:
case HTP_OP_GLU_GEGLU_QUICK:
case HTP_OP_GLU_GEGLU_ERF:
case HTP_OP_GLU_REGLU:
case HTP_OP_GLU_SIGMOID:
return true;
default:
return false;
}
}
struct htp_glu_vtcm_layout {
size_t total_bytes;
size_t off_src0;
size_t off_src1;
size_t off_dst;
size_t row_size_aligned;
size_t src0_bytes_per_thread;
size_t src1_bytes_per_thread;
size_t dst_bytes_per_thread;
uint32_t vtcm_row_per_thread;
uint32_t block;
};
static inline void htp_glu_vtcm_layout_build(
struct htp_glu_vtcm_layout * L,
size_t dst_row_size,
uint32_t n_threads,
uint32_t vtcm_row_per_thread
) {
const size_t row_size_aligned = hex_round_up(dst_row_size, 128);
L->row_size_aligned = row_size_aligned;
L->vtcm_row_per_thread = vtcm_row_per_thread;
L->block = vtcm_row_per_thread >> 1;
L->src0_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;
L->src1_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;
L->dst_bytes_per_thread = row_size_aligned * vtcm_row_per_thread;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
VTCM_LAYOUT_ALLOC(off, off_src1, L->src1_bytes_per_thread * n_threads);
VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
L->total_bytes = off;
}
#endif // HTP_GLU_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/hex-bitmap.h
+24
View File
@@ -0,0 +1,24 @@
#ifndef HEX_BITMAP_H
#define HEX_BITMAP_H
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
static inline void bitmap_set(uint32_t * bitmap, uint32_t idx) {
bitmap[idx / 32] |= (1U << (idx % 32));
}
static inline void bitmap_clear(uint32_t * bitmap, uint32_t idx) {
bitmap[idx / 32] &= ~(1U << (idx % 32));
}
static inline bool bitmap_test(const uint32_t * bitmap, uint32_t idx) {
return (bitmap[idx / 32] & (1U << (idx % 32))) != 0;
}
static inline void bitmap_reset(uint32_t * bitmap, size_t size_in_bits) {
memset(bitmap, 0, ((size_in_bits + 31) / 32) * sizeof(uint32_t));
}
#endif // HEX_BITMAP_H
llama.cpp/ggml/src/ggml-hexagon/htp/hex-common.h
+93
View File
@@ -0,0 +1,93 @@
#ifndef HEX_COMMON_H
#define HEX_COMMON_H
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#ifndef SIZE_MAX
#define SIZE_MAX ((size_t)-1)
#endif
#ifndef MAX
#define MAX(a, b) ((a) > (b) ? (a) : (b))
#endif
#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#endif
#ifndef HEX_L2_LINE_SIZE
#define HEX_L2_LINE_SIZE 128
#endif
static inline uint32_t hex_ceil_pow2(uint32_t x) {
if (x <= 1) { return 1; }
int p = 2;
x--;
while (x >>= 1) { p <<= 1; }
return p;
}
static inline size_t hmx_ceil_div(size_t num, size_t den) {
return (num + den - 1) / den;
}
static inline int32_t hex_is_aligned(const void * addr, uint32_t align) {
return ((size_t) addr & (align - 1)) == 0;
}
static inline size_t hex_align_up(size_t v, size_t align) {
return hmx_ceil_div(v, align) * align;
}
static inline size_t hex_align_down(size_t v, size_t align) {
return (v / align) * align;
}
static inline int32_t hex_is_one_chunk(void * addr, uint32_t n, uint32_t chunk_size) {
uint32_t left_off = (size_t) addr & (chunk_size - 1);
uint32_t right_off = left_off + n;
return right_off <= chunk_size;
}
static inline uint32_t hex_round_up(uint32_t n, uint32_t m) {
return m * ((n + m - 1) / m);
}
static inline size_t hex_smin(size_t a, size_t b) {
return a < b ? a : b;
}
static inline size_t hex_smax(size_t a, size_t b) {
return a > b ? a : b;
}
static inline void hex_swap_ptr(void ** p1, void ** p2) {
void * t = *p1;
*p1 = *p2;
*p2 = t;
}
static inline bool hex_mul_overflow(size_t a, size_t b, size_t *out) {
if (a != 0 && b > SIZE_MAX / a) return true;
*out = a * b;
return false;
}
static inline bool hex_add_overflow(size_t a, size_t b, size_t *out) {
if (a > SIZE_MAX - b) return true;
*out = a + b;
return false;
}
static inline uint32_t hex_gcd_u32(uint32_t a, uint32_t b) {
while (b != 0) {
uint32_t t = b;
b = a % b;
a = t;
}
return a;
}
#endif // HEX_COMMON_H
llama.cpp/ggml/src/ggml-hexagon/htp/hex-dump.h
+86
View File
@@ -0,0 +1,86 @@
#ifndef HEX_DUMP_H
#define HEX_DUMP_H
#include <HAP_farf.h>
static inline void hex_dump_int8_line(char * pref, const int8_t * x, int n) {
char str[1024], *p = str, *p_end = str + sizeof(str);
p += snprintf(p, p_end - p, "%s: ", pref);
for (int i = 0; i < n && p < p_end; i++) {
p += snprintf(p, p_end - p, "%d, ", x[i]);
}
FARF(HIGH, "%s\n", str);
}
static inline void hex_dump_uint8_line(char * pref, const uint8_t * x, uint32_t n) {
char str[1024], *p = str, *p_end = str + sizeof(str);
p += snprintf(p, p_end - p, "%s: ", pref);
for (int i = 0; i < n && p < p_end; i++) {
p += snprintf(p, p_end - p, "%d, ", x[i]);
}
FARF(HIGH, "%s\n", str);
}
static inline void hex_dump_uint32_line(char * pref, const uint32_t * x, uint32_t n) {
char str[1024], *p = str, *p_end = str + sizeof(str);
p += snprintf(p, p_end - p, "%s: ", pref);
for (int i = 0; i < n; i++) {
p += snprintf(p, p_end - p, "%u, ", (unsigned int) x[i]);
}
FARF(HIGH, "%s\n", str);
}
static inline void hex_dump_int32_line(char * pref, const int32_t * x, uint32_t n) {
char str[1024], *p = str, *p_end = str + sizeof(str);
p += snprintf(p, p_end - p, "%s: ", pref);
for (int i = 0; i < n; i++) {
p += snprintf(p, p_end - p, "%d, ", (int) x[i]);
}
FARF(HIGH, "%s\n", str);
}
static inline void hex_dump_f16_line(char * pref, const __fp16 * x, uint32_t n) {
char str[1024], *p = str, *p_end = str + sizeof(str);
p += snprintf(p, p_end - p, "%s: ", pref);
for (int i = 0; i < n; i++) {
p += snprintf(p, p_end - p, "%.6f, ", (float) x[i]);
}
FARF(HIGH, "%s\n", str);
}
static inline void hex_dump_f32_line(char * pref, const float * x, uint32_t n) {
char str[1024], *p = str, *p_end = str + sizeof(str);
p += snprintf(p, p_end - p, "%s: ", pref);
for (int i = 0; i < n; i++) {
p += snprintf(p, p_end - p, "%.6f, ", x[i]);
}
FARF(HIGH, "%s\n", str);
}
static inline void hex_dump_f32(char * pref, const float * x, uint32_t n) {
uint32_t n0 = n / 16;
uint32_t n1 = n % 16;
uint32_t i = 0;
for (; i < n0; i++) {
hex_dump_f32_line(pref, x + (16 * i), 16);
}
if (n1) {
hex_dump_f32_line(pref, x + (16 * i), n1);
}
}
static inline void hex_dump_f16(char * pref, const __fp16 * x, uint32_t n) {
uint32_t n0 = n / 16;
uint32_t n1 = n % 16;
uint32_t i = 0;
for (; i < n0; i++) {
hex_dump_f16_line(pref, x + (16 * i), 16);
}
if (n1) {
hex_dump_f16_line(pref, x + (16 * i), n1);
}
}
#endif /* HEX_DUMP_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hex-fastdiv.h
+37
View File
@@ -0,0 +1,37 @@
#ifndef HEX_FASTDIV_H
#define HEX_FASTDIV_H
// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.
// Precompute mp (m' in the paper) and L such that division
// can be computed using a multiply (high 32b of 64b result)
// and a shift:
//
// n/d = (mulhi(n, mp) + n) >> L;
struct fastdiv_values {
uint32_t mp;
uint32_t l;
};
static inline struct fastdiv_values init_fastdiv_values(uint32_t d) {
struct fastdiv_values result = { 0, 0 };
// compute L = ceil(log2(d));
while (result.l < 32 && ((uint32_t) 1 << result.l) < d) {
++(result.l);
}
result.mp = (uint32_t) (((uint64_t) 1 << 32) * (((uint64_t) 1 << result.l) - d) / d + 1);
return result;
}
static inline uint32_t fastdiv(uint32_t n, const struct fastdiv_values * vals) {
// Compute high 32 bits of n * mp
const uint32_t hi = (uint32_t) (((uint64_t) n * vals->mp) >> 32); // mulhi(n, mp)
// add n, apply bit shift
return (hi + n) >> vals->l;
}
static inline uint32_t fastmodulo(uint32_t n, uint32_t d, const struct fastdiv_values * vals) {
return n - fastdiv(n, vals) * d;
}
#endif /* HEX_FASTDIV_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hex-profile.h
+66
View File
@@ -0,0 +1,66 @@
#ifndef HEX_PROFILE_H
#define HEX_PROFILE_H
#include <stdbool.h>
#include <stdint.h>
#include <qurt.h>
#include "hex-utils.h"
#include "htp-ops.h"
#define HTP_TRACE_EVT_START 0
#define HTP_TRACE_EVT_STOP 1
#ifndef HEX_NUM_PMU_COUNTERS
#define HEX_NUM_PMU_COUNTERS 8
#endif
static inline void hex_get_pmu(uint32_t counters[]) {
#if __HVX_ARCH__ >= 79
asm volatile("%0 = upmucnt0" : "=r"(counters[0]));
asm volatile("%0 = upmucnt1" : "=r"(counters[1]));
asm volatile("%0 = upmucnt2" : "=r"(counters[2]));
asm volatile("%0 = upmucnt3" : "=r"(counters[3]));
asm volatile("%0 = upmucnt4" : "=r"(counters[4]));
asm volatile("%0 = upmucnt5" : "=r"(counters[5]));
asm volatile("%0 = upmucnt6" : "=r"(counters[6]));
asm volatile("%0 = upmucnt7" : "=r"(counters[7]));
#else
counters[0] = qurt_pmu_get(QURT_PMUCNT0);
counters[1] = qurt_pmu_get(QURT_PMUCNT1);
counters[2] = qurt_pmu_get(QURT_PMUCNT2);
counters[3] = qurt_pmu_get(QURT_PMUCNT3);
counters[4] = qurt_pmu_get(QURT_PMUCNT4);
counters[5] = qurt_pmu_get(QURT_PMUCNT5);
counters[6] = qurt_pmu_get(QURT_PMUCNT6);
counters[7] = qurt_pmu_get(QURT_PMUCNT7);
#endif
}
struct htp_thread_trace {
uint32_t count;
uint32_t max_events;
struct htp_trace_desc * events;
};
static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) {
if (!tr->max_events) {
return;
}
uint32_t i = tr->count++;
if (i < tr->max_events) {
tr->events[i].id = id;
tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);
tr->events[i].cycles = (uint32_t) hex_get_cycles();
}
}
static inline void htp_trace_event_start(struct htp_thread_trace * tr, uint16_t id, uint16_t info) {
htp_trace_event(tr, id, info, HTP_TRACE_EVT_START);
}
static inline void htp_trace_event_stop(struct htp_thread_trace * tr, uint16_t id, uint16_t info) {
htp_trace_event(tr, id, info, HTP_TRACE_EVT_STOP);
}
#endif /* HEX_PROFILE_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hex-utils.h
+58
View File
@@ -0,0 +1,58 @@
#ifndef HEX_UTILS_H
#define HEX_UTILS_H
#include <stdbool.h>
#include <stdint.h>
#include <qurt_memory.h>
#include <qurt.h>
#include "hexagon_types.h"
#include "hexagon_protos.h"
#include "hex-fastdiv.h"
#include "hex-dump.h"
#include "hex-common.h"
static inline uint64_t hex_get_cycles() {
uint64_t cycles = 0;
asm volatile(" %0 = c15:14\n" : "=r"(cycles));
return cycles;
}
static inline uint64_t hex_get_pktcnt() {
uint64_t pktcnt;
asm volatile(" %0 = c19:18\n" : "=r"(pktcnt));
return pktcnt;
}
static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride, uint32_t height) {
const uint64_t control = Q6_P_combine_RR(stride, Q6_R_combine_RlRl(width, height));
Q6_l2fetch_AP((void *) p, control);
}
static inline void hex_l2fetch_block(const void * addr, size_t size) {
if (size == 0) return;
const uint32_t width = 16384; // 16KB rows
const uint32_t height = (size + width - 1) / width;
hex_l2fetch(addr, width, width, height);
}
#define HEX_L2_LINE_SIZE 128
#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread)
#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024)
#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024)
static inline void hex_l2flush(void * addr, size_t size) {
if (size == 0) return;
const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1);
const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1);
for (uint32_t i = s; i < e; i += HEX_L2_LINE_SIZE) {
Q6_dccleaninva_A((void *) i);
}
}
static inline void hex_pause() {
asm volatile(" pause(#255)\n");
}
#endif /* HEX_UTILS_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hmx-fa-kernels.h
+705
View File
@@ -0,0 +1,705 @@
#ifndef HMX_FA_KERNELS_H
#define HMX_FA_KERNELS_H
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include "hvx-utils.h"
#include "hmx-utils.h"
#include "hex-fastdiv.h"
// HMX-specific parameters, offsets and inner kernels for Flash Attention
// Scatter offsets for diagonal tile: entry[2i] = i*136, entry[2i+1] = i*136+6
// 136 = 4 * 32 + 8 = byte offset to diagonal in a 32x32 fp16 interleaved tile
static const int16_t d_tile_scatter_offsets[64] __attribute__((aligned(128))) = {
0 * 136, 0 * 136 + 6,
1 * 136, 1 * 136 + 6,
2 * 136, 2 * 136 + 6,
3 * 136, 3 * 136 + 6,
4 * 136, 4 * 136 + 6,
5 * 136, 5 * 136 + 6,
6 * 136, 6 * 136 + 6,
7 * 136, 7 * 136 + 6,
8 * 136, 8 * 136 + 6,
9 * 136, 9 * 136 + 6,
10 * 136, 10 * 136 + 6,
11 * 136, 11 * 136 + 6,
12 * 136, 12 * 136 + 6,
13 * 136, 13 * 136 + 6,
14 * 136, 14 * 136 + 6,
15 * 136, 15 * 136 + 6,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
0, 0,
};
// Inner HMX tile computation kernels
static inline void hmx_fa_qk_dot_tile(
const __fp16 * row_tiles,
const __fp16 * col_tiles,
__fp16 * out_tile,
size_t n_dot_tiles
) {
if (n_dot_tiles == 2) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
HMX_LOAD_MPY_F16("%3", "%4", "%0")
:
: "r"(2047),
"r"(row_tiles + 0 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 0 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 1 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 1 * HMX_FP16_TILE_N_ELMS)
);
} else if (n_dot_tiles == 4) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
HMX_LOAD_MPY_F16("%3", "%4", "%0")
HMX_LOAD_MPY_F16("%5", "%6", "%0")
HMX_LOAD_MPY_F16("%7", "%8", "%0")
:
: "r"(2047),
"r"(row_tiles + 0 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 0 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 1 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 1 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 2 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 2 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 3 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 3 * HMX_FP16_TILE_N_ELMS)
);
} else if (n_dot_tiles == 8) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
HMX_LOAD_MPY_F16("%3", "%4", "%0")
HMX_LOAD_MPY_F16("%5", "%6", "%0")
HMX_LOAD_MPY_F16("%7", "%8", "%0")
HMX_LOAD_MPY_F16("%9", "%10", "%0")
HMX_LOAD_MPY_F16("%11", "%12", "%0")
HMX_LOAD_MPY_F16("%13", "%14", "%0")
HMX_LOAD_MPY_F16("%15", "%16", "%0")
:
: "r"(2047),
"r"(row_tiles + 0 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 0 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 1 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 1 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 2 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 2 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 3 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 3 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 4 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 4 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 5 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 5 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 6 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 6 * HMX_FP16_TILE_N_ELMS),
"r"(row_tiles + 7 * HMX_FP16_TILE_N_ELMS), "r"(col_tiles + 7 * HMX_FP16_TILE_N_ELMS)
);
} else {
for (size_t k = 0; k < n_dot_tiles; ++k) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
:
: "r"(2047), "r"(row_tiles), "r"(col_tiles)
);
row_tiles += HMX_FP16_TILE_N_ELMS;
col_tiles += HMX_FP16_TILE_N_ELMS;
}
}
asm volatile(
HMX_STORE_AFTER_F16("%0", "%1")
:
: "r"(out_tile), "r"(0)
: "memory"
);
}
static inline void hmx_fa_o_update_tile(
const __fp16 * d_diag,
const __fp16 * o_rc,
const __fp16 * p_tile_in,
const __fp16 * v_tile_in,
__fp16 * o_tile_out,
size_t n_col_tiles
) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
:
: "r"(2047), "r"(d_diag), "r"(o_rc)
);
if (n_col_tiles == 2) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
HMX_LOAD_MPY_F16("%3", "%4", "%0")
:
: "r"(2047),
"r"(p_tile_in + 0 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 0 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 1 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 1 * HMX_FP16_TILE_N_ELMS)
);
} else if (n_col_tiles == 4) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
HMX_LOAD_MPY_F16("%3", "%4", "%0")
HMX_LOAD_MPY_F16("%5", "%6", "%0")
HMX_LOAD_MPY_F16("%7", "%8", "%0")
:
: "r"(2047),
"r"(p_tile_in + 0 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 0 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 1 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 1 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 2 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 2 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 3 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 3 * HMX_FP16_TILE_N_ELMS)
);
} else if (n_col_tiles == 8) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
HMX_LOAD_MPY_F16("%3", "%4", "%0")
HMX_LOAD_MPY_F16("%5", "%6", "%0")
HMX_LOAD_MPY_F16("%7", "%8", "%0")
HMX_LOAD_MPY_F16("%9", "%10", "%0")
HMX_LOAD_MPY_F16("%11", "%12", "%0")
HMX_LOAD_MPY_F16("%13", "%14", "%0")
HMX_LOAD_MPY_F16("%15", "%16", "%0")
:
: "r"(2047),
"r"(p_tile_in + 0 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 0 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 1 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 1 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 2 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 2 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 3 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 3 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 4 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 4 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 5 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 5 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 6 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 6 * HMX_FP16_TILE_N_ELMS),
"r"(p_tile_in + 7 * HMX_FP16_TILE_N_ELMS), "r"(v_tile_in + 7 * HMX_FP16_TILE_N_ELMS)
);
} else {
for (size_t k = 0; k < n_col_tiles; ++k) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
:
: "r"(2047), "r"(p_tile_in), "r"(v_tile_in)
);
p_tile_in += HMX_FP16_TILE_N_ELMS;
v_tile_in += HMX_FP16_TILE_N_ELMS;
}
}
asm volatile(
HMX_STORE_AFTER_F16("%0", "%1")
:
: "r"(o_tile_out), "r"(0)
: "memory"
);
}
static inline void hmx_fa_o_norm_tile(
const __fp16 * d_diag,
const __fp16 * o_rc,
__fp16 * o_out
) {
asm volatile(
HMX_LOAD_MPY_F16("%1", "%2", "%0")
:
: "r"(2047), "r"(d_diag), "r"(o_rc)
);
asm volatile(
HMX_STORE_AFTER_F16("%0", "%1")
:
: "r"(o_out), "r"(0)
: "memory"
);
}
static inline void hmx_fa_q_prep_fp32_d2(
__fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm,
size_t start, size_t end, size_t g_rows_end,
size_t DK, size_t G, size_t n_rows_q,
const struct fastdiv_values * div_G, bool q_transposed
) {
for (size_t r = start; r < end; r += 2) {
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
__fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK;
if (r >= g_rows_end) {
((HVX_Vector *) (out_base + 0 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero();
((HVX_Vector *) (out_base + 1 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero();
continue;
}
const size_t q_idx0 = fastdiv(r + 0, div_G);
const size_t h_idx0 = fastmodulo(r + 0, G, div_G);
const size_t q_idx1 = fastdiv(r + 1, div_G);
const size_t h_idx1 = fastmodulo(r + 1, G, div_G);
const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0);
const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1);
const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(float));
const HVX_Vector * pv_in1 = (r + 1 < g_rows_end)
? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(float))
: NULL;
{
HVX_Vector v0 = pv_in0[0];
HVX_Vector v1 = pv_in1 ? pv_in1[0] : Q6_V_vzero();
HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1);
((HVX_Vector *) (out_base + 0 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = v_hf;
}
{
HVX_Vector v0 = pv_in0[1];
HVX_Vector v1 = pv_in1 ? pv_in1[1] : Q6_V_vzero();
HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1);
((HVX_Vector *) (out_base + 1 * HMX_FP16_TILE_N_ELMS))[r1 / 2] = v_hf;
}
}
}
static inline void hmx_fa_q_prep_fp32_d4(
__fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm,
size_t start, size_t end, size_t g_rows_end,
size_t DK, size_t G, size_t n_rows_q,
const struct fastdiv_values * div_G, bool q_transposed
) {
for (size_t r = start; r < end; r += 2) {
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
__fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK;
if (r >= g_rows_end) {
for (uint32_t d = 0; d < 4; ++d) {
((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero();
}
continue;
}
const size_t q_idx0 = fastdiv(r + 0, div_G);
const size_t h_idx0 = fastmodulo(r + 0, G, div_G);
const size_t q_idx1 = fastdiv(r + 1, div_G);
const size_t h_idx1 = fastmodulo(r + 1, G, div_G);
const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0);
const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1);
const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(float));
const HVX_Vector * pv_in1 = (r + 1 < g_rows_end)
? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(float))
: NULL;
for (uint32_t d = 0; d < 4; ++d) {
HVX_Vector v0 = pv_in0[d];
HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero();
HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1);
((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = v_hf;
}
}
}
static inline void hmx_fa_q_prep_fp32(
__fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm,
size_t start, size_t end, size_t g_rows_end,
size_t DK, size_t G, size_t n_rows_q,
const struct fastdiv_values * div_G, uint32_t d_limit, bool q_transposed
) {
for (size_t r = start; r < end; r += 2) {
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
__fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK;
if (r >= g_rows_end) {
for (uint32_t d = 0; d < d_limit; ++d) {
((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero();
}
continue;
}
const size_t q_idx0 = fastdiv(r + 0, div_G);
const size_t h_idx0 = fastmodulo(r + 0, G, div_G);
const size_t q_idx1 = fastdiv(r + 1, div_G);
const size_t h_idx1 = fastmodulo(r + 1, G, div_G);
const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0);
const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1);
const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(float));
const HVX_Vector * pv_in1 = (r + 1 < g_rows_end)
? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(float))
: NULL;
for (uint32_t d = 0; d < d_limit; ++d) {
HVX_Vector v0 = pv_in0[d];
HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero();
HVX_Vector v_hf = hvx_vec_f32_to_f16_shuff(v0, v1);
HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS);
out_tile[r1 / 2] = v_hf;
}
}
}
static inline void hmx_fa_q_prep_fp16_d1(
__fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm,
size_t start, size_t end, size_t g_rows_end,
size_t DK, size_t G, size_t n_rows_q,
const struct fastdiv_values * div_G, bool q_transposed
) {
for (size_t r = start; r < end; r += 2) {
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
__fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK;
if (r >= g_rows_end) {
__fp16 * out_dtile = out_base + 0 * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
*pv_out0 = Q6_V_vzero();
*pv_out1 = Q6_V_vzero();
continue;
}
const size_t q_idx0 = fastdiv(r + 0, div_G);
const size_t h_idx0 = fastmodulo(r + 0, G, div_G);
const size_t q_idx1 = fastdiv(r + 1, div_G);
const size_t h_idx1 = fastmodulo(r + 1, G, div_G);
const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0);
const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1);
const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(__fp16));
const HVX_Vector * pv_in1 = (r + 1 < g_rows_end)
? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(__fp16))
: NULL;
HVX_Vector v0 = pv_in0[0];
HVX_Vector v1 = pv_in1 ? pv_in1[0] : Q6_V_vzero();
HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2);
__fp16 * out_dtile = out_base + 0 * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
*pv_out0 = Q6_V_lo_W(vp);
*pv_out1 = Q6_V_hi_W(vp);
}
}
static inline void hmx_fa_q_prep_fp16_d2(
__fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm,
size_t start, size_t end, size_t g_rows_end,
size_t DK, size_t G, size_t n_rows_q,
const struct fastdiv_values * div_G, bool q_transposed
) {
for (size_t r = start; r < end; r += 2) {
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
__fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK;
if (r >= g_rows_end) {
for (uint32_t d = 0; d < 2; ++d) {
__fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
*pv_out0 = Q6_V_vzero();
*pv_out1 = Q6_V_vzero();
}
continue;
}
const size_t q_idx0 = fastdiv(r + 0, div_G);
const size_t h_idx0 = fastmodulo(r + 0, G, div_G);
const size_t q_idx1 = fastdiv(r + 1, div_G);
const size_t h_idx1 = fastmodulo(r + 1, G, div_G);
const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0);
const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1);
const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(__fp16));
const HVX_Vector * pv_in1 = (r + 1 < g_rows_end)
? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(__fp16))
: NULL;
{
HVX_Vector v0 = pv_in0[0];
HVX_Vector v1 = pv_in1 ? pv_in1[0] : Q6_V_vzero();
HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2);
__fp16 * out_dtile = out_base + 0 * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
*pv_out0 = Q6_V_lo_W(vp);
*pv_out1 = Q6_V_hi_W(vp);
}
{
HVX_Vector v0 = pv_in0[1];
HVX_Vector v1 = pv_in1 ? pv_in1[1] : Q6_V_vzero();
HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2);
__fp16 * out_dtile = out_base + 1 * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
*pv_out0 = Q6_V_lo_W(vp);
*pv_out1 = Q6_V_hi_W(vp);
}
}
}
static inline void hmx_fa_q_prep_fp16(
__fp16 * vtcm_q_tiles, const uint8_t * temp_q_vtcm,
size_t start, size_t end, size_t g_rows_end,
size_t DK, size_t G, size_t n_rows_q,
const struct fastdiv_values * div_G, uint32_t d_limit, bool q_transposed
) {
for (size_t r = start; r < end; r += 2) {
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
__fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * DK;
if (r >= g_rows_end) {
for (uint32_t d = 0; d < d_limit; ++d) {
__fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
*pv_out0 = Q6_V_vzero();
*pv_out1 = Q6_V_vzero();
}
continue;
}
const size_t q_idx0 = fastdiv(r + 0, div_G);
const size_t h_idx0 = fastmodulo(r + 0, G, div_G);
const size_t q_idx1 = fastdiv(r + 1, div_G);
const size_t h_idx1 = fastmodulo(r + 1, G, div_G);
const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0);
const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1);
const HVX_Vector * pv_in0 = (const HVX_Vector *) (temp_q_vtcm + offset0 * DK * sizeof(__fp16));
const HVX_Vector * pv_in1 = (r + 1 < g_rows_end)
? (const HVX_Vector *) (temp_q_vtcm + offset1 * DK * sizeof(__fp16))
: NULL;
for (uint32_t d = 0; d < d_limit; ++d) {
HVX_Vector v0 = pv_in0[d];
HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero();
HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2);
__fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
*pv_out0 = Q6_V_lo_W(vp);
*pv_out1 = Q6_V_hi_W(vp);
}
}
}
// Head-dim-padded Q-prep (f32). Used when DK is not a multiple of 64.
static inline void hmx_fa_q_prep_fp32_pad(__fp16 * vtcm_q_tiles,
const uint8_t * temp_q_vtcm,
size_t start,
size_t end,
size_t g_rows_end,
size_t dk_in,
size_t dk_out,
size_t G,
size_t n_rows_q,
const struct fastdiv_values * div_G,
bool q_transposed) {
const uint32_t n_out_tiles = (uint32_t) (dk_out / 32);
for (size_t r = start; r < end; r += 2) {
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
__fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * dk_out;
if (r >= g_rows_end) {
for (uint32_t d = 0; d < n_out_tiles; ++d) {
((HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS))[r1 / 2] = Q6_V_vzero();
}
continue;
}
const size_t q_idx0 = fastdiv(r + 0, div_G);
const size_t h_idx0 = fastmodulo(r + 0, G, div_G);
const size_t q_idx1 = fastdiv(r + 1, div_G);
const size_t h_idx1 = fastmodulo(r + 1, G, div_G);
const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0);
const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1);
const HVX_UVector * pv_in0 = (const HVX_UVector *) (temp_q_vtcm + offset0 * dk_in * sizeof(float));
const HVX_UVector * pv_in1 = (r + 1 < g_rows_end) ? (const HVX_UVector *) (temp_q_vtcm + offset1 * dk_in * sizeof(float)) : NULL;
for (uint32_t d = 0; d < n_out_tiles; ++d) {
HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS);
const size_t base_lane = (size_t) d * 32;
const size_t real_lanes = (base_lane < dk_in) ? hex_smin(32, dk_in - base_lane) : 0;
if (real_lanes == 0) {
out_tile[r1 / 2] = Q6_V_vzero();
continue;
}
HVX_Vector v0 = pv_in0[d];
HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero();
if (real_lanes < 32) {
// Straddle tile: keep the first real_lanes floats, zero the padded tail so
// the packed f16 lanes beyond DK are zero.
const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(float)));
v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero());
v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero());
}
out_tile[r1 / 2] = hvx_vec_f32_to_f16_shuff(v0, v1);
}
}
}
// Head-dim-padded Q-prep (f16). Used when DK is not a multiple of 64.
static inline void hmx_fa_q_prep_fp16_pad(__fp16 * vtcm_q_tiles,
const uint8_t * temp_q_vtcm,
size_t start,
size_t end,
size_t g_rows_end,
size_t dk_in,
size_t dk_out,
size_t G,
size_t n_rows_q,
const struct fastdiv_values * div_G,
bool q_transposed) {
const uint32_t n_out_pairs = (uint32_t) (dk_out / 64);
for (size_t r = start; r < end; r += 2) {
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
__fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * dk_out;
if (r >= g_rows_end) {
for (uint32_t d = 0; d < n_out_pairs; ++d) {
__fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
*pv_out0 = Q6_V_vzero();
*pv_out1 = Q6_V_vzero();
}
continue;
}
const size_t q_idx0 = fastdiv(r + 0, div_G);
const size_t h_idx0 = fastmodulo(r + 0, G, div_G);
const size_t q_idx1 = fastdiv(r + 1, div_G);
const size_t h_idx1 = fastmodulo(r + 1, G, div_G);
const size_t offset0 = q_transposed ? (h_idx0 * n_rows_q + q_idx0) : (q_idx0 * G + h_idx0);
const size_t offset1 = q_transposed ? (h_idx1 * n_rows_q + q_idx1) : (q_idx1 * G + h_idx1);
const HVX_UVector * pv_in0 = (const HVX_UVector *) (temp_q_vtcm + offset0 * dk_in * sizeof(__fp16));
const HVX_UVector * pv_in1 = (r + 1 < g_rows_end) ? (const HVX_UVector *) (temp_q_vtcm + offset1 * dk_in * sizeof(__fp16)) : NULL;
for (uint32_t d = 0; d < n_out_pairs; ++d) {
__fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
const size_t base_lane = (size_t) d * 64;
const size_t real_lanes = (base_lane < dk_in) ? hex_smin(64, dk_in - base_lane) : 0;
if (real_lanes == 0) {
*pv_out0 = Q6_V_vzero();
*pv_out1 = Q6_V_vzero();
continue;
}
HVX_Vector v0 = pv_in0[d];
HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero();
if (real_lanes < 64) {
const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(__fp16)));
v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero());
v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero());
}
HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2);
*pv_out0 = Q6_V_lo_W(vp);
*pv_out1 = Q6_V_hi_W(vp);
}
}
}
static inline void hmx_fa_q_prep_fallback(
__fp16 * vtcm_q_tiles, uintptr_t q_data,
size_t q_nb1, size_t q_nb2, size_t q_nb3,
uint32_t q_start, uint32_t kv_head, uint32_t ib3,
size_t start, size_t end, size_t n_rows_g,
size_t G, size_t dk_in, size_t dk_out, bool is_q_fp32,
const struct fastdiv_values * div_G
) {
for (size_t r = start; r < end; r += 2) {
const size_t q_idx0 = fastdiv(r + 0, div_G);
const size_t h_idx0 = fastmodulo(r + 0, G, div_G);
const size_t q_idx1 = fastdiv(r + 1, div_G);
const size_t h_idx1 = fastmodulo(r + 1, G, div_G);
const uint8_t * q_ptr0 = (r + 0 < n_rows_g) ? ((const uint8_t *) q_data + (q_start + q_idx0) * q_nb1 +
(kv_head * G + h_idx0) * q_nb2 + ib3 * q_nb3) :
NULL;
const uint8_t * q_ptr1 = (r + 1 < n_rows_g) ? ((const uint8_t *) q_data + (q_start + q_idx1) * q_nb1 +
(kv_head * G + h_idx1) * q_nb2 + ib3 * q_nb3) :
NULL;
size_t r0 = r / HMX_FP16_TILE_N_ROWS;
size_t r1 = r % HMX_FP16_TILE_N_ROWS;
__fp16 * out_base = vtcm_q_tiles + r0 * HMX_FP16_TILE_N_ROWS * dk_out;
if (is_q_fp32) {
const HVX_UVector * pv_in0 = q_ptr0 ? (const HVX_UVector *) q_ptr0 : NULL;
const HVX_UVector * pv_in1 = q_ptr1 ? (const HVX_UVector *) q_ptr1 : NULL;
for (uint32_t d = 0; d < dk_out / 32; ++d) {
HVX_Vector * out_tile = (HVX_Vector *) (out_base + d * HMX_FP16_TILE_N_ELMS);
const size_t base_lane = (size_t) d * 32;
const size_t real_lanes = (base_lane < dk_in) ? hex_smin(32, dk_in - base_lane) : 0;
if (real_lanes == 0) {
out_tile[r1 / 2] = Q6_V_vzero();
continue;
}
HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero();
HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero();
if (real_lanes < 32) {
const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(float)));
v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero());
v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero());
}
out_tile[r1 / 2] = hvx_vec_f32_to_f16_shuff(v0, v1);
}
} else {
const HVX_UVector * pv_in0 = q_ptr0 ? (const HVX_UVector *) q_ptr0 : NULL;
const HVX_UVector * pv_in1 = q_ptr1 ? (const HVX_UVector *) q_ptr1 : NULL;
for (uint32_t d = 0; d < dk_out / 64; ++d) {
__fp16 * out_dtile = out_base + d * HMX_FP16_TILE_N_ELMS * 2;
HVX_Vector * pv_out0 = ((HVX_Vector *) out_dtile) + r1 / 2;
HVX_Vector * pv_out1 = pv_out0 + 16;
const size_t base_lane = (size_t) d * 64;
const size_t real_lanes = (base_lane < dk_in) ? hex_smin(64, dk_in - base_lane) : 0;
if (real_lanes == 0) {
*pv_out0 = Q6_V_vzero();
*pv_out1 = Q6_V_vzero();
continue;
}
HVX_Vector v0 = pv_in0 ? pv_in0[d] : Q6_V_vzero();
HVX_Vector v1 = pv_in1 ? pv_in1[d] : Q6_V_vzero();
if (real_lanes < 64) {
const HVX_VectorPred keep = Q6_Q_vsetq_R((uint32_t) (real_lanes * sizeof(__fp16)));
v0 = Q6_V_vmux_QVV(keep, v0, Q6_V_vzero());
v1 = Q6_V_vmux_QVV(keep, v1, Q6_V_vzero());
}
HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2);
*pv_out0 = Q6_V_lo_W(vp);
*pv_out1 = Q6_V_hi_W(vp);
}
}
}
}
#endif /* HMX_FA_KERNELS_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
+1708
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp/hmx-queue.c
+170
View File
@@ -0,0 +1,170 @@
#pragma clang diagnostic ignored "-Wunused-function"
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <qurt_thread.h>
#include <qurt_futex.h>
#include <qurt_hvx.h>
#include <HAP_compute_res.h>
#include "hmx-queue.h"
#define QURT_LOWEST_PRIO (254)
static inline void hmx_lock(hmx_queue_t q)
{
if (!q->hmx_locked) {
HAP_compute_res_hmx_lock(q->hap_rctx);
q->hmx_locked = true;
}
}
static inline void hmx_unlock(hmx_queue_t q)
{
if (q->hmx_locked) {
HAP_compute_res_hmx_unlock(q->hap_rctx);
q->hmx_locked = false;
}
}
static inline void hmx_queue_process(hmx_queue_t q, bool* killed) {
unsigned int ir = atomic_load(&q->idx_read);
while (ir != atomic_load(&q->idx_write)) {
struct hmx_queue_desc *d = &q->desc[ir];
if (!d->done) {
FARF(HIGH, "hmx-queue-process: ir %u func %p data %p", ir, d->func, d->data);
uintptr_t sig = (uintptr_t) d->func;
switch (sig) {
case HMX_QUEUE_NOOP: /* noop */; break;
case HMX_QUEUE_KILL: *killed = true; break;
case HMX_QUEUE_SUSPEND: hmx_unlock(q); break;
case HMX_QUEUE_WAKEUP: hmx_lock(q); break;
default:
hmx_lock(q);
htp_trace_event_start(q->trace, HTP_TRACE_EVT_HMX_COMP, ir);
d->func(d->data);
htp_trace_event_stop(q->trace, HTP_TRACE_EVT_HMX_COMP, ir);
break;
}
atomic_fetch_add(&d->done, 1);
}
ir = (ir + 1) & q->idx_mask;
atomic_store(&q->idx_read, ir);
}
}
static void hmx_queue_thread(void * arg) {
hmx_queue_t q = (hmx_queue_t) arg;
FARF(HIGH, "hmx-queue-thread: started");
bool killed = false;
unsigned int poll_cnt = HMX_QUEUE_POLL_COUNT;
unsigned int prev_seqn = 0;
while (!killed) {
unsigned int seqn = atomic_load(&q->seqn);
if (seqn == prev_seqn) {
// drop HVX context while spinning
if (poll_cnt > 1 && poll_cnt == HMX_QUEUE_POLL_COUNT) {
qurt_hvx_unlock();
}
if (--poll_cnt) { hex_pause(); continue; }
FARF(HIGH, "hmx-queue-thread: sleeping");
qurt_futex_wait(&q->seqn, prev_seqn);
poll_cnt = HMX_QUEUE_POLL_COUNT;
continue;
}
prev_seqn = seqn;
poll_cnt = HMX_QUEUE_POLL_COUNT;
FARF(HIGH, "hmx-queue-thread: new work");
hmx_queue_process(q, &killed);
}
FARF(HIGH, "hmx-queue-thread: stopped");
}
size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) {
capacity = hex_ceil_pow2(capacity);
size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE);
size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE);
size_t size_stack = stack_size;
return size_q + size_desc + size_stack;
}
size_t hmx_queue_alignof(void) {
return HEX_L2_LINE_SIZE;
}
hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) {
capacity = hex_ceil_pow2(capacity);
size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE);
size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE);
uint8_t * block = (uint8_t *) ptr;
hmx_queue_t q = (hmx_queue_t) block; block += size_q;
memset(q, 0, sizeof(struct hmx_queue_s));
q->capacity = capacity;
q->idx_mask = capacity - 1;
q->hap_rctx = hap_rctx;
q->external_mem = true;
q->desc = (struct hmx_queue_desc *) block; block += size_desc;
memset(q->desc, 0, capacity * sizeof(struct hmx_queue_desc));
q->stack = block;
memset(q->stack, 0, stack_size);
q->trace = trace;
// Match caller thread priority (same pattern as worker-pool.c).
int prio = qurt_thread_get_priority(qurt_thread_get_id());
if (prio < 1) {
prio = 1;
}
if (prio > QURT_LOWEST_PRIO) {
prio = QURT_LOWEST_PRIO;
}
qurt_thread_attr_t attr;
qurt_thread_attr_init(&attr);
qurt_thread_attr_set_stack_addr(&attr, q->stack);
qurt_thread_attr_set_stack_size(&attr, stack_size);
qurt_thread_attr_set_priority(&attr, prio);
qurt_thread_attr_set_name(&attr, "hmx-queue");
int err = qurt_thread_create(&q->thread, &attr, hmx_queue_thread, q);
if (err) {
FARF(ERROR, "hmx-worker: thread create failed (%d)", err);
return NULL;
}
FARF(HIGH, "hmx-queue: capacity %u\n", capacity);
return q;
}
void hmx_queue_free(hmx_queue_t q) {
if (!q) {
return;
}
// Tell the worker to exit.
hmx_queue_flush(q);
hmx_queue_signal(q, HMX_QUEUE_KILL);
hmx_queue_flush(q);
int status;
qurt_thread_join(q->thread, &status);
}
llama.cpp/ggml/src/ggml-hexagon/htp/hmx-queue.h
+160
View File
@@ -0,0 +1,160 @@
#ifndef HMX_QUEUE_H
#define HMX_QUEUE_H
#include <stdbool.h>
#include <stdint.h>
#include <stdatomic.h>
#include <hexagon_types.h>
#include <qurt_thread.h>
#include <qurt_futex.h>
#include <HAP_farf.h>
#include "hex-utils.h"
#include "hex-profile.h"
#ifdef __cplusplus
extern "C" {
#endif
#if __HVX_ARCH__ > 79
#define HMX_QUEUE_POLL_COUNT 2000
#else
#define HMX_QUEUE_POLL_COUNT 1
#endif
typedef void (*hmx_queue_func)(void *);
// Dummy funcs used as signals
enum hmx_queue_signal {
HMX_QUEUE_NOOP = 0, // aka NULL
HMX_QUEUE_WAKEUP,
HMX_QUEUE_SUSPEND,
HMX_QUEUE_KILL
};
struct hmx_queue_desc {
hmx_queue_func func;
void * data;
atomic_uint done;
};
struct hmx_queue_s {
struct hmx_queue_desc * desc;
atomic_uint idx_write; // updated by producer (push)
atomic_uint idx_read; // updated by consumer (process)
unsigned int idx_pop; // updated by producer (pop)
uint32_t idx_mask;
uint32_t capacity;
atomic_uint seqn; // incremented for all pushes, used with futex
qurt_thread_t thread;
void * stack;
uint32_t hap_rctx;
bool hmx_locked;
struct htp_thread_trace * trace;
bool external_mem; // memory owned externally
};
typedef struct hmx_queue_s * hmx_queue_t;
size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size);
size_t hmx_queue_alignof(void);
hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace);
void hmx_queue_free(hmx_queue_t q);
static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) {
struct hmx_queue_desc d = { func, data };
return d;
}
static inline bool hmx_queue_push(hmx_queue_t q, struct hmx_queue_desc d) {
unsigned int ir = atomic_load(&q->idx_read);
unsigned int iw = atomic_load(&q->idx_write);
if (((iw + 1) & q->idx_mask) == ir) {
FARF(HIGH, "hmx-queue-push: queue is full\n");
return false;
}
atomic_store(&d.done, 0);
FARF(HIGH, "hmx-queue-push: iw %u func %p data %p\n", iw, d.func, d.data);
q->desc[iw] = d;
atomic_store(&q->idx_write, (iw + 1) & q->idx_mask);
// wake up our thread
atomic_fetch_add(&q->seqn, 1);
qurt_futex_wake(&q->seqn, 1);
return true;
}
static inline bool hmx_queue_signal(hmx_queue_t q, enum hmx_queue_signal sig) {
return hmx_queue_push(q, hmx_queue_make_desc((hmx_queue_func) sig, NULL));
}
static inline bool hmx_queue_empty(hmx_queue_t q) {
return q->idx_pop == atomic_load(&q->idx_write);
}
static inline uint32_t hmx_queue_depth(hmx_queue_t q) {
return (atomic_load(&q->idx_write) - atomic_load(&q->idx_read)) & q->idx_mask;
}
static inline uint32_t hmx_queue_capacity(hmx_queue_t q) {
return q->capacity;
}
static inline struct hmx_queue_desc hmx_queue_pop_one(hmx_queue_t q) {
unsigned int ip = q->idx_pop;
unsigned int iw = atomic_load(&q->idx_write);
struct hmx_queue_desc rd = { NULL, NULL };
if (ip == iw) {
return rd;
}
// Wait for desc to complete
struct hmx_queue_desc * d = &q->desc[ip];
while (!atomic_load(&d->done)) {
FARF(HIGH, "hmx-queue-pop: waiting for HMX queue : %u\n", ip);
hex_pause();
}
rd = *d;
q->idx_pop = (ip + 1) & q->idx_mask;
FARF(HIGH, "hmx-queue-pop: ip %u func %p data %p\n", ip, rd.func, rd.data);
return rd;
}
static inline struct hmx_queue_desc hmx_queue_pop(hmx_queue_t q) {
while (1) {
struct hmx_queue_desc d = hmx_queue_pop_one(q);
uint32_t sig = (uint32_t) d.func;
if (sig && sig <= HMX_QUEUE_KILL)
continue;
return d;
}
}
static inline void hmx_queue_flush(hmx_queue_t q) {
while (hmx_queue_pop_one(q).func != NULL) ;
}
static inline void hmx_queue_wakeup(hmx_queue_t q) {
hmx_queue_signal(q, HMX_QUEUE_WAKEUP);
}
static inline void hmx_queue_suspend(hmx_queue_t q) {
hmx_queue_signal(q, HMX_QUEUE_SUSPEND);
}
#ifdef __cplusplus
} // extern "C"
#endif
#endif /* HMX_QUEUE_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hmx-utils.h
+230
View File
@@ -0,0 +1,230 @@
// HMX tile-level inline helpers (FP16 32x32 tile operations).
// Ported from htp-ops-lib/include/dsp/hmx_utils.h. (https://github.com/haozixu/htp-ops-lib)
#ifndef HMX_UTILS_H
#define HMX_UTILS_H
#include "hvx-base.h"
#include <assert.h>
#include <hexagon_types.h>
#include <stddef.h>
#define HMX_FP16_TILE_N_ROWS 32
#define HMX_FP16_TILE_N_COLS 32
#define HMX_FP16_TILE_N_ELMS 1024
#define HMX_FP16_TILE_SIZE 2048
// Initialise aligned 256-byte area with scale vector + zero padding.
static inline void hmx_init_column_scales(void *out_scales, HVX_Vector v_scale) {
volatile HVX_Vector *pv = (HVX_Vector *) out_scales;
pv[0] = v_scale;
pv[1] = Q6_V_vzero();
}
// --- Shared scatter offsets and interleave helper ---
// vscatter offsets for fused dequant+transpose: write K-values directly to [K][N] tile.
// word[i] = i*128 maps K-row-pair i to byte offset i*128.
// Column offset (n*4) is added at runtime. Entries 0..15 cover one tile (region 2047);
// entries 16..31 cover the next adjacent tile (region 4095) - pick region size at the
// call site to scatter into one tile (masked) or two contiguous tiles (unmasked).
static const int32_t hmx_transpose_scatter_offsets[32] __attribute__((aligned(VLEN))) = {
0 * 128, 1 * 128, 2 * 128, 3 * 128, 4 * 128, 5 * 128, 6 * 128, 7 * 128, 8 * 128, 9 * 128, 10 * 128,
11 * 128, 12 * 128, 13 * 128, 14 * 128, 15 * 128, 16 * 128, 17 * 128, 18 * 128, 19 * 128, 20 * 128, 21 * 128,
22 * 128, 23 * 128, 24 * 128, 25 * 128, 26 * 128, 27 * 128, 28 * 128, 29 * 128, 30 * 128, 31 * 128,
};
// Scatter row-major FP16 data (in VTCM scratch) into transposed [K][N] tiles.
// vtcm_src: [n_cols][src_stride] row-major fp16 (only first k elements per row are used)
// vtcm_dst: [n_col_tiles][n_k_tiles][HMX_FP16_TILE_N_ELMS] tile-major interleaved fp16
// Processes rows [start_row, end_row) for multi-thread slicing.
// Full range: start_row=0, end_row=n_cols.
static inline void hmx_interleave_rows_to_tiles(__fp16 * restrict vtcm_dst,
const __fp16 * restrict vtcm_src,
uint32_t n_cols,
uint32_t k,
size_t src_stride,
uint32_t start_row,
uint32_t end_row) {
assert(k % HMX_FP16_TILE_N_COLS == 0);
const uint32_t n_k_tiles = k / HMX_FP16_TILE_N_COLS;
const HVX_Vector v_scat_base = hvx_vmem(hmx_transpose_scatter_offsets);
const HVX_Vector v_scat_step = Q6_V_vsplat_R(4);
const HVX_VectorPred q_mask64 = Q6_Q_vsetq_R(64);
// Each hvx_vmemu load brings 64 fp16 = 128 bytes covering 2 adjacent K-tiles.
// When n_k_tiles is even, scatter into 2 K-tiles per call (region 4095, no mask)
// using the upper half of hmx_transpose_scatter_offsets. Tail one K-tile (when
// n_k_tiles is odd) falls back to single-tile masked scatter.
const bool pair_scatter = (n_k_tiles & 1) == 0;
const size_t pair_region = (size_t) (2 * HMX_FP16_TILE_SIZE - 1);
const size_t single_region = (size_t) (HMX_FP16_TILE_SIZE - 1);
__builtin_assume(k > 0);
__builtin_assume(end_row > start_row);
if (pair_scatter) {
// Step c by 64 fp16 (two K-tiles per scatter), advance dst by 2 tiles per iter.
const uint32_t c_step = 2 * HMX_FP16_TILE_N_COLS;
const size_t c_byte_step = (size_t) c_step * sizeof(__fp16);
const size_t dst_step = 2 * (size_t) HMX_FP16_TILE_N_ELMS;
const uint32_t n_c_iters = k / c_step;
for (uint32_t r = start_row; r < end_row; r += 2) {
const uint32_t ct = r / HMX_FP16_TILE_N_ROWS;
const uint32_t local_r = r % HMX_FP16_TILE_N_ROWS;
const bool row0_valid = r < n_cols;
const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_cols;
const HVX_Vector v_off0 = Q6_Vw_vadd_VwVw(v_scat_base, Q6_V_vsplat_R(local_r * 4));
const HVX_Vector v_off1 = Q6_Vw_vadd_VwVw(v_off0, v_scat_step);
__fp16 * tile_base = vtcm_dst + (size_t) ct * n_k_tiles * HMX_FP16_TILE_N_ELMS;
const uint8_t * p0 = row0_valid ? (const uint8_t *) (vtcm_src + r * src_stride) : NULL;
const uint8_t * p1 = next_row_valid ? (const uint8_t *) (vtcm_src + (r + 1) * src_stride) : NULL;
assert(!p0 || hex_is_aligned(p0, 128));
assert(!p1 || hex_is_aligned(p1, 128));
assert(c_byte_step % 128 == 0);
if (p0 && p1) {
for (uint32_t i = 0; i < n_c_iters; ++i) {
HVX_Vector v0 = hvx_vmem(p0); p0 += c_byte_step;
HVX_Vector v1 = hvx_vmem(p1); p1 += c_byte_step;
Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off0, v0);
Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off1, v1);
tile_base += dst_step;
}
} else {
const HVX_Vector vzero = Q6_V_vzero();
for (uint32_t i = 0; i < n_c_iters; ++i) {
HVX_Vector v0 = p0 ? hvx_vmem(p0) : vzero;
if (p0) p0 += c_byte_step;
HVX_Vector v1 = p1 ? hvx_vmem(p1) : vzero;
if (p1) p1 += c_byte_step;
Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off0, v0);
Q6_vscatter_RMVwV((size_t) tile_base, pair_region, v_off1, v1);
tile_base += dst_step;
}
}
}
} else {
// Fallback: scatter one K-tile per call (region 2047, masked).
const uint32_t c_step = HMX_FP16_TILE_N_COLS;
const size_t c_byte_step = (size_t) c_step * sizeof(__fp16);
const size_t dst_step = (size_t) HMX_FP16_TILE_N_ELMS;
const uint32_t n_c_iters = k / c_step;
for (uint32_t r = start_row; r < end_row; r += 2) {
const uint32_t ct = r / HMX_FP16_TILE_N_ROWS;
const uint32_t local_r = r % HMX_FP16_TILE_N_ROWS;
const bool row0_valid = r < n_cols;
const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_cols;
const HVX_Vector v_off0 = Q6_Vw_vadd_VwVw(v_scat_base, Q6_V_vsplat_R(local_r * 4));
const HVX_Vector v_off1 = Q6_Vw_vadd_VwVw(v_off0, v_scat_step);
__fp16 * tile_base = vtcm_dst + (size_t) ct * n_k_tiles * HMX_FP16_TILE_N_ELMS;
const uint8_t * p0 = row0_valid ? (const uint8_t *) (vtcm_src + r * src_stride) : NULL;
const uint8_t * p1 = next_row_valid ? (const uint8_t *) (vtcm_src + (r + 1) * src_stride) : NULL;
if (p0 && p1) {
for (uint32_t i = 0; i < n_c_iters; ++i) {
HVX_Vector v0 = hvx_vmemu(p0); p0 += c_byte_step;
HVX_Vector v1 = hvx_vmemu(p1); p1 += c_byte_step;
Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off0, v0);
Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off1, v1);
tile_base += dst_step;
}
} else {
const HVX_Vector vzero = Q6_V_vzero();
for (uint32_t i = 0; i < n_c_iters; ++i) {
HVX_Vector v0 = p0 ? hvx_vmemu(p0) : vzero;
if (p0) p0 += c_byte_step;
HVX_Vector v1 = p1 ? hvx_vmemu(p1) : vzero;
if (p1) p1 += c_byte_step;
Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off0, v0);
Q6_vscatter_QRMVwV(q_mask64, (size_t) tile_base, single_region, v_off1, v1);
tile_base += dst_step;
}
}
}
}
}
// Interleave row-major FP16 data into column-major tile format.
// Input: [n_rows, head_dim] row-major. Output: tile[dim_tile][row_tile].
// Processes rows [start_row, end_row) for multi-thread slicing.
// Full range: start_row=0, end_row=n_rows.
static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out,
const __fp16 * restrict src,
uint32_t n_rows,
uint32_t head_dim,
size_t src_stride,
uint32_t n_row_tiles,
uint32_t start_row,
uint32_t end_row) {
__builtin_assume(head_dim > 0);
const size_t tile_stride_elms = (size_t) n_row_tiles * HMX_FP16_TILE_N_ELMS;
for (uint32_t r = start_row; r < end_row; r += 2) {
const bool next_row_valid = (r + 1) < end_row && (r + 1) < n_rows;
const HVX_Vector * pv_in0 = (const HVX_Vector *) (src + r * src_stride);
const HVX_Vector * pv_in1 = next_row_valid ? (const HVX_Vector *) (src + (r + 1) * src_stride) : NULL;
// Row-pair invariants hoisted out of the c loop.
const uint32_t r0 = r / HMX_FP16_TILE_N_ROWS;
const uint32_t r1_half = (r % HMX_FP16_TILE_N_ROWS) / 2;
// tb0 starts at tile (c0=0, r0); tb1 at the adjacent dim-tile (c0=1, r0).
// Each c step (+= 64) advances both by 2 dim-tiles worth of fp16.
__fp16 * tb0 = tiles_out + (size_t) r0 * HMX_FP16_TILE_N_ELMS;
__fp16 * tb1 = tb0 + tile_stride_elms;
const size_t tb_step = 2 * tile_stride_elms;
if (pv_in1) {
for (uint32_t c = 0; c < head_dim; c += 64) {
HVX_Vector v0 = *pv_in0++;
HVX_Vector v1 = *pv_in1++;
HVX_VectorPair vp = Q6_W_vshuff_VVR(v1, v0, -2);
((HVX_Vector *) tb0)[r1_half] = Q6_V_lo_W(vp);
((HVX_Vector *) tb1)[r1_half] = Q6_V_hi_W(vp);
tb0 += tb_step;
tb1 += tb_step;
}
} else {
const HVX_Vector vzero = Q6_V_vzero();
for (uint32_t c = 0; c < head_dim; c += 64) {
HVX_Vector v0 = *pv_in0++;
HVX_VectorPair vp = Q6_W_vshuff_VVR(vzero, v0, -2);
((HVX_Vector *) tb0)[r1_half] = Q6_V_lo_W(vp);
((HVX_Vector *) tb1)[r1_half] = Q6_V_hi_W(vp);
tb0 += tb_step;
tb1 += tb_step;
}
}
}
}
// --- HMX inline asm macros for load-store packetization ---
#define HMX_LOAD_MPY_F16(act, wt, range) \
"{\n" \
" activation.hf = mxmem(" act ", " range ")\n" \
" weight.hf = mxmem(" wt ", " range ")\n" \
"}\n"
#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \
"{\n" \
" activation.hf = mxmem(" act ", " range "):deep\n" \
" weight.hf = mxmem(" wt ", " range ")\n" \
"}\n"
#define HMX_STORE_AFTER_F16(out, scale_reg) \
"mxmem(" out ", " scale_reg "):after.hf = acc\n"
#define HMX_SET_BIAS(scales) \
"bias = mxmem2(" scales ")\n"
#define HMX_CLRACC_F16() \
"mxclracc.hf\n"
#endif // HMX_UTILS_H
llama.cpp/ggml/src/ggml-hexagon/htp/htp-ctx.h
+190
View File
@@ -0,0 +1,190 @@
#ifndef HTP_CTX_H
#define HTP_CTX_H
#include "dma-queue.h"
#include "hmx-queue.h"
#include "htp-dcache.h"
#include "htp-ops.h"
#include "hex-profile.h"
#include "work-queue.h"
#include "hex-fastdiv.h"
#include <assert.h>
#include <dspqueue.h>
#include <stdatomic.h>
#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
#ifndef HTP_MAX_NTHREADS
#define HTP_MAX_NTHREADS 10
#endif
#define HTP_MAX_MMAPS 64
// Memory mapping
struct htp_mmap {
uint64_t size;
uint64_t base;
uint32_t fd;
uint32_t flags;
};
// Scratchpad state
struct htp_spad {
const struct htp_tensor * src; // original src of the data (for reuse)
uint8_t * data; // pointer to an area in vtcm
uint32_t stride; // stride used inside this spad
uint32_t size; // total size
uint32_t size_per_thread; // size per thread
};
struct htp_mdev_group {
uint16_t idx;
uint16_t count;
struct fastdiv_values count_div;
uint8_t * fence_base;
uint32_t fence_seq;
};
struct htp_context;
// Context while processing an Op
// TODO: fold this into the main context
struct htp_ops_context {
struct htp_context * ctx;
enum htp_op_code op; // FIXME: rename to opcode
int32_t op_params[HTP_OP_MAX_PARAMS];
int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS];
const struct htp_tensor * src[HTP_OP_MAX_INPUTS];
union {
const struct htp_tensor * dst;
const struct htp_tensor * dsts[HTP_OP_MAX_OUTPUTS];
};
// TODO convert these to an array
struct htp_spad src0_spad;
struct htp_spad src1_spad;
struct htp_spad dst_spad;
uint32_t flags;
uint32_t n_threads;
struct fastdiv_values n_threads_div;
int status;
// Must stay last: not cleared by htp_ops_context_reset(), ops init the caches they use
union {
dma_cache_dm dma_cache_dm[HTP_MAX_NTHREADS];
dma_cache_fa dma_cache_fa[HTP_MAX_NTHREADS];
} __attribute__((aligned(128)));
};
// Main context for htp DSP backend
struct htp_context {
dspqueue_t dsp_queue;
struct htp_mmap mmap[HTP_MAX_MMAPS];
dma_queue_t dma[HTP_MAX_NTHREADS];
struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1];
work_queue_t work_queue;
hmx_queue_t hmx_queue;
uint32_t n_threads;
struct fastdiv_values n_threads_div;
int thread_id;
int thread_prio;
bool hmx_enabled;
bool etm;
uint32_t profiler;
uint8_t * vtcm_base;
size_t vtcm_size;
uint32_t vtcm_rctx;
atomic_bool vtcm_valid;
atomic_bool vtcm_needs_release;
uint64_t max_vmem;
struct htp_dirty_range dirty_ranges[HTP_MAX_DIRTY_RANGES];
// Persistent DDR scratchpad for MUL_MAT_ID mappings
void * ddr_spad_base;
size_t ddr_spad_size;
struct htp_mdev_group mdev;
struct htp_ops_context octx;
qurt_thread_t main_thread;
void * main_stack;
atomic_bool killed;
size_t footprint;
};
static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct htp_context * ctx) {
memset(octx, 0, offsetof(struct htp_ops_context, dma_cache_dm));
octx->ctx = ctx;
octx->n_threads = ctx->n_threads;
octx->n_threads_div = ctx->n_threads_div;
}
static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) {
if (n_threads == 0 || n_threads > octx->ctx->n_threads) {
return false;
}
if (n_threads != octx->n_threads) {
octx->n_threads = n_threads;
octx->n_threads_div = n_threads == octx->ctx->n_threads
? octx->ctx->n_threads_div
: init_fastdiv_values(n_threads);
}
return true;
}
static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) {
if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) {
octx->status = status;
}
}
int op_matmul(struct htp_ops_context * octx);
int op_matmul_id(struct htp_ops_context * octx);
int op_matmul_nx(struct htp_ops_context * octx);
int op_matmul_id_nx(struct htp_ops_context * octx);
int op_binary(struct htp_ops_context * octx);
int op_unary(struct htp_ops_context * octx);
int op_sum(struct htp_ops_context * octx);
int op_sum_rows(struct htp_ops_context * octx);
int op_argmax(struct htp_ops_context * octx);
int op_glu(struct htp_ops_context * octx);
int op_softmax(struct htp_ops_context * octx);
int op_add_id(struct htp_ops_context * octx);
int op_rope(struct htp_ops_context * octx);
int op_flash_attn_ext(struct htp_ops_context * octx);
int op_set_rows(struct htp_ops_context * octx);
int op_get_rows(struct htp_ops_context * octx);
int op_cpy(struct htp_ops_context * octx);
int op_repeat(struct htp_ops_context * octx);
int op_argsort(struct htp_ops_context * octx);
int op_top_k(struct htp_ops_context * octx);
int op_ssm_conv(struct htp_ops_context * octx);
int op_cumsum(struct htp_ops_context * octx);
int op_fill(struct htp_ops_context * octx);
int op_concat(struct htp_ops_context * octx);
int op_diag(struct htp_ops_context * octx);
int op_solve_tri(struct htp_ops_context * octx);
int op_gated_delta_net(struct htp_ops_context * octx);
int op_pad(struct htp_ops_context * octx);
int op_im2col(struct htp_ops_context * octx);
int op_im2col_3d(struct htp_ops_context * octx);
int op_pool_2d(struct htp_ops_context * octx);
int op_pool_1d(struct htp_ops_context * octx);
int op_allreduce(struct htp_ops_context * octx);
int op_roll(struct htp_ops_context * octx);
#endif /* HTP_CTX_H */
llama.cpp/ggml/src/ggml-hexagon/htp/htp-dcache.c
+360
View File
@@ -0,0 +1,360 @@
#include "htp-dcache.h"
#include <qurt.h>
#include <qurt_memory.h>
#include <HAP_farf.h>
#include "hex-common.h"
#include "hex-utils.h"
#include "hex-fastdiv.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "work-queue.h"
struct l2flush_range {
uint32_t start; // line-aligned start address
uint32_t end; // line-aligned end address
uint32_t block_first; // global block index of this range's first block
uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial)
};
struct l2flush_multi_task {
struct htp_thread_trace * trace;
struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES];
uint32_t n_ranges;
uint32_t total_blocks;
uint32_t blocks_per_thread;
};
static void flush_all_dcache(struct htp_context * ctx) {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0);
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
hex_l2fetch_block(ctx, ctx->footprint);
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0);
memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges));
}
static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) {
struct l2flush_multi_task * task = (struct l2flush_multi_task *) data;
(void) n;
const uint32_t gb_first = i * task->blocks_per_thread;
uint32_t gb_last = gb_first + task->blocks_per_thread;
if (gb_last > task->total_blocks) {
gb_last = task->total_blocks;
}
if (gb_first >= gb_last) {
return;
}
struct htp_thread_trace * tr = &task->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);
for (uint32_t r = 0; r < task->n_ranges; r++) {
const struct l2flush_range * rg = &task->ranges[r];
const uint32_t rb_first = rg->block_first;
const uint32_t rb_last = rg->block_first + rg->n_blocks;
const uint32_t lo = gb_first > rb_first ? gb_first : rb_first;
const uint32_t hi = gb_last < rb_last ? gb_last : rb_last;
if (lo >= hi) {
continue;
}
const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE;
uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE;
if (e > rg->end) {
e = rg->end;
}
hex_l2flush((void *) (uintptr_t) s, e - s);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);
}
static void merge_dirty_ranges(struct htp_context * ctx) {
for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) {
struct htp_dirty_range * r = &ctx->dirty_ranges[i];
if (!r->start) continue;
for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES;) {
struct htp_dirty_range * s = &ctx->dirty_ranges[j];
if (i == j || !s->start || r->end < s->start || s->end < r->start) {
j++;
continue;
}
r->start = MIN(r->start, s->start);
r->end = MAX(r->end, s->end);
s->start = 0;
s->end = 0;
j = 0;
}
}
}
void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
const struct htp_tensor * pending[HTP_OP_MAX_OUTPUTS];
uint32_t n_pending = 0;
for (uint32_t i = 0; i < n; i++) {
const struct htp_tensor * t = tensors[i];
if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) {
continue;
}
uint32_t t_start = t->data;
uint32_t t_end = t_start + t->size;
bool merged = false;
for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES; j++) {
struct htp_dirty_range * r = &ctx->dirty_ranges[j];
if (!r->start) continue;
if (r->start <= t_end && t_start <= r->end) {
uint32_t new_start = (t_start < r->start) ? t_start : r->start;
uint32_t new_end = (t_end > r->end) ? t_end : r->end;
r->start = new_start;
r->end = new_end;
merged = true;
}
}
if (!merged) {
pending[n_pending++] = t;
}
}
merge_dirty_ranges(ctx);
if (n_pending == 0) {
return;
}
uint32_t empty_indices[HTP_MAX_DIRTY_RANGES];
uint32_t active_indices[HTP_MAX_DIRTY_RANGES];
uint32_t n_active = 0;
uint32_t n_empty = 0;
for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES; j++) {
if (ctx->dirty_ranges[j].start) {
active_indices[n_active++] = j;
} else {
empty_indices[n_empty++] = j;
}
}
if (n_pending <= n_empty) {
for (uint32_t i = 0; i < n_pending; i++) {
uint32_t idx = empty_indices[i];
struct htp_dirty_range * r = &ctx->dirty_ranges[idx];
r->start = pending[i]->data;
r->end = pending[i]->data + pending[i]->size;
}
merge_dirty_ranges(ctx);
return;
}
uint32_t n_evict = n_pending - n_empty;
uint32_t total_evict_size = 0;
for (uint32_t i = 0; i < n_evict; i++) {
uint32_t idx = active_indices[i];
struct htp_dirty_range * r = &ctx->dirty_ranges[idx];
total_evict_size += r->end - r->start;
}
if (total_evict_size > HEX_L2_FLUSH_ALL_THRESHOLD) {
flush_all_dcache(ctx);
for (uint32_t i = 0; i < n_pending; i++) {
struct htp_dirty_range * r = &ctx->dirty_ranges[i];
r->start = pending[i]->data;
r->end = pending[i]->data + pending[i]->size;
}
merge_dirty_ranges(ctx);
return;
}
if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) {
struct l2flush_multi_task task;
task.trace = ctx->trace;
task.n_ranges = n_evict;
uint32_t block_acc = 0;
for (uint32_t i = 0; i < n_evict; i++) {
uint32_t idx = active_indices[i];
struct htp_dirty_range * r = &ctx->dirty_ranges[idx];
struct l2flush_range * rg = &task.ranges[i];
rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE);
rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE);
rg->block_first = block_acc;
rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
block_acc += rg->n_blocks;
}
task.total_blocks = block_acc;
task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div);
work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads);
} else {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0);
for (uint32_t i = 0; i < n_evict; i++) {
uint32_t idx = active_indices[i];
struct htp_dirty_range * r = &ctx->dirty_ranges[idx];
uint32_t size = r->end - r->start;
hex_l2flush((void *) (uintptr_t) r->start, size);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0);
}
for (uint32_t i = 0; i < n_evict; i++) {
uint32_t idx = active_indices[i];
struct htp_dirty_range * r = &ctx->dirty_ranges[idx];
r->start = pending[i]->data;
r->end = pending[i]->data + pending[i]->size;
}
for (uint32_t i = 0; i < n_empty; i++) {
uint32_t idx = empty_indices[i];
struct htp_dirty_range * r = &ctx->dirty_ranges[idx];
r->start = pending[n_evict + i]->data;
r->end = pending[n_evict + i]->data + pending[n_evict + i]->size;
}
merge_dirty_ranges(ctx);
}
static void make_tensor_clean(struct htp_context * ctx, const struct htp_tensor * t) {
if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) {
return;
}
uint32_t t_start = t->data;
uint32_t t_end = t_start + t->size;
for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) {
struct htp_dirty_range * r = &ctx->dirty_ranges[i];
if (!r->start) continue;
if (r->start < t_end && t_start < r->end) {
if (t_start <= r->start && r->end <= t_end) {
r->start = 0;
r->end = 0;
} else if (t_start <= r->start) {
r->start = t_end;
} else if (r->end <= t_end) {
r->end = t_start;
}
}
}
}
static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_tensor * t) {
if (!t || (t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE))) {
return false;
}
uint32_t t_start = t->data;
uint32_t t_end = t_start + t->size;
for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) {
struct htp_dirty_range * r = &ctx->dirty_ranges[i];
if (!r->start) continue;
if (r->start < t_end && t_start < r->end) {
return true;
}
}
return false;
}
static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) {
if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {
struct l2flush_multi_task task;
task.trace = ctx->trace;
task.n_ranges = n_ranges;
uint32_t block_acc = 0;
for (uint32_t i = 0; i < n_ranges; i++) {
const struct htp_dirty_range * r = &ranges[i];
struct l2flush_range * rg = &task.ranges[i];
rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE);
rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE);
rg->block_first = block_acc;
rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
block_acc += rg->n_blocks;
}
task.total_blocks = block_acc;
task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div);
work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads);
} else {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0);
for (uint32_t i = 0; i < n_ranges; i++) {
const struct htp_dirty_range * r = &ranges[i];
hex_l2flush((void *) (uintptr_t) r->start, r->end - r->start);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0);
}
}
void htp_dcache_flush_dirty_ranges(struct htp_context * ctx) {
struct htp_dirty_range ranges[HTP_MAX_DIRTY_RANGES];
uint32_t n_ranges = 0;
uint64_t total_dirty = 0;
for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) {
const struct htp_dirty_range * r = &ctx->dirty_ranges[i];
if (!r->start) {
continue;
}
ranges[n_ranges++] = *r;
total_dirty += r->end - r->start;
}
if (total_dirty == 0) {
return;
}
if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) {
flush_all_dcache(ctx);
return;
}
flush_dirty_ranges(ctx, ranges, n_ranges, total_dirty);
memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges));
}
void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS];
struct htp_dirty_range ranges[HTP_OP_MAX_INPUTS];
uint32_t n_dirty = 0;
uint64_t total_dirty = 0;
for (uint32_t i = 0; i < n; i++) {
const struct htp_tensor * t = tensors[i];
if (is_tensor_dirty(ctx, t)) {
dirty_tensors[n_dirty++] = t;
ranges[n_dirty - 1].start = t->data;
ranges[n_dirty - 1].end = t->data + t->size;
total_dirty += t->size;
}
}
if (total_dirty == 0) {
return;
}
if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) {
flush_all_dcache(ctx);
return;
}
flush_dirty_ranges(ctx, ranges, n_dirty, total_dirty);
for (uint32_t i = 0; i < n_dirty; i++) {
make_tensor_clean(ctx, dirty_tensors[i]);
}
}
llama.cpp/ggml/src/ggml-hexagon/htp/htp-dcache.h
+20
View File
@@ -0,0 +1,20 @@
#ifndef HTP_DCACHE_H
#define HTP_DCACHE_H
#include <stdint.h>
#include "htp-ops.h"
#define HTP_MAX_DIRTY_RANGES 64
struct htp_dirty_range {
uint32_t start;
uint32_t end;
};
struct htp_context;
void htp_dcache_flush_dirty_ranges(struct htp_context * ctx);
void htp_dcache_flush_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
void htp_dcache_mark_dirty_tensors(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
#endif // HTP_DCACHE_H
llama.cpp/ggml/src/ggml-hexagon/htp/htp-fence.h
+89
View File
@@ -0,0 +1,89 @@
#ifndef HTP_FENCE_H
#define HTP_FENCE_H
#include <stdatomic.h>
#include <stdint.h>
#include <HAP_farf.h>
#include "hex-utils.h"
#include "htp-ops.h"
#include "htp-ctx.h"
static inline atomic_uint * htp_mdev_fence_slot(const void * fence_base, uint32_t idx) {
return (atomic_uint *) ((const uint8_t *) fence_base + (size_t) idx * HTP_FENCE_SLOT_SIZE);
}
static inline void htp_fence_write(void * fence_ptr, uint32_t seq, uint32_t status) {
atomic_uint * fence = (atomic_uint *) fence_ptr;
atomic_store(&fence[1], status);
atomic_store(&fence[0], seq);
asm volatile ("syncht" : : : "memory");
Q6_dccleaninva_A((void *) fence);
}
static inline void htp_fence_read(const void * fence_ptr, uint32_t * seq, uint32_t * status) {
const atomic_uint * fence = (const atomic_uint *) fence_ptr;
Q6_dccleaninva_A((void *) fence);
asm volatile ("syncht" : : : "memory");
*seq = atomic_load(&fence[0]);
*status = atomic_load(&fence[1]);
}
static inline void htp_mdev_group_barrier(struct htp_ops_context * octx) {
struct htp_context * ctx = octx->ctx;
if (ctx->mdev.count <= 1) {
return;
}
const uint32_t seq = ++ctx->mdev.fence_seq;
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
const uint32_t mdev_idx = ctx->mdev.idx;
const uint32_t mdev_count = ctx->mdev.count;
uint8_t * fence_base = ctx->mdev.fence_base;
atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx);
htp_fence_write(my_fence, seq, octx->status);
for (uint32_t d = 0; d < mdev_count; d++) {
if (d == mdev_idx) continue;
atomic_uint * peer_fence = htp_mdev_fence_slot(fence_base, d);
uint64_t spins = 0;
while (1) {
uint32_t peer_seq;
uint32_t peer_status;
htp_fence_read(peer_fence, &peer_seq, &peer_status);
if ((int32_t)(peer_seq - seq) >= 0) {
if (peer_status > HTP_STATUS_OK) {
FARF(ERROR, "ggml-hex: mdev %u peer %u failed with status %u : seq 0x%08x\n",
mdev_idx, d, peer_status, seq);
htp_ops_context_set_status(octx, peer_status);
}
break;
}
if (++spins == 10000) {
FARF(ALWAYS, "ggml-hex: mdev %u waiting for mdev %u : seq 0x%08x (b %u op %u) my-fence %p peer-fence %p peer-seq 0x%08x (diff %d)\n",
mdev_idx, d, seq, seq >> 12, seq & 0xfff, my_fence, peer_fence, peer_seq, (int32_t)(peer_seq - seq));
}
if (spins > HTP_FENCE_TIMEOUT) {
FARF(ERROR, "ggml-hex: mdev %u timeout waiting for mdev %u : seq 0x%08x (b %u op %u) peer-fence %p peer-seq 0x%08x\n",
mdev_idx, d, seq, seq >> 12, seq & 0xfff, peer_fence, peer_seq);
htp_ops_context_set_status(octx, HTP_STATUS_INTERNAL_ERR);
break;
}
hex_pause();
}
}
asm volatile ("syncht" : : : "memory");
if (octx->status > HTP_STATUS_OK) {
htp_fence_write(my_fence, seq, octx->status);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
}
#endif // HTP_FENCE_H
llama.cpp/ggml/src/ggml-hexagon/htp/htp-ops.h
+281
View File
@@ -0,0 +1,281 @@
#ifndef HTP_OPS_H
#define HTP_OPS_H
#include <assert.h>
// ggml-common.h must be included prio to this header
enum htp_status {
HTP_STATUS_OK = 1,
HTP_STATUS_INTERNAL_ERR = 2,
HTP_STATUS_NO_SUPPORT = 3,
HTP_STATUS_INVAL_PARAMS = 4,
HTP_STATUS_VTCM_TOO_SMALL = 5,
};
// First set of values must match the ggml_type.
// Duplicated here because we can't include full ggml.h in the htp build.
// We have some static_asserts in the cpp code to ensure things are in sync.
enum htp_data_type {
HTP_TYPE_F32 = 0,
HTP_TYPE_F16 = 1,
HTP_TYPE_Q4_0 = 2,
HTP_TYPE_Q4_1 = 3,
HTP_TYPE_Q8_0 = 8,
HTP_TYPE_Q2_K = 10,
HTP_TYPE_Q3_K = 11,
HTP_TYPE_Q4_K = 12,
HTP_TYPE_Q5_K = 13,
HTP_TYPE_Q6_K = 14,
HTP_TYPE_IQ4_NL = 20,
HTP_TYPE_I32 = 26,
HTP_TYPE_I64 = 27,
HTP_TYPE_BF16 = 30,
HTP_TYPE_MXFP4 = 39,
HTP_TYPE_Q1_0 = 41,
HTP_TYPE_Q2_0 = 42,
HTP_TYPE_INVALID
};
// Do not reorder first 4 (used as an index)
enum htp_op_code {
HTP_OP_MUL = 0,
HTP_OP_ADD = 1,
HTP_OP_SUB = 2,
HTP_OP_DIV = 3,
HTP_OP_MUL_MAT,
HTP_OP_MUL_MAT_ID,
HTP_OP_MUL_MAT_NX,
HTP_OP_MUL_MAT_ID_NX,
HTP_OP_MUL_MAT_ADD,
HTP_OP_RMS_NORM,
HTP_OP_RMS_NORM_MUL,
HTP_OP_UNARY_SILU,
HTP_OP_UNARY_GELU,
HTP_OP_UNARY_SIGMOID,
HTP_OP_UNARY_EXP,
HTP_OP_UNARY_NEG,
HTP_OP_UNARY_SOFTPLUS,
HTP_OP_UNARY_TANH,
HTP_OP_UNARY_ABS,
HTP_OP_UNARY_LOG,
HTP_OP_UNARY_RELU,
HTP_OP_UNARY_STEP,
HTP_OP_GLU_SWIGLU,
HTP_OP_GLU_SWIGLU_OAI,
HTP_OP_GLU_GEGLU,
HTP_OP_GLU_GEGLU_QUICK,
HTP_OP_SOFTMAX,
HTP_OP_ADD_ID,
HTP_OP_ROPE,
HTP_OP_FLASH_ATTN_EXT,
HTP_OP_SET_ROWS,
HTP_OP_GET_ROWS,
HTP_OP_SCALE,
HTP_OP_CPY,
HTP_OP_CPY_FENCE,
HTP_OP_ARGSORT,
HTP_OP_TOP_K,
HTP_OP_SQR,
HTP_OP_SQRT,
HTP_OP_SUM,
HTP_OP_SUM_ROWS,
HTP_OP_SSM_CONV,
HTP_OP_REPEAT,
HTP_OP_CUMSUM,
HTP_OP_FILL,
HTP_OP_DIAG,
HTP_OP_SOLVE_TRI,
HTP_OP_L2_NORM,
HTP_OP_GATED_DELTA_NET,
HTP_OP_TRI,
HTP_OP_PAD,
HTP_OP_NORM,
HTP_OP_CONCAT,
HTP_OP_CLAMP,
HTP_OP_LEAKY_RELU,
HTP_OP_IM2COL,
HTP_OP_FENCE,
HTP_OP_ALLREDUCE,
HTP_OP_ALLREDUCE_ADD,
HTP_OP_GLU_SWIGLU_CLAMP,
HTP_OP_MDEV_GROUP,
HTP_OP_ROLL,
HTP_OP_ARGMAX,
HTP_OP_UNARY_GELU_ERF,
HTP_OP_GLU_GEGLU_ERF,
HTP_OP_POOL_2D,
HTP_OP_POOL_1D,
HTP_OP_UNARY_GELU_QUICK,
HTP_OP_IM2COL_3D,
HTP_OP_HADAMARD,
HTP_OP_GLU_REGLU,
HTP_OP_GLU_SIGMOID,
HTP_OP_UNARY_CEIL,
HTP_OP_UNARY_SGN,
HTP_OP_INVALID
};
#define HTP_OP_MAX_DIMS 4 // aka GGML_MAX_DIMS
#define HTP_OP_MAX_INPUTS 10 // aka GGML_MAX_SRCS
#define HTP_OP_MAX_OUTPUTS 4
#define HTP_OP_MAX_PARAMS 16 // aka GGML_MAX_OP_PARAMS
#define HTP_OP_MAX_KERN_PARAMS 32
#define HTP_OP_MAX_BUFS 64
#define HTP_OP_MAX_TENSORS 65534 // must stay under 64K (uint16)
#define HTP_FENCE_TIMEOUT (1000000000ULL)
#define HTP_FENCE_SLOT_SIZE 128
#define HTP_OP_MAX_VMEM_DEFAULT (3355443200u)
#define HTP_MMAP_MAX_VMEM (2147483648u)
enum htp_tensor_flags {
HTP_TENSOR_WEIGHT = (1U << 0), // Tensor buffer model weight data (not compute)
HTP_TENSOR_TILED = (1U << 1), // Tensor is in tiled format
HTP_TENSOR_FENCE = (1U << 2), // Tensor is synchronization fence (explicitly managed)
HTP_TENSOR_FUSEABLE = (1U << 3), // Tensor can be fused (host-only)
};
enum htp_buf_flags {
HTP_BUF_EXTENDED = (1U << 0),
};
// Tensor descriptor
struct htp_tensor {
uint64_t data; // Buffer offset in the messages, and data pointer on the NPU
uint32_t size; // Data size in bytes
uint32_t flags; // Buffer / tensor flags
uint32_t type; // Data type
uint16_t bi; // Buffer index
uint16_t ti; // Tensor index
uint32_t ne[HTP_OP_MAX_DIMS]; // Number of elements
uint32_t nb[HTP_OP_MAX_DIMS]; // Stride in bytes (see ggml.h ggml_tensor)
};
// Buffer descriptor
struct htp_buf_desc {
uint64_t base; // base address
uint64_t size; // total size
uint32_t flags; // HTP_BUF_*
uint32_t fd; // file descriptor
};
enum htp_op_flags {
HTP_OPFLAGS_STUB = (1U << 0),
};
// Op descriptor
struct htp_op_desc {
uint32_t opcode; // GGML/HTP Op
uint32_t flags; // Op flags
int32_t params[HTP_OP_MAX_PARAMS]; // Params for the op, e.g. epsilon of RMS norm
int32_t kernel_params[HTP_OP_MAX_KERN_PARAMS]; // generic blob for host-precomputed parameters
uint16_t src[HTP_OP_MAX_INPUTS]; // Input tensors indices
uint16_t dst[HTP_OP_MAX_OUTPUTS]; // Output tensor indices
uint16_t pad[2]; // padding to align to 64 bits
};
#ifndef HTP_MAX_NTHREADS
#define HTP_MAX_NTHREADS 10
#endif
#define HTP_TRACE_MAX_EVENTS 256
enum htp_profiler_mode {
HTP_PROF_DISABLED = 0,
HTP_PROF_BASIC = 1,
HTP_PROF_PMU = 2,
HTP_PROF_TRACE = 3,
};
// DCVS/HMX power profile requested for the session.
enum htp_power_mode {
HTP_POWER_MODE_LOW_POWER_SAVER = 0,
HTP_POWER_MODE_POWER_SAVER = 1,
HTP_POWER_MODE_HIGH_POWER_SAVER = 2,
HTP_POWER_MODE_LOW_BALANCED = 3,
HTP_POWER_MODE_BALANCED = 4,
HTP_POWER_MODE_HIGH_PERFORMANCE = 5,
HTP_POWER_MODE_SUSTAINED_HIGH_PERFORMANCE = 6,
HTP_POWER_MODE_BURST = 7,
HTP_POWER_MODE_COUNT
};
enum htp_trace_event_id {
HTP_TRACE_EVT_DMA = 0,
HTP_TRACE_EVT_L2FLUSH = 1,
HTP_TRACE_EVT_INIT = 2,
HTP_TRACE_EVT_BUFF = 3,
HTP_TRACE_EVT_FENCE = 4,
HTP_TRACE_EVT_HVX_COMP = 20,
HTP_TRACE_EVT_HVX_A_QUANT = 21,
HTP_TRACE_EVT_HVX_A_PREP = 22,
HTP_TRACE_EVT_HVX_W_DEQUANT = 23,
HTP_TRACE_EVT_HVX_W_PREP = 24,
HTP_TRACE_EVT_HVX_O_PROC = 25,
HTP_TRACE_EVT_HVX_FA_QK = 26,
HTP_TRACE_EVT_HVX_FA_SFM = 27,
HTP_TRACE_EVT_HVX_FA_Q_PREP = 28,
HTP_TRACE_EVT_HVX_FA_K_PREP = 29,
HTP_TRACE_EVT_HVX_FA_V_PREP = 30,
HTP_TRACE_EVT_HVX_GDN_PREP = 31,
HTP_TRACE_EVT_HVX_GDN_SOLVE = 32,
HTP_TRACE_EVT_HVX_GDN_V_PREP = 33,
HTP_TRACE_EVT_HVX_GDN_D_PREP = 34,
HTP_TRACE_EVT_HVX_GDN_OUT = 35,
HTP_TRACE_EVT_HVX_GDN_STATE = 36,
HTP_TRACE_EVT_HVX_GDN_REM = 37,
HTP_TRACE_EVT_HMX_COMP = 40,
};
struct htp_trace_desc {
uint32_t cycles; // lower 32-bits of cycle counter
uint16_t id; // Event ID
uint16_t info; // bit 15: is_stop. bits 14-0: tile/chunk index or other metadata.
};
#define HTP_PROF_PMU_NCNT 8
// Profile descriptor
struct htp_prof_desc {
uint32_t opcode; // GGML/HTP Op
uint32_t usecs; // Number of usec
uint32_t cycles_start; // Start cycle counter
uint32_t cycles_stop; // Stop cycle counter
uint32_t pmu[HTP_PROF_PMU_NCNT]; // PMU counters
};
struct htp_opbatch_req {
uint64_t seq; // Sequence number
uint32_t n_bufs; // Number of buffers
uint32_t n_tensors; // Number of tensors
uint32_t n_ops; // Number of ops
uint32_t n_traces; // Number of trace descriptors per thread
// struct htp_buf_desc bufs[]; -- dspqueue buf 0
// struct htp_tensor tensors[]; -- dspqueue buf 0
// struct htp_op_desc ops[]; -- dspqueue buf 0
};
struct htp_opbatch_rsp {
uint64_t seq; // Sequence number
uint64_t cycles_start; // Start cycle counter
uint64_t cycles_stop; // Stop cycle counter
uint32_t status; // HTP_STATUS_...
uint32_t n_bufs; // Number of buffers
uint32_t n_tensors; // Number of tensors
uint32_t n_ops; // Number of op profile descriptors
uint32_t usecs; // Number of usec
uint32_t n_traces[HTP_MAX_NTHREADS + 1];
// struct htp_prof_desc profs[]; -- dspqueue buf 0
};
#endif /* HTP_OPS_H */
llama.cpp/ggml/src/ggml-hexagon/htp/htp-swiv.py
+512
View File
@@ -0,0 +1,512 @@
#!/usr/bin/env python3
import argparse
import logging
import os
import shutil
import struct
import sys
import tempfile
import zlib
ELF_MAGIC = b'\x7fELF'
ELF_CLASS_INDEX = 4
ELFCLASS32 = 1
ELFCLASS64 = 2
ELF_VERSION_INDEX = 6
EV_CURRENT = 1
MAX_PHDR_COUNT = 100
LOAD_TYPE = 0x1
SHT_NOBITS = 8
SWIV_MAGIC = b'SWIV'
SWIV_SEG_SIZE_MIN = 16
# SHT_LOUSER plus ASCII "SWIV".
SWIV_SECTION_TYPE = 0xD3574956
logger = logging.getLogger("htp-swiv")
class Elf_Ehdr_common:
s = struct.Struct('16sHHI')
def __init__(self, data):
unpacked_data = (Elf_Ehdr_common.s).unpack(data)
self.e_ident = unpacked_data[0]
self.e_type = unpacked_data[1]
self.e_machine = unpacked_data[2]
self.e_version = unpacked_data[3]
class Elf32_Ehdr:
s = struct.Struct('16sHHIIIIIHHHHHH')
def __init__(self, data):
unpacked_data = (Elf32_Ehdr.s).unpack(data)
self.e_ident = unpacked_data[0]
self.e_type = unpacked_data[1]
self.e_machine = unpacked_data[2]
self.e_version = unpacked_data[3]
self.e_entry = unpacked_data[4]
self.e_phoff = unpacked_data[5]
self.e_shoff = unpacked_data[6]
self.e_flags = unpacked_data[7]
self.e_ehsize = unpacked_data[8]
self.e_phentsize = unpacked_data[9]
self.e_phnum = unpacked_data[10]
self.e_shentsize = unpacked_data[11]
self.e_shnum = unpacked_data[12]
self.e_shstrndx = unpacked_data[13]
def get_packed_data(self):
values = [self.e_ident,
self.e_type,
self.e_machine,
self.e_version,
self.e_entry,
self.e_phoff,
self.e_shoff,
self.e_flags,
self.e_ehsize,
self.e_phentsize,
self.e_phnum,
self.e_shentsize,
self.e_shnum,
self.e_shstrndx
]
return (Elf32_Ehdr.s).pack(*values)
class Elf32_Phdr:
s = struct.Struct('I' * 8)
def __init__(self, data):
unpacked_data = (Elf32_Phdr.s).unpack(data)
self.p_type = unpacked_data[0]
self.p_offset = unpacked_data[1]
self.p_vaddr = unpacked_data[2]
self.p_paddr = unpacked_data[3]
self.p_filesz = unpacked_data[4]
self.p_memsz = unpacked_data[5]
self.p_flags = unpacked_data[6]
self.p_align = unpacked_data[7]
def get_packed_data(self):
values = [self.p_type,
self.p_offset,
self.p_vaddr,
self.p_paddr,
self.p_filesz,
self.p_memsz,
self.p_flags,
self.p_align
]
return (Elf32_Phdr.s).pack(*values)
class Elf32_Shdr:
s = struct.Struct('I' * 10)
def __init__(self, data):
unpacked_data = (Elf32_Shdr.s).unpack(data)
self.sh_name = unpacked_data[0]
self.sh_type = unpacked_data[1]
self.sh_flags = unpacked_data[2]
self.sh_addr = unpacked_data[3]
self.sh_offset = unpacked_data[4]
self.sh_size = unpacked_data[5]
self.sh_link = unpacked_data[6]
self.sh_info = unpacked_data[7]
self.sh_addralign = unpacked_data[8]
self.sh_entsize = unpacked_data[9]
self.sh_name_str = ""
def get_packed_data(self):
values = [self.sh_name,
self.sh_type,
self.sh_flags,
self.sh_addr,
self.sh_offset,
self.sh_size,
self.sh_link,
self.sh_info,
self.sh_addralign,
self.sh_entsize]
return (Elf32_Shdr.s).pack(*values)
class Elf64_Ehdr:
s = struct.Struct('16sHHIQQQIHHHHHH')
def __init__(self, data):
unpacked_data = (Elf64_Ehdr.s).unpack(data)
self.e_ident = unpacked_data[0]
self.e_type = unpacked_data[1]
self.e_machine = unpacked_data[2]
self.e_version = unpacked_data[3]
self.e_entry = unpacked_data[4]
self.e_phoff = unpacked_data[5]
self.e_shoff = unpacked_data[6]
self.e_flags = unpacked_data[7]
self.e_ehsize = unpacked_data[8]
self.e_phentsize = unpacked_data[9]
self.e_phnum = unpacked_data[10]
self.e_shentsize = unpacked_data[11]
self.e_shnum = unpacked_data[12]
self.e_shstrndx = unpacked_data[13]
def get_packed_data(self):
values = [self.e_ident,
self.e_type,
self.e_machine,
self.e_version,
self.e_entry,
self.e_phoff,
self.e_shoff,
self.e_flags,
self.e_ehsize,
self.e_phentsize,
self.e_phnum,
self.e_shentsize,
self.e_shnum,
self.e_shstrndx
]
return (Elf64_Ehdr.s).pack(*values)
class Elf64_Phdr:
s = struct.Struct('IIQQQQQQ')
def __init__(self, data):
unpacked_data = (Elf64_Phdr.s).unpack(data)
self.p_type = unpacked_data[0]
self.p_flags = unpacked_data[1]
self.p_offset = unpacked_data[2]
self.p_vaddr = unpacked_data[3]
self.p_paddr = unpacked_data[4]
self.p_filesz = unpacked_data[5]
self.p_memsz = unpacked_data[6]
self.p_align = unpacked_data[7]
def get_packed_data(self):
values = [self.p_type,
self.p_flags,
self.p_offset,
self.p_vaddr,
self.p_paddr,
self.p_filesz,
self.p_memsz,
self.p_align
]
return (Elf64_Phdr.s).pack(*values)
class Elf64_Shdr:
s = struct.Struct('IIQQQQIIQQ')
def __init__(self, data):
unpacked_data = (Elf64_Shdr.s).unpack(data)
self.sh_name = unpacked_data[0]
self.sh_type = unpacked_data[1]
self.sh_flags = unpacked_data[2]
self.sh_addr = unpacked_data[3]
self.sh_offset = unpacked_data[4]
self.sh_size = unpacked_data[5]
self.sh_link = unpacked_data[6]
self.sh_info = unpacked_data[7]
self.sh_addralign = unpacked_data[8]
self.sh_entsize = unpacked_data[9]
self.sh_name_str = ""
def get_packed_data(self):
values = [self.sh_name,
self.sh_type,
self.sh_flags,
self.sh_addr,
self.sh_offset,
self.sh_size,
self.sh_link,
self.sh_info,
self.sh_addralign,
self.sh_entsize]
return (Elf64_Shdr.s).pack(*values)
class SWIV_Seg:
s = struct.Struct('4sIQ')
def __init__(self):
self.magic = SWIV_MAGIC
self.crc = 0
self.padding = 0
def get_packed_data(self):
values = [self.magic,
self.crc,
self.padding
]
return (SWIV_Seg.s).pack(*values)
def is_elf64(elf_header):
return elf_header.e_ident[ELF_CLASS_INDEX] == ELFCLASS64
def verify_elf_header(elf_header):
ident = elf_header.e_ident
return (ident.startswith(ELF_MAGIC)
and ident[ELF_CLASS_INDEX] in (ELFCLASS32, ELFCLASS64)
and ident[ELF_VERSION_INDEX] == EV_CURRENT)
def file_copy_offset(in_fp, in_off, out_fp, out_off, num_bytes):
in_fp.seek(in_off)
read_in = in_fp.read(num_bytes)
out_fp.seek(out_off)
out_fp.write(read_in)
return num_bytes
def preprocess_elf_file(elf_file_name):
with open(elf_file_name, 'rb') as elf_fp:
elf_header = Elf_Ehdr_common(elf_fp.read(Elf_Ehdr_common.s.size))
if verify_elf_header(elf_header) is False:
raise RuntimeError("Input ELF verification failed : " + elf_file_name)
elf_fp.seek(0)
if is_elf64(elf_header):
elf_header = Elf64_Ehdr(elf_fp.read(Elf64_Ehdr.s.size))
else:
elf_header = Elf32_Ehdr(elf_fp.read(Elf32_Ehdr.s.size))
phdr_table = []
phdr_size = elf_header.e_phentsize
file_offset = elf_header.e_phoff
elf_fp.seek(file_offset)
for _ in range(elf_header.e_phnum):
if is_elf64(elf_header):
phdr_table.append(Elf64_Phdr(elf_fp.read(phdr_size)))
else:
phdr_table.append(Elf32_Phdr(elf_fp.read(phdr_size)))
return [elf_header, phdr_table]
def process_section_header(elf_file_name):
with open(elf_file_name, 'rb') as elf_fp:
elf_header = Elf_Ehdr_common(elf_fp.read(Elf_Ehdr_common.s.size))
if verify_elf_header(elf_header) is False:
raise RuntimeError("Input ELF verification failed : " + elf_file_name)
elf_fp.seek(0)
if is_elf64(elf_header):
elf_header = Elf64_Ehdr(elf_fp.read(Elf64_Ehdr.s.size))
else:
elf_header = Elf32_Ehdr(elf_fp.read(Elf32_Ehdr.s.size))
shdr_table = []
shdr_size = elf_header.e_shentsize
elf_fp.seek(elf_header.e_shoff)
for _ in range(elf_header.e_shnum):
if is_elf64(elf_header):
shdr_table.append(Elf64_Shdr(elf_fp.read(shdr_size)))
else:
shdr_table.append(Elf32_Shdr(elf_fp.read(shdr_size)))
return shdr_table
def crc32_update_file(elf_fp, offset, size, crc):
elf_fp.seek(offset)
remaining = size
while remaining > 0:
chunk = elf_fp.read(min(1024 * 1024, remaining))
if not chunk:
raise RuntimeError("Unexpected EOF while calculating CRC")
crc = zlib.crc32(chunk, crc)
remaining -= len(chunk)
return crc
def crc32_update_zeros(size, crc):
zero_chunk = b'\x00' * min(1024 * 1024, size)
remaining = size
while remaining > 0:
chunk = zero_chunk[:min(len(zero_chunk), remaining)]
crc = zlib.crc32(chunk, crc)
remaining -= len(chunk)
return crc
def crc32_calculate(elf_fp, phdr_table):
crc = 0
# SWIV validates loadable segment contents, including BSS as zeros.
for curr_phdr in phdr_table:
if curr_phdr.p_type != LOAD_TYPE:
continue
crc = crc32_update_file(elf_fp, curr_phdr.p_offset, curr_phdr.p_filesz, crc)
zero_size = curr_phdr.p_memsz - curr_phdr.p_filesz
if zero_size > 0:
crc = crc32_update_zeros(zero_size, crc)
crc &= 0xffffffff
logger.info("calculated CRC: %#x", crc)
return crc
def verify_swiv_seg(elf_in_fp, offset):
elf_in_fp.seek(offset)
swiv_seg = SWIV_Seg()
unpacked_data = (SWIV_Seg.s).unpack(elf_in_fp.read(SWIV_SEG_SIZE_MIN))
swiv_seg.magic = unpacked_data[0]
return swiv_seg.magic == SWIV_MAGIC
def same_output_file(input_elf, output_elf):
try:
return os.path.exists(output_elf) and os.path.samefile(input_elf, output_elf)
except FileNotFoundError:
return False
def make_output_target(input_elf, output_elf):
if not same_output_file(input_elf, output_elf):
return output_elf, None
output_dir = os.path.dirname(os.path.abspath(output_elf))
output_base = os.path.basename(output_elf)
fd, temp_output = tempfile.mkstemp(prefix=output_base + ".", suffix=".tmp", dir=output_dir)
os.close(fd)
shutil.copymode(input_elf, temp_output)
return temp_output, temp_output
def swiv_generate(input_elf, output_elf):
output_target, temp_output = make_output_target(input_elf, output_elf)
try:
with open(input_elf, "rb") as elf_in_fp, open(output_target, "wb+") as elf_out_fp:
[elf_header, phdr_table] = preprocess_elf_file(input_elf)
num_phdrs = elf_header.e_phnum
num_shdrs = elf_header.e_shnum
phdr_size = elf_header.e_phentsize
if num_phdrs == 0 or num_phdrs > MAX_PHDR_COUNT:
raise RuntimeError("Input ELF has invalid number of program headers.")
if elf_header.e_shoff == 0 or num_shdrs == 0:
raise RuntimeError("Input ELF does not contain a section header table.")
shdr_table_input = process_section_header(input_elf)
shdr_table_output = process_section_header(input_elf)
last_phdr = phdr_table[num_phdrs - 1]
if last_phdr.p_memsz == SWIV_SEG_SIZE_MIN and verify_swiv_seg(elf_in_fp, last_phdr.p_offset):
raise RuntimeError("A SWIV segment is detected in the input ELF.")
last_shdr = shdr_table_input[elf_header.e_shnum - 1]
if last_shdr.sh_size == SWIV_SEG_SIZE_MIN and verify_swiv_seg(elf_in_fp, last_shdr.sh_offset):
raise RuntimeError("A SWIV section is detected in the input ELF.")
if is_elf64(elf_header):
swiv_shdr = Elf64_Shdr(b'\0' * Elf64_Shdr.s.size)
else:
swiv_shdr = Elf32_Shdr(b'\0' * Elf32_Shdr.s.size)
swiv_shdr.sh_name = 0
swiv_shdr.sh_type = SWIV_SECTION_TYPE
swiv_shdr.sh_flags = 0
swiv_shdr.sh_addr = 0
swiv_shdr.sh_size = SWIV_SEG_SIZE_MIN
swiv_shdr.sh_offset = elf_header.e_shoff + (elf_header.e_shnum + 1) * elf_header.e_shentsize
shdr_table_output.append(swiv_shdr)
elf_header.e_shnum += 1
if elf_header.e_phoff != 0:
for phdr in phdr_table:
file_copy_offset(elf_in_fp, phdr.p_offset, elf_out_fp, phdr.p_offset, phdr.p_filesz)
elf_out_fp.seek(0)
elf_out_fp.write(elf_header.get_packed_data())
phdr_start = elf_header.e_phoff
for phdr in phdr_table:
elf_out_fp.seek(phdr_start)
elf_out_fp.write(phdr.get_packed_data())
phdr_start += phdr_size
for shdr in shdr_table_input[:num_shdrs]:
if shdr.sh_type == SHT_NOBITS or shdr.sh_size == 0:
continue
file_copy_offset(elf_in_fp, shdr.sh_offset, elf_out_fp, shdr.sh_offset, shdr.sh_size)
shdr_start = elf_header.e_shoff
for shdr in shdr_table_output:
elf_out_fp.seek(shdr_start)
elf_out_fp.write(shdr.get_packed_data())
shdr_start += elf_header.e_shentsize
elf_out_fp.flush()
swiv_seg = SWIV_Seg()
swiv_seg.crc = crc32_calculate(elf_out_fp, phdr_table)
elf_out_fp.seek(swiv_shdr.sh_offset)
elf_out_fp.write(swiv_seg.get_packed_data())
if temp_output is not None:
os.replace(temp_output, output_elf)
temp_output = None
return 0
finally:
if temp_output is not None and os.path.exists(temp_output):
os.unlink(temp_output)
def main(argv):
parser = argparse.ArgumentParser(description="Add a Software Image Validation (SWIV) section to an HTP ELF file.")
parser.add_argument("-i", "--input", dest="input_file", metavar="INPUT", required=True)
parser.add_argument("-o", "--output", dest="output_file", metavar="OUTPUT", required=True)
parser.add_argument("-v", "--verbose", action="store_true", help="show calculated CRC")
args = parser.parse_args(argv)
logging.basicConfig(
level=logging.INFO if args.verbose else logging.WARNING,
format="%(message)s",
)
try:
return swiv_generate(args.input_file, args.output_file)
except RuntimeError as exc:
logger.error("ERROR: %s", exc)
return 2
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
llama.cpp/ggml/src/ggml-hexagon/htp/htp-tensor.h
+199
View File
@@ -0,0 +1,199 @@
#ifndef HTP_TENSOR_H
#define HTP_TENSOR_H
#include <stdint.h>
#include <stdbool.h>
#include "htp-ops.h"
#include "hex-bitmap.h"
#include "hex-common.h"
#include "hex-fastdiv.h"
#define HTP_TENSOR_MDEV_LINE_SIZE 128
// Currently HMX requires 32x32 tile formats (do not change)
#define HTP_TENSOR_TILE_N_COLS 32
#define HTP_TENSOR_TILE_N_ROWS 32
#define HTP_TENSOR_TILE_SIZE_F16 2048
#define HTP_TENSOR_TILE_SIZE_Q4_0 576
#define HTP_TENSOR_TILE_SIZE_Q4_1 640
#define HTP_TENSOR_TILE_SIZE_Q8_0 1088
#define HTP_TENSOR_TILE_SIZE_IQ4_NL 576
#define HTP_TENSOR_TILE_SIZE_MXFP4 544
#define HTP_TENSOR_TILE_SIZE_Q4_K 640
#define HTP_TENSOR_TILE_SIZE_Q5_K 768
#define HTP_TENSOR_TILE_SIZE_Q6_K 896
#define HTP_TENSOR_TILE_SIZE_Q3_K 512
#define HTP_TENSOR_TILE_SIZE_Q2_K 336
#define HTP_TENSOR_TILE_K_Q1_0 128
#define HTP_TENSOR_TILE_SIZE_Q1_0 576
#define HTP_TENSOR_TILE_K_Q2_0 64
#define HTP_TENSOR_TILE_SIZE_Q2_0 576
// Tiled weight tensor layouts (Q4_0, Q8_0, IQ4_NL, MXFP4, Q1_0, Q2_0, K-quants):
// See docs/backend/snapdragon/developer.md ("Tiled Weight Formats").
struct htp_tensor_mdev_range {
uint32_t start;
uint32_t count;
};
static inline bool htp_tensor_is_extended(const struct htp_tensor * t) {
return t && (t->data >> 32) != 0;
}
static inline bool htp_tensor_is_contiguous(const struct htp_tensor * t, uint32_t type_size) {
uint32_t next_nb = type_size;
if (t->ne[0] != 1 && t->nb[0] != next_nb) {
return false;
}
next_nb *= t->ne[0];
for (int i = 1; i < HTP_OP_MAX_DIMS; i++) {
if (t->ne[i] != 1 && t->nb[i] != next_nb) {
return false;
}
next_nb *= t->ne[i];
}
return true;
}
static inline bool htp_tensor_is_permuted(const struct htp_tensor * t) {
return t->nb[0] > t->nb[1] || t->nb[1] > t->nb[2] || t->nb[2] > t->nb[3];
}
static inline bool htp_tensor_mdev_data_aligned(const struct htp_tensor * t) {
return ((uintptr_t) t->data & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0;
}
static inline bool htp_tensor_can_row_partition(const struct htp_tensor * t, uint32_t elem_size) {
if (!htp_tensor_mdev_data_aligned(t)) {
return false;
}
if (t->ne[0] != 1 && t->nb[0] != elem_size) {
return false;
}
if (htp_tensor_is_permuted(t)) {
return false;
}
if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false;
if (t->ne[2] > 1 && (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false;
if (t->ne[3] > 1 && (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false;
return true;
}
static inline bool htp_tensor_mdev_rows_per_chunk(const struct htp_tensor * t, uint32_t elem_size, uint32_t row_size, uint32_t * rows_per_chunk) {
*rows_per_chunk = 0;
if (!htp_tensor_mdev_data_aligned(t)) {
return false;
}
if (t->ne[0] != 1 && t->nb[0] != elem_size) {
return false;
}
if (htp_tensor_is_permuted(t)) {
return false;
}
if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 &&
(t->ne[2] <= 1 || (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0) &&
(t->ne[3] <= 1 || (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) {
*rows_per_chunk = 1;
return true;
}
if (t->nb[1] == row_size &&
(t->ne[2] <= 1 || t->nb[2] == t->nb[1] * t->ne[1]) &&
(t->ne[3] <= 1 || t->nb[3] == t->nb[2] * t->ne[2])) {
*rows_per_chunk = (row_size > 0) ? (HTP_TENSOR_MDEV_LINE_SIZE / hex_gcd_u32(row_size, HTP_TENSOR_MDEV_LINE_SIZE)) : 1;
return true;
}
return false;
}
static inline struct htp_tensor_mdev_range htp_tensor_mdev_partition(uint32_t total_units, uint32_t units_per_chunk, uint32_t mdev_idx, uint32_t mdev_count, const struct fastdiv_values * mdev_count_div) {
struct htp_tensor_mdev_range range = { 0, total_units };
if (mdev_count <= 1) {
return range;
}
if (units_per_chunk == 0) {
range.start = (mdev_idx == 0) ? 0 : total_units;
range.count = (mdev_idx == 0) ? total_units : 0;
return range;
}
const uint32_t total_chunks = total_units / units_per_chunk;
if (total_chunks < mdev_count) {
range.start = (mdev_idx == 0) ? 0 : total_units;
range.count = (mdev_idx == 0) ? total_units : 0;
return range;
}
const uint32_t chunks_per_mdev = fastdiv(total_chunks + mdev_count - 1, mdev_count_div);
range.start = MIN(mdev_idx * chunks_per_mdev * units_per_chunk, total_units);
if (mdev_idx == mdev_count - 1) {
range.count = total_units - range.start;
} else {
range.count = MIN(chunks_per_mdev * units_per_chunk, total_units - range.start);
}
return range;
}
static inline uint32_t htp_tensor_get_tile_size(int type) {
switch (type) {
case HTP_TYPE_Q4_0: return HTP_TENSOR_TILE_SIZE_Q4_0;
case HTP_TYPE_IQ4_NL: return HTP_TENSOR_TILE_SIZE_IQ4_NL;
case HTP_TYPE_Q4_1: return HTP_TENSOR_TILE_SIZE_Q4_1;
case HTP_TYPE_Q4_K: return HTP_TENSOR_TILE_SIZE_Q4_K;
case HTP_TYPE_Q8_0: return HTP_TENSOR_TILE_SIZE_Q8_0;
case HTP_TYPE_Q5_K: return HTP_TENSOR_TILE_SIZE_Q5_K;
case HTP_TYPE_Q6_K: return HTP_TENSOR_TILE_SIZE_Q6_K;
case HTP_TYPE_Q3_K: return HTP_TENSOR_TILE_SIZE_Q3_K;
case HTP_TYPE_Q2_K: return HTP_TENSOR_TILE_SIZE_Q2_K;
case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_SIZE_Q1_0;
case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_SIZE_Q2_0;
case HTP_TYPE_MXFP4: return HTP_TENSOR_TILE_SIZE_MXFP4;
case HTP_TYPE_F16: return HTP_TENSOR_TILE_SIZE_F16;
default: return 0;
}
}
static inline uint32_t htp_tensor_get_tile_k(int type) {
switch (type) {
case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_K_Q1_0;
case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_K_Q2_0;
default: return 32;
}
}
static inline uint32_t htp_tensor_get_tile_padded_size(int type) {
return (htp_tensor_get_tile_size(type) + 127u) & ~127u;
}
static inline bool htp_tensor_is_tiled(const struct htp_tensor * t) {
return t && (t->flags & HTP_TENSOR_TILED) != 0;
}
static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) {
switch (type) {
case HTP_TYPE_F32: return ne00 * 4;
case HTP_TYPE_F16:
case HTP_TYPE_BF16: return ne00 * 2;
case HTP_TYPE_Q8_0: return (ne00 / 32) * 34;
case HTP_TYPE_Q4_0:
case HTP_TYPE_IQ4_NL: return (ne00 / 32) * 18;
case HTP_TYPE_Q4_1: return (ne00 / 32) * 20;
case HTP_TYPE_MXFP4: return (ne00 / 32) * 17;
case HTP_TYPE_Q2_K: return (ne00 / 256) * 84;
case HTP_TYPE_Q2_0: return (ne00 / 64) * 18;
case HTP_TYPE_Q3_K: return (ne00 / 256) * 110;
case HTP_TYPE_Q4_K: return (ne00 / 256) * 144;
case HTP_TYPE_Q5_K: return (ne00 / 256) * 176;
case HTP_TYPE_Q6_K: return (ne00 / 256) * 210;
case HTP_TYPE_Q1_0: return (ne00 / HTP_TENSOR_TILE_K_Q1_0) * 18;
case HTP_TYPE_I32: return ne00 * 4;
case HTP_TYPE_I64: return ne00 * 8;
default: return 0;
}
}
#endif // HTP_TENSOR_H
llama.cpp/ggml/src/ggml-hexagon/htp/htp-vtcm.h
+19
View File
@@ -0,0 +1,19 @@
#ifndef HTP_VTCM_H
#define HTP_VTCM_H
#include <stddef.h>
#include <stdint.h>
static inline uint8_t *vtcm_seq_alloc(uint8_t **vtcm_ptr, size_t size) {
uint8_t *p = *vtcm_ptr;
*vtcm_ptr += size;
return p;
}
#define VTCM_LAYOUT_ALLOC(off, field, sz) do { (L)->field = (off); (off) += (sz); } while (0)
#define VTCM_LAYOUT_ALLOC_OPTIONAL(off, field, sz, cond) do { if (cond) { VTCM_LAYOUT_ALLOC(off, field, sz); } else { (L)->field = 0; } } while (0)
#define VTCM_LAYOUT_PTR(type, base, offset) ((type *)((uint8_t *)(base) + (offset)))
#define VTCM_LAYOUT_PTR_OPTIONAL(type, base, offset, cond) ((cond) ? VTCM_LAYOUT_PTR(type, base, offset) : NULL)
#endif // HTP_VTCM_H
llama.cpp/ggml/src/ggml-hexagon/htp/htp_iface.idl
+23
View File
@@ -0,0 +1,23 @@
// FastRPC IDL interface for GGML HTP
#ifndef HTP_IDL
#define HTP_IDL
#include "AEEStdDef.idl"
#include "remote.idl"
struct htp_iface_pmu_conf {
uint32 events[8];
};
interface htp_iface : remote_handle64 {
AEEResult start(in uint32 sess_id, in uint64 dsp_queue_id, in uint32 n_hvx, in uint32 n_hmx, in uint64 max_vmem, in uint32 power_mode);
AEEResult stop();
AEEResult mmap(in uint32 fd, in uint64 size);
AEEResult munmap(in uint32 fd);
AEEResult profiler(in uint32 mode, in htp_iface_pmu_conf pmu);
AEEResult etm(in uint32 enable);
AEEResult hwinfo(rout uint32 n_threads, rout uint32 n_hvx, rout uint32 n_hmx, rout uint64 vtcm_size);
};
#endif /* HTP_IDL */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-arith.h
+953
View File
@@ -0,0 +1,953 @@
#ifndef HVX_ARITH_H
#define HVX_ARITH_H
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include <math.h>
#include "hvx-base.h"
#include "hvx-floor.h"
#include "hex-utils.h"
//
// Binary operations (add, mul, sub)
//
#define UNUSED(x) (void)(x)
#define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \
do { \
dst_type * vdst = (dst_type *) dst; \
src0_type * vsrc0 = (src0_type *) src0; \
src1_type * vsrc1 = (src1_type *) src1; \
\
const uint32_t epv = 128 / (elem_size); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \
} \
if (nloe) { \
HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \
vec_store((void *) &vdst[i], nloe * (elem_size), v); \
} \
} while(0)
#if __HVX_ARCH__ < 79
#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b))
#define HVX_OP_SUB_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a, b))
#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b))
#else
#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b)
#define HVX_OP_SUB_F32(a, b) Q6_Vsf_vsub_VsfVsf(a, b)
#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b)
#endif
#define HVX_OP_ADD_F16(a, b) hvx_vec_add_f16_f16(a, b)
#define HVX_OP_SUB_F16(a, b) hvx_vec_sub_f16_f16(a, b)
#define HVX_OP_MUL_F16(a, b) hvx_vec_mul_f16_f16(a, b)
// Generic macro to define alignment permutations for an op
#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \
static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
DEFINE_HVX_BINARY_OP_VARIANTS(hvx_add_f32, HVX_OP_ADD_F32, float)
DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f32, HVX_OP_SUB_F32, float)
DEFINE_HVX_BINARY_OP_VARIANTS(hvx_mul_f32, HVX_OP_MUL_F32, float)
DEFINE_HVX_BINARY_OP_VARIANTS(hvx_add_f16, HVX_OP_ADD_F16, _Float16)
DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f16, HVX_OP_SUB_F16, _Float16)
DEFINE_HVX_BINARY_OP_VARIANTS(hvx_mul_f16, HVX_OP_MUL_F16, _Float16)
// Dispatcher logic
#define HVX_BINARY_DISPATCHER(OP_NAME) \
static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \
if (hex_is_aligned((void *) dst, 128)) { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \
else OP_NAME##_aau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \
else OP_NAME##_auu(dst, src0, src1, num_elems); \
} \
} else { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \
else OP_NAME##_uau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \
else OP_NAME##_uuu(dst, src0, src1, num_elems); \
} \
} \
}
HVX_BINARY_DISPATCHER(hvx_add_f32)
HVX_BINARY_DISPATCHER(hvx_sub_f32)
HVX_BINARY_DISPATCHER(hvx_mul_f32)
HVX_BINARY_DISPATCHER(hvx_add_f16)
HVX_BINARY_DISPATCHER(hvx_sub_f16)
HVX_BINARY_DISPATCHER(hvx_mul_f16)
// Mul-Mul Optimized
static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint8_t * restrict src2, const uint32_t num_elems) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src0 % 128 == 0);
assert((unsigned long) src1 % 128 == 0);
assert((unsigned long) src2 % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc0 = (HVX_Vector *) src0;
HVX_Vector * restrict vsrc1 = (HVX_Vector *) src1;
HVX_Vector * restrict vsrc2 = (HVX_Vector *) src2;
const uint32_t elem_size = sizeof(float);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = num_elems / epv;
const uint32_t nloe = num_elems % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_Vector v1 = HVX_OP_MUL_F32(vsrc0[i], vsrc1[i]);
vdst[i] = HVX_OP_MUL(v1, vsrc2[i]);
}
if (nloe) {
HVX_Vector v1 = HVX_OP_MUL_F32(vsrc0[i], vsrc1[i]);
HVX_Vector v2 = HVX_OP_MUL_F32(v1, vsrc2[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v2);
}
}
static inline void hvx_mul_mul_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint8_t * restrict src2, const uint32_t num_elems) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src0 % 128 == 0);
assert((unsigned long) src1 % 128 == 0);
assert((unsigned long) src2 % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc0 = (HVX_Vector *) src0;
HVX_Vector * restrict vsrc1 = (HVX_Vector *) src1;
HVX_Vector * restrict vsrc2 = (HVX_Vector *) src2;
const uint32_t elem_size = sizeof(_Float16);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = num_elems / epv;
const uint32_t nloe = num_elems % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]);
vdst[i] = HVX_OP_MUL_F16(v1, vsrc2[i]);
}
if (nloe) {
HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]);
HVX_Vector v2 = HVX_OP_MUL_F16(v1, vsrc2[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v2);
}
}
// Scalar Operations
#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / (elem_size); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
HVX_Vector v = vsrc[i]; \
vdst[i] = scalar_op_macro(v); \
} \
if (nloe) { \
HVX_Vector v = vsrc[i]; \
v = scalar_op_macro(v); \
vec_store((void *) &vdst[i], nloe * (elem_size), v); \
} \
} while(0)
#define HVX_OP_ADD_SCALAR_F32(v) \
({ \
const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VwVw(inf, v); \
HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \
Q6_V_vmux_QVV(pred_inf, inf, out); \
})
#define HVX_OP_MUL_SCALAR_F32(v) HVX_OP_MUL_F32(v, val_vec)
#define HVX_OP_SUB_SCALAR_F32(v) HVX_OP_SUB_F32(v, val_vec)
#define HVX_OP_ADD_SCALAR_F16(v) \
({ \
const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VhVh(inf, v); \
HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \
Q6_V_vmux_QVV(pred_inf, inf, out); \
})
#define HVX_OP_MUL_SCALAR_F16(v) HVX_OP_MUL_F16(v, val_vec)
#define HVX_OP_SUB_SCALAR_F16(v) HVX_OP_SUB_F16(v, val_vec)
// Scalar Variants
// Generic macro to define alignment permutations for an op
#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \
static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src % 128 == 0); \
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
assert((uintptr_t) dst % 128 == 0); \
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
assert((uintptr_t) src % 128 == 0); \
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_add_scalar_f32, HVX_OP_ADD_SCALAR_F32, hvx_vec_splat_f32, float)
DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f32, HVX_OP_SUB_SCALAR_F32, hvx_vec_splat_f32, float)
DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_mul_scalar_f32, HVX_OP_MUL_SCALAR_F32, hvx_vec_splat_f32, float)
DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_add_scalar_f16, HVX_OP_ADD_SCALAR_F16, hvx_vec_splat_f16, _Float16)
DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f16, HVX_OP_SUB_SCALAR_F16, hvx_vec_splat_f16, _Float16)
DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_mul_scalar_f16, HVX_OP_MUL_SCALAR_F16, hvx_vec_splat_f16, _Float16)
// Dispatcher logic
#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \
static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, const uint32_t num_elems) { \
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \
OP_NAME##_aa(dst, src, val, num_elems); \
} else if (hex_is_aligned((void *) dst, 128)) { \
OP_NAME##_au(dst, src, val, num_elems); \
} else if (hex_is_aligned((void *) src, 128)) { \
OP_NAME##_ua(dst, src, val, num_elems); \
} else { \
OP_NAME##_uu(dst, src, val, num_elems); \
} \
}
HVX_BINARY_SCALAR_DISPATCHER(hvx_add_scalar_f32, float)
HVX_BINARY_SCALAR_DISPATCHER(hvx_sub_scalar_f32, float)
HVX_BINARY_SCALAR_DISPATCHER(hvx_mul_scalar_f32, float)
HVX_BINARY_SCALAR_DISPATCHER(hvx_add_scalar_f16, _Float16)
HVX_BINARY_SCALAR_DISPATCHER(hvx_sub_scalar_f16, _Float16)
HVX_BINARY_SCALAR_DISPATCHER(hvx_mul_scalar_f16, _Float16)
// MIN Scalar variants
#define HVX_OP_MIN_SCALAR(v) Q6_Vsf_vmin_VsfVsf(val_vec, v)
static inline void hvx_min_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f32(val);
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_MIN_SCALAR);
}
static inline void hvx_min_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f32(val);
assert((unsigned long) dst % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_MIN_SCALAR);
}
static inline void hvx_min_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f32(val);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_MIN_SCALAR);
}
static inline void hvx_min_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f32(val);
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_MIN_SCALAR);
}
static inline void hvx_min_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float val, const int num_elems) {
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
hvx_min_scalar_f32_aa(dst, src, val, num_elems);
} else if (hex_is_aligned((void *) dst, 128)) {
hvx_min_scalar_f32_au(dst, src, val, num_elems);
} else if (hex_is_aligned((void *) src, 128)) {
hvx_min_scalar_f32_ua(dst, src, val, num_elems);
} else {
hvx_min_scalar_f32_uu(dst, src, val, num_elems);
}
}
// MAX Scalar variants
#define HVX_OP_MAX_SCALAR(v) Q6_Vsf_vmax_VsfVsf(val_vec, v)
static inline void hvx_max_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f32(val);
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_MAX_SCALAR);
}
static inline void hvx_max_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f32(val);
assert((unsigned long) dst % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_MAX_SCALAR);
}
static inline void hvx_max_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f32(val);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_MAX_SCALAR);
}
static inline void hvx_max_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f32(val);
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_MAX_SCALAR);
}
static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float val, const int num_elems) {
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
hvx_max_scalar_f32_aa(dst, src, val, num_elems);
} else if (hex_is_aligned((void *) dst, 128)) {
hvx_max_scalar_f32_au(dst, src, val, num_elems);
} else if (hex_is_aligned((void *) src, 128)) {
hvx_max_scalar_f32_ua(dst, src, val, num_elems);
} else {
hvx_max_scalar_f32_uu(dst, src, val, num_elems);
}
}
#define HVX_OP_MIN_SCALAR_F16(v) Q6_Vhf_vmin_VhfVhf(val_vec, v)
static inline void hvx_min_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f16(val);
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16);
}
static inline void hvx_min_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f16(val);
assert((unsigned long) dst % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16);
}
static inline void hvx_min_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f16(val);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16);
}
static inline void hvx_min_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f16(val);
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16);
}
static inline void hvx_min_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, const int num_elems) {
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
hvx_min_scalar_f16_aa(dst, src, val, num_elems);
} else if (hex_is_aligned((void *) dst, 128)) {
hvx_min_scalar_f16_au(dst, src, val, num_elems);
} else if (hex_is_aligned((void *) src, 128)) {
hvx_min_scalar_f16_ua(dst, src, val, num_elems);
} else {
hvx_min_scalar_f16_uu(dst, src, val, num_elems);
}
}
// CLAMP Scalar variants
#define HVX_OP_CLAMP_SCALAR(v) \
({ \
HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VsfVsf(v, max_vec); \
HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VsfVsf(min_vec, v); \
HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \
Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \
})
static inline void hvx_clamp_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f32(min);
const HVX_Vector max_vec = hvx_vec_splat_f32(max);
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR);
}
static inline void hvx_clamp_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f32(min);
const HVX_Vector max_vec = hvx_vec_splat_f32(max);
assert((unsigned long) dst % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR);
}
static inline void hvx_clamp_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f32(min);
const HVX_Vector max_vec = hvx_vec_splat_f32(max);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR);
}
static inline void hvx_clamp_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f32(min);
const HVX_Vector max_vec = hvx_vec_splat_f32(max);
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR);
}
static inline void hvx_clamp_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, const int num_elems) {
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
hvx_clamp_scalar_f32_aa(dst, src, min, max, num_elems);
} else if (hex_is_aligned((void *) dst, 128)) {
hvx_clamp_scalar_f32_au(dst, src, min, max, num_elems);
} else if (hex_is_aligned((void *) src, 128)) {
hvx_clamp_scalar_f32_ua(dst, src, min, max, num_elems);
} else {
hvx_clamp_scalar_f32_uu(dst, src, min, max, num_elems);
}
}
#define HVX_OP_CLAMP_SCALAR_F16(v) \
({ \
HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VhfVhf(v, max_vec); \
HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VhfVhf(min_vec, v); \
HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \
Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \
})
static inline void hvx_clamp_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f16(min);
const HVX_Vector max_vec = hvx_vec_splat_f16(max);
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16);
}
static inline void hvx_clamp_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f16(min);
const HVX_Vector max_vec = hvx_vec_splat_f16(max);
assert((unsigned long) dst % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_CLAMP_SCALAR_F16);
}
static inline void hvx_clamp_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f16(min);
const HVX_Vector max_vec = hvx_vec_splat_f16(max);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16);
}
static inline void hvx_clamp_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, uint32_t n) {
const HVX_Vector min_vec = hvx_vec_splat_f16(min);
const HVX_Vector max_vec = hvx_vec_splat_f16(max);
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_CLAMP_SCALAR_F16);
}
static inline void hvx_clamp_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 min, const _Float16 max, const int num_elems) {
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
hvx_clamp_scalar_f16_aa(dst, src, min, max, num_elems);
} else if (hex_is_aligned((void *) dst, 128)) {
hvx_clamp_scalar_f16_au(dst, src, min, max, num_elems);
} else if (hex_is_aligned((void *) src, 128)) {
hvx_clamp_scalar_f16_ua(dst, src, min, max, num_elems);
} else {
hvx_clamp_scalar_f16_uu(dst, src, min, max, num_elems);
}
}
#define HVX_OP_LEAKY_RELU_SCALAR(v) \
({ \
HVX_VectorPred pred_neg = Q6_Q_vcmp_gt_VsfVsf(zero_vec, v); \
HVX_Vector scaled = HVX_OP_MUL_F32(v, ns_vec); \
Q6_V_vmux_QVV(pred_neg, scaled, v); \
})
static inline void hvx_leaky_relu_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) {
const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f);
const HVX_Vector ns_vec = hvx_vec_splat_f32(ns);
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(float), hvx_vec_store_a, HVX_OP_LEAKY_RELU_SCALAR);
}
static inline void hvx_leaky_relu_scalar_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) {
const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f);
const HVX_Vector ns_vec = hvx_vec_splat_f32(ns);
assert((unsigned long) dst % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(float), hvx_vec_store_a, HVX_OP_LEAKY_RELU_SCALAR);
}
static inline void hvx_leaky_relu_scalar_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) {
const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f);
const HVX_Vector ns_vec = hvx_vec_splat_f32(ns);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(float), hvx_vec_store_u, HVX_OP_LEAKY_RELU_SCALAR);
}
static inline void hvx_leaky_relu_scalar_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, uint32_t n) {
const HVX_Vector zero_vec = hvx_vec_splat_f32(0.0f);
const HVX_Vector ns_vec = hvx_vec_splat_f32(ns);
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(float), hvx_vec_store_u, HVX_OP_LEAKY_RELU_SCALAR);
}
static inline void hvx_leaky_relu_scalar_f32(uint8_t * restrict dst, const uint8_t * restrict src, const float ns, const int num_elems) {
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
hvx_leaky_relu_scalar_f32_aa(dst, src, ns, num_elems);
} else if (hex_is_aligned((void *) dst, 128)) {
hvx_leaky_relu_scalar_f32_au(dst, src, ns, num_elems);
} else if (hex_is_aligned((void *) src, 128)) {
hvx_leaky_relu_scalar_f32_ua(dst, src, ns, num_elems);
} else {
hvx_leaky_relu_scalar_f32_uu(dst, src, ns, num_elems);
}
}
//
// Abs
//
static inline void hvx_abs_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(float);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
vdst[i] = hvx_vec_abs_f32(vsrc[i]);
}
if (nloe) {
HVX_Vector v = hvx_vec_abs_f32(vsrc[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
#define hvx_abs_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(_Float16); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_abs_f16(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = hvx_vec_abs_f16(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
static inline void hvx_abs_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_abs_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_abs_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_abs_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_abs_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_abs_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_abs_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_abs_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_abs_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) {
if (hex_is_aligned((void *) dst, 128)) {
if (hex_is_aligned((void *) src, 128)) {
hvx_abs_f16_aa(dst, src, num_elems);
} else {
hvx_abs_f16_au(dst, src, num_elems);
}
} else {
if (hex_is_aligned((void *) src, 128)) {
hvx_abs_f16_ua(dst, src, num_elems);
} else {
hvx_abs_f16_uu(dst, src, num_elems);
}
}
}
//
// Step
//
static inline void hvx_step_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(float);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
vdst[i] = hvx_vec_step_f32(vsrc[i]);
}
if (nloe) {
HVX_Vector v = hvx_vec_step_f32(vsrc[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
static inline void hvx_sgn_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(float);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
vdst[i] = hvx_vec_sgn_f32(vsrc[i]);
}
if (nloe) {
HVX_Vector v = hvx_vec_sgn_f32(vsrc[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
static inline void hvx_sgn_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(_Float16);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
vdst[i] = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p)));
}
if (nloe) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_sgn_f32(Q6_V_lo_W(p)), hvx_vec_sgn_f32(Q6_V_hi_W(p)));
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
static inline void hvx_step_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(_Float16);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
vdst[i] = hvx_vec_step_f16(vsrc[i]);
}
if (nloe) {
HVX_Vector v = hvx_vec_step_f16(vsrc[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
//
// Ceil
//
static inline void hvx_ceil_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(float);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
vdst[i] = hvx_vec_ceil_f32(vsrc[i]);
}
if (nloe) {
HVX_Vector v = hvx_vec_ceil_f32(vsrc[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
static inline void hvx_ceil_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(_Float16);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
vdst[i] = hvx_vec_f32_to_f16(hvx_vec_ceil_f32(Q6_V_lo_W(p)), hvx_vec_ceil_f32(Q6_V_hi_W(p)));
}
if (nloe) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
HVX_Vector v = hvx_vec_f32_to_f16(hvx_vec_ceil_f32(Q6_V_lo_W(p)), hvx_vec_ceil_f32(Q6_V_hi_W(p)));
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
//
// Square
//
#define hvx_sqr_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(float); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = HVX_OP_MUL_F32(vsrc[i], vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
static inline void hvx_sqr_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_sqr_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_sqr_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_sqr_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_sqr_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_sqr_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_sqr_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_sqr_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_sqr_f32(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) {
if (hex_is_aligned((void *) dst, 128)) {
if (hex_is_aligned((void *) src, 128)) {
hvx_sqr_f32_aa(dst, src, num_elems);
} else {
hvx_sqr_f32_au(dst, src, num_elems);
}
} else {
if (hex_is_aligned((void *) src, 128)) {
hvx_sqr_f32_ua(dst, src, num_elems);
} else {
hvx_sqr_f32_uu(dst, src, num_elems);
}
}
}
#define hvx_sqr_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(_Float16); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = HVX_OP_MUL_F16(vsrc[i], vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
static inline void hvx_sqr_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_sqr_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_sqr_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_sqr_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_sqr_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_sqr_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_sqr_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_sqr_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) {
if (hex_is_aligned((void *) dst, 128)) {
if (hex_is_aligned((void *) src, 128)) {
hvx_sqr_f16_aa(dst, src, num_elems);
} else {
hvx_sqr_f16_au(dst, src, num_elems);
}
} else {
if (hex_is_aligned((void *) src, 128)) {
hvx_sqr_f16_ua(dst, src, num_elems);
} else {
hvx_sqr_f16_uu(dst, src, num_elems);
}
}
}
#undef HVX_OP_ADD_F32
#undef HVX_OP_SUB_F32
#undef HVX_OP_MUL_F32
#undef HVX_OP_ADD_F16
#undef HVX_OP_SUB_F16
#undef HVX_OP_MUL_F16
#undef hvx_arith_loop_body
#undef HVX_OP_ADD_SCALAR_F32
#undef HVX_OP_SUB_SCALAR_F32
#undef HVX_OP_MUL_SCALAR_F32
#undef HVX_OP_ADD_SCALAR_F16
#undef HVX_OP_SUB_SCALAR_F16
#undef HVX_OP_MUL_SCALAR_F16
#undef hvx_scalar_loop_body
#undef HVX_OP_MIN_SCALAR
#undef HVX_OP_MAX_SCALAR
#undef HVX_OP_MIN_SCALAR_F16
#undef HVX_OP_CLAMP_SCALAR
#undef HVX_OP_CLAMP_SCALAR_F16
#undef HVX_OP_LEAKY_RELU_SCALAR
#undef DEFINE_HVX_BINARY_OP_VARIANTS
#undef DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS
#undef HVX_BINARY_DISPATCHER
#undef UNUSED
#endif // HVX_ARITH_H
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-base.h
+450
View File
@@ -0,0 +1,450 @@
#ifndef HVX_BASE_H
#define HVX_BASE_H
#include <stdbool.h>
#include <stdint.h>
#include <math.h>
#include <assert.h>
#include "hex-utils.h"
#include "hvx-types.h"
#define hvx_vmem(A) *((HVX_Vector *)(A))
#define hvx_vmemu(A) *((HVX_UVector *)(A))
static inline void hvx_vec_store_u(void * restrict dst, uint32_t n, HVX_Vector v) {
// Rotate as needed.
v = Q6_V_vlalign_VVR(v, v, (size_t) dst);
uint32_t left_off = (size_t) dst & 127;
uint32_t right_off = left_off + n;
HVX_VectorPred ql_not = Q6_Q_vsetq_R((size_t) dst);
HVX_VectorPred qr = Q6_Q_vsetq2_R(right_off);
if (right_off > 128) {
Q6_vmem_QRIV(qr, (HVX_Vector *) dst + 1, v);
// all 1's
qr = Q6_Q_vcmp_eq_VbVb(v, v);
}
ql_not = Q6_Q_or_QQn(ql_not, qr);
Q6_vmem_QnRIV(ql_not, (HVX_Vector *) dst, v);
}
static inline void hvx_vec_store_a(void * restrict dst, uint32_t n, HVX_Vector v) {
assert((unsigned long) dst % 128 == 0);
HVX_VectorPred m = Q6_Q_or_QQn(Q6_Q_vsetq_R((unsigned long) dst), Q6_Q_vsetq2_R(n));
Q6_vmem_QnRIV(m, (HVX_Vector *) dst, v);
}
static inline HVX_Vector hvx_vec_splat_f32(float v) {
union { float f; uint32_t i; } u = { .f = v };
return Q6_V_vsplat_R(u.i);
}
static inline HVX_Vector hvx_vec_splat_f16(_Float16 v) {
union { __fp16 f; uint16_t i; } u = { .f = v };
return Q6_Vh_vsplat_R(u.i);
}
static inline HVX_Vector hvx_vec_repl4(HVX_Vector v) {
// vdelta control to replicate first 4 bytes across all elements
static const uint8_t __attribute__((aligned(128))) repl[128] = {
0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
};
HVX_Vector ctrl = *(HVX_Vector *) repl;
return Q6_V_vdelta_VV(v, ctrl);
}
static inline float hvx_vec_get_f32(HVX_Vector v) {
float __attribute__((aligned(128))) x;
hvx_vec_store_a(&x, 4, v);
return x;
}
static inline int32_t hvx_vec_get_i32(HVX_Vector v) {
int32_t __attribute__((aligned(128))) x;
hvx_vec_store_a(&x, 4, v);
return x;
}
static inline _Float16 hvx_vec_get_f16(HVX_Vector v) {
_Float16 __attribute__((aligned(128))) x;
hvx_vec_store_a(&x, 2, v);
return x;
}
static inline HVX_Vector hvx_vec_abs_f16(HVX_Vector v) {
// abs by clearing the fp16 sign bit
HVX_Vector mask = Q6_Vh_vsplat_R(0x7fff);
return Q6_V_vand_VV(v, mask);
}
static inline HVX_Vector hvx_vec_neg_f16(HVX_Vector v) {
// neg by setting the fp16 sign bit
HVX_Vector mask = Q6_Vh_vsplat_R(0x8000);
return Q6_V_vxor_VV(v, mask);
}
static inline HVX_Vector hvx_vec_abs_f32(HVX_Vector v) {
// abs by clearing the fp32 sign bit
HVX_Vector mask = Q6_V_vsplat_R(0x7fffffff);
return Q6_V_vand_VV(v, mask);
}
static inline HVX_Vector hvx_vec_neg_f32(HVX_Vector v) {
#if __HVX_ARCH__ > 75
return Q6_Vsf_vfneg_Vsf(v);
#else
// neg by setting the fp32 sign bit
HVX_Vector mask = Q6_V_vsplat_R(0x80000000);
return Q6_V_vxor_VV(v, mask);
#endif // __HVX_ARCH__ > 75
}
static inline HVX_Vector hvx_vec_step_f32(HVX_Vector v) {
const HVX_Vector zero = Q6_V_vzero();
const HVX_Vector one = hvx_vec_splat_f32(1.0f);
HVX_VectorPred q = Q6_Q_vcmp_gt_VsfVsf(v, zero);
return Q6_V_vmux_QVV(q, one, zero);
}
static inline HVX_Vector hvx_vec_sgn_f32(HVX_Vector v) {
const HVX_Vector zero = Q6_V_vzero();
const HVX_Vector one = hvx_vec_splat_f32(1.0f);
const HVX_Vector sign_mask = Q6_V_vsplat_R(0x80000000);
const HVX_Vector mag = hvx_vec_abs_f32(v);
const HVX_Vector signed_one = Q6_V_vor_VV(one, Q6_V_vand_VV(v, sign_mask));
// The ordered magnitude comparison maps NaN and both signed zeros to zero.
const HVX_VectorPred nonzero = Q6_Q_vcmp_gt_VsfVsf(mag, zero);
return Q6_V_vmux_QVV(nonzero, signed_one, zero);
}
static inline HVX_Vector hvx_vec_step_f16(HVX_Vector v) {
const HVX_Vector zero = Q6_V_vzero();
const HVX_Vector one = hvx_vec_splat_f16((_Float16) 1.0f);
HVX_VectorPred q = Q6_Q_vcmp_gt_VhfVhf(v, zero);
return Q6_V_vmux_QVV(q, one, zero);
}
static inline HVX_VectorPred hvx_vec_is_nan_f16(HVX_Vector v) {
const HVX_Vector vnan_exp = Q6_Vh_vsplat_R(0x7C00);
const HVX_Vector vnan_frac = Q6_Vh_vsplat_R(0x7FFF);
// get pred of which are NaN, i.e., exponent bits all 1s and fraction bits non 0s
HVX_VectorPred p_exp = Q6_Q_vcmp_eq_VhVh(Q6_V_vand_VV(v, vnan_exp), vnan_exp);
HVX_VectorPred p_frac = Q6_Q_not_Q(Q6_Q_vcmp_eq_VhVh(Q6_V_vand_VV(v, vnan_frac), vnan_exp));
return Q6_Q_and_QQ(p_exp, p_frac);
}
static inline HVX_Vector hvx_vec_f32_to_f16_shuff(HVX_Vector v0, HVX_Vector v1) {
#if __HVX_ARCH__ >= 81
HVX_Vector q0 = Q6_Vqf32_equals_Vsf(v0);
HVX_Vector q1 = Q6_Vqf32_equals_Vsf(v1);
#else
const HVX_Vector zero = Q6_V_vzero();
HVX_Vector q0 = Q6_Vqf32_vadd_VsfVsf(v0, zero);
HVX_Vector q1 = Q6_Vqf32_vadd_VsfVsf(v1, zero);
#endif
return Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(q1, q0));
}
static inline HVX_Vector hvx_vec_f32_to_f16(HVX_Vector v0, HVX_Vector v1) {
return Q6_Vh_vdeal_Vh(hvx_vec_f32_to_f16_shuff(v0, v1));
}
#if __HVX_ARCH__ >= 79
static inline HVX_VectorPair hvx_vec_f16_to_f32_shuff(HVX_Vector v) {
const HVX_Vector one = hvx_vec_splat_f16(1.0);
HVX_VectorPair p = Q6_Wsf_vmpy_VhfVhf(v, one);
return Q6_W_vcombine_VV(Q6_V_hi_W(p), Q6_V_lo_W(p));
}
static inline HVX_VectorPair hvx_vec_f16_to_f32(HVX_Vector v) {
const HVX_Vector one = hvx_vec_splat_f16(1.0);
HVX_VectorPair p = Q6_Wsf_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(v), one);
return Q6_W_vcombine_VV(Q6_V_hi_W(p), Q6_V_lo_W(p));
}
#else
static inline HVX_VectorPair hvx_vec_f16_to_f32_shuff(HVX_Vector v) {
const HVX_Vector one = hvx_vec_splat_f16(1.0);
HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(v, one);
return Q6_W_vcombine_VV(Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)), Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)));
}
static inline HVX_VectorPair hvx_vec_f16_to_f32(HVX_Vector v) {
const HVX_Vector one = hvx_vec_splat_f16(1.0);
HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(v), one);
return Q6_W_vcombine_VV(Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)), Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)));
}
#endif
static inline HVX_Vector hvx_vec_i16_from_hf_rnd_sat(HVX_Vector vin) {
// This looks complicated.
// Ideally should just be Q6_Vh_equals_Vhf(vin)
// but that instruction does not do proper rounding.
// convert to qf32, multiplying by 1.0 in the process.
HVX_VectorPair v32 = Q6_Wqf32_vmpy_VhfVhf(vin, Q6_Vh_vsplat_R(0x3C00));
// 'in-range' values are +/32752.
// add 192K to it, convert to sf
HVX_Vector v192K = Q6_V_vsplat_R(0x48400000);
HVX_Vector vsf_0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_V_lo_W(v32), v192K));
HVX_Vector vsf_1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_V_hi_W(v32), v192K));
// for in-range cases, result is {163858... 229360} so the exponent is always 144.
// if we extract bits 21..0 as a signed quantity, and round 6 bits off, that will be the answer.
// Start by <<10 to get the final 'sign' bit in bit 15...
vsf_0 = Q6_Vw_vasl_VwR(vsf_0, 10);
vsf_1 = Q6_Vw_vasl_VwR(vsf_1, 10);
// now round down to 16
return Q6_Vh_vround_VwVw_sat(vsf_1, vsf_0);
}
#if __HVX_ARCH__ < 79
static inline HVX_VectorPair hvx_vec_mpyacc_f32_f16(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y)
{
HVX_VectorPair m = Q6_Wqf32_vmpy_VhfVhf(x, y);
HVX_Vector a0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_V_lo_W(m), Q6_V_lo_W(acc)));
HVX_Vector a1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_V_hi_W(m), Q6_V_hi_W(acc)));
return Q6_W_vcombine_VV(a1, a0);
}
#else
static inline HVX_VectorPair hvx_vec_mpyacc_f32_f16(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y)
{
return Q6_Wsf_vmpyacc_WsfVhfVhf(acc, x, y);
}
#endif
#if __HVX_ARCH__ < 79
static inline HVX_Vector hvx_vec_add_f16_f16(HVX_Vector a, HVX_Vector b)
{
const HVX_Vector negone = Q6_Vh_vsplat_R(0xBC00); // -1.0 in IEEE FP16
const HVX_Vector one = Q6_Vh_vsplat_R(0x3C00); // 1.0 in IEEE FP16
HVX_VectorPair a_p = Q6_Wqf32_vmpy_VhfVhf(a, one);
HVX_VectorPair b_p = Q6_Wqf32_vmpy_VhfVhf(b, negone);
HVX_Vector a0 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_lo_W(a_p), Q6_V_lo_W(b_p));
HVX_Vector a1 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_V_hi_W(a_p), Q6_V_hi_W(b_p));
return Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(a1, a0));
}
static inline HVX_Vector hvx_vec_sub_f16_f16(HVX_Vector a, HVX_Vector b)
{
const HVX_Vector negone = Q6_Vh_vsplat_R(0xBC00); // -1.0 in IEEE FP16
const HVX_Vector one = Q6_Vh_vsplat_R(0x3C00); // 1.0 in IEEE FP16
HVX_VectorPair a_p = Q6_Wqf32_vmpy_VhfVhf(a, one);
HVX_VectorPair b_p = Q6_Wqf32_vmpy_VhfVhf(b, negone);
HVX_Vector a0 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(a_p), Q6_V_lo_W(b_p));
HVX_Vector a1 = Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_hi_W(a_p), Q6_V_hi_W(b_p));
return Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(a1, a0));
}
static inline HVX_Vector hvx_vec_mul_f16_f16(HVX_Vector a, HVX_Vector b)
{
return Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(a, b));
}
static inline HVX_Vector hvx_vec_add_f32_f32(HVX_Vector a, HVX_Vector b) {
return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b));
}
static inline HVX_Vector hvx_vec_sub_f32_f32(HVX_Vector a, HVX_Vector b) {
return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a, b));
}
static inline HVX_Vector hvx_vec_mul_f32_f32(HVX_Vector a, HVX_Vector b) {
return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b));
}
#else
static inline HVX_Vector hvx_vec_add_f16_f16(HVX_Vector a, HVX_Vector b)
{
return Q6_Vhf_vadd_VhfVhf(a, b);
}
static inline HVX_Vector hvx_vec_sub_f16_f16(HVX_Vector a, HVX_Vector b)
{
return Q6_Vhf_vsub_VhfVhf(a, b);
}
static inline HVX_Vector hvx_vec_mul_f16_f16(HVX_Vector a, HVX_Vector b)
{
return Q6_Vhf_vmpy_VhfVhf(a, b);
}
static inline HVX_Vector hvx_vec_add_f32_f32(HVX_Vector a, HVX_Vector b) {
return Q6_Vsf_vadd_VsfVsf(a, b);
}
static inline HVX_Vector hvx_vec_sub_f32_f32(HVX_Vector a, HVX_Vector b) {
return Q6_Vsf_vsub_VsfVsf(a, b);
}
static inline HVX_Vector hvx_vec_mul_f32_f32(HVX_Vector a, HVX_Vector b) {
return Q6_Vsf_vmpy_VsfVsf(a, b);
}
#endif // __HVX_ARCH__ < 79
static inline HVX_Vector hvx_vec_load_act_tile(const uint8_t * y_q, uint32_t kt, HVX_Vector * v_act_all) {
if (kt % 4 == 0) {
*v_act_all = hvx_vmem(y_q + kt * 32);
return *v_act_all;
} else if (kt % 4 == 1) {
return Q6_V_vror_VR(*v_act_all, 32);
} else if (kt % 4 == 2) {
return Q6_V_vror_VR(*v_act_all, 64);
} else {
return Q6_V_vror_VR(*v_act_all, 96);
}
}
// Full tile, in place. tmp: 32 vectors scratch.
// Use when result must stay in m. Slowest (spills).
static inline void hvx_transpose_32x32_w_shuff_inplace(HVX_Vector * restrict m, HVX_Vector * restrict tmp) {
for (int i = 0; i < 16; ++i) {
HVX_VectorPair p = Q6_W_vshuff_VVR(m[2*i + 1], m[2*i], -4);
tmp[2*i + 0] = Q6_V_lo_W(p);
tmp[2*i + 1] = Q6_V_hi_W(p);
}
for (int b = 0; b < 32; b += 4) {
HVX_VectorPair p0 = Q6_W_vshuff_VVR(tmp[b + 2], tmp[b + 0], -8);
HVX_VectorPair p1 = Q6_W_vshuff_VVR(tmp[b + 3], tmp[b + 1], -8);
m[b + 0] = Q6_V_lo_W(p0); m[b + 1] = Q6_V_hi_W(p0);
m[b + 2] = Q6_V_lo_W(p1); m[b + 3] = Q6_V_hi_W(p1);
}
for (int b = 0; b < 32; b += 8) {
for (int i = 0; i < 4; ++i) {
HVX_VectorPair p = Q6_W_vshuff_VVR(m[b + i + 4], m[b + i], -16);
tmp[b + 2*i + 0] = Q6_V_lo_W(p);
tmp[b + 2*i + 1] = Q6_V_hi_W(p);
}
}
for (int b = 0; b < 32; b += 16) {
for (int i = 0; i < 8; ++i) {
HVX_VectorPair p = Q6_W_vshuff_VVR(tmp[b + i + 8], tmp[b + i], -32);
m[b + 2*i + 0] = Q6_V_lo_W(p);
m[b + 2*i + 1] = Q6_V_hi_W(p);
}
}
for (int i = 0; i < 16; ++i) {
HVX_VectorPair p = Q6_W_vshuff_VVR(m[i + 16], m[i], -64);
tmp[2 * i + 0] = Q6_V_lo_W(p);
tmp[2 * i + 1] = Q6_V_hi_W(p);
}
for (int i = 0; i < 32; ++i) {
m[i] = tmp[i];
}
}
// Rows [0, nrows) of the transpose of src[0, ncols) to dst. src is clobbered, must not alias dst.
// Use when not in VTCM or for full tiles.
static inline void hvx_transpose_32x32_w_shuff(uint8_t * restrict dst, uint32_t dst_stride, HVX_Vector * restrict src,
uint32_t nrows, uint32_t ncols) {
for (uint32_t g = 0; g < ncols; g += 8) {
HVX_Vector * s = src + g;
HVX_VectorPair a0 = Q6_W_vshuff_VVR(s[1], s[0], -4);
HVX_VectorPair a1 = Q6_W_vshuff_VVR(s[3], s[2], -4);
HVX_VectorPair a2 = Q6_W_vshuff_VVR(s[5], s[4], -4);
HVX_VectorPair a3 = Q6_W_vshuff_VVR(s[7], s[6], -4);
HVX_VectorPair b0 = Q6_W_vshuff_VVR(Q6_V_lo_W(a1), Q6_V_lo_W(a0), -8);
HVX_VectorPair b1 = Q6_W_vshuff_VVR(Q6_V_hi_W(a1), Q6_V_hi_W(a0), -8);
HVX_VectorPair b2 = Q6_W_vshuff_VVR(Q6_V_lo_W(a3), Q6_V_lo_W(a2), -8);
HVX_VectorPair b3 = Q6_W_vshuff_VVR(Q6_V_hi_W(a3), Q6_V_hi_W(a2), -8);
HVX_VectorPair c0 = Q6_W_vshuff_VVR(Q6_V_lo_W(b2), Q6_V_lo_W(b0), -16);
HVX_VectorPair c1 = Q6_W_vshuff_VVR(Q6_V_hi_W(b2), Q6_V_hi_W(b0), -16);
HVX_VectorPair c2 = Q6_W_vshuff_VVR(Q6_V_lo_W(b3), Q6_V_lo_W(b1), -16);
HVX_VectorPair c3 = Q6_W_vshuff_VVR(Q6_V_hi_W(b3), Q6_V_hi_W(b1), -16);
s[0] = Q6_V_lo_W(c0); s[1] = Q6_V_hi_W(c0);
s[2] = Q6_V_lo_W(c1); s[3] = Q6_V_hi_W(c1);
s[4] = Q6_V_lo_W(c2); s[5] = Q6_V_hi_W(c2);
s[6] = Q6_V_lo_W(c3); s[7] = Q6_V_hi_W(c3);
}
for (uint32_t k = 0; 4 * k < nrows; k++) {
HVX_VectorPair d0 = Q6_W_vshuff_VVR(src[k + 8], src[k], -32);
HVX_VectorPair d1 = Q6_W_vshuff_VVR(src[k + 24], src[k + 16], -32);
HVX_VectorPair e0 = Q6_W_vshuff_VVR(Q6_V_lo_W(d1), Q6_V_lo_W(d0), -64);
HVX_VectorPair e1 = Q6_W_vshuff_VVR(Q6_V_hi_W(d1), Q6_V_hi_W(d0), -64);
uint8_t * d = dst + 4 * k * dst_stride;
*(HVX_Vector *) (d + 0 * dst_stride) = Q6_V_lo_W(e0);
*(HVX_Vector *) (d + 1 * dst_stride) = Q6_V_hi_W(e0);
*(HVX_Vector *) (d + 2 * dst_stride) = Q6_V_lo_W(e1);
*(HVX_Vector *) (d + 3 * dst_stride) = Q6_V_hi_W(e1);
}
}
// Specialized store that will follow outstanding scatters or gathers to make sure they complete
static inline void hvx_gather_sync(void * vtcm_addr) {
asm volatile("vmem(%0+#0):scatter_release\n"
"v0 = vmem(%0+#0)\n" :: "r"(vtcm_addr) : "v0", "memory");
}
// Lane k = k * stride. Offsets for the _gather transposes.
static inline HVX_Vector hvx_vec_gather_offsets_w(uint32_t stride) {
int32_t offsets[32] __attribute__((aligned(128)));
for (int k = 0; k < 32; k++) {
offsets[k] = k * stride;
}
return *(HVX_Vector *) offsets;
}
static inline HVX_Vector hvx_vec_gather_offsets_h(uint32_t stride) {
int16_t offsets[64] __attribute__((aligned(128)));
for (int k = 0; k < 64; k++) {
offsets[k] = k * stride;
}
return *(HVX_Vector *) offsets;
}
// Rows [0, nrows) of the transpose of ncols src vectors (src_stride apart) to dst.
// src, dst in VTCM. Async: hvx_gather_sync() before use.
// Use for partial tiles (small ncols). Fewest packets.
static inline void hvx_transpose_32x32_w_gather(uint8_t * restrict dst, uint32_t dst_stride, const uint8_t * restrict src,
uint32_t src_stride, HVX_Vector offsets, uint32_t nrows, uint32_t ncols) {
const uint32_t mu = ncols * src_stride;
#pragma unroll(4)
for (uint32_t r = 0; r < nrows; r++) {
Q6_vgather_ARMVw((HVX_Vector *) (dst + r * dst_stride), (size_t) (src + r * sizeof(int32_t)), mu, offsets);
}
}
// 16-bit version of hvx_transpose_32x32_w_gather.
static inline void hvx_transpose_64x64_h_gather(uint8_t * restrict dst, uint32_t dst_stride, const uint8_t * restrict src,
uint32_t src_stride, HVX_Vector offsets, uint32_t nrows, uint32_t ncols) {
const uint32_t mu = ncols * src_stride;
#pragma unroll(4)
for (uint32_t r = 0; r < nrows; r++) {
Q6_vgather_ARMVh((HVX_Vector *) (dst + r * dst_stride), (size_t) (src + r * sizeof(int16_t)), mu, offsets);
}
}
#endif /* HVX_BASE_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-copy.h
+332
View File
@@ -0,0 +1,332 @@
#ifndef HVX_COPY_H
#define HVX_COPY_H
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include "hvx-base.h"
#define hvx_splat_pragma(x) _Pragma(#x)
#define hvx_splat_loop_body(dst_type, vec_store, unroll_cnt) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
\
uint32_t nvec = n / (128 / elem_size); \
uint32_t nloe = n % (128 / elem_size); \
\
uint32_t i = 0; \
\
hvx_splat_pragma(unroll(unroll_cnt)) \
for (; i < nvec; i++) { \
vdst[i] = src; \
} \
if (nloe) { \
vec_store((void *) &vdst[i], nloe * elem_size, src); \
} \
} while(0)
static inline void hvx_splat_a(void * restrict dst, HVX_Vector src, uint32_t n, uint32_t elem_size) {
assert((unsigned long) dst % 128 == 0);
hvx_splat_loop_body(HVX_Vector, hvx_vec_store_a, 4);
}
static inline void hvx_splat_u(void * restrict dst, HVX_Vector src, uint32_t n, uint32_t elem_size) {
hvx_splat_loop_body(HVX_UVector, hvx_vec_store_u, 4);
}
static inline void hvx_splat_f32_a(void * restrict dst, float v, uint32_t n) {
hvx_splat_a(dst, hvx_vec_splat_f32(v), n, sizeof(float));
}
static inline void hvx_splat_f32_u(void * restrict dst, float v, uint32_t n) {
hvx_splat_u(dst, hvx_vec_splat_f32(v), n, sizeof(float));
}
static inline void hvx_splat_f16_a(void * restrict dst, _Float16 v, uint32_t n) {
hvx_splat_a(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16));
}
static inline void hvx_splat_f16_u(void * restrict dst, _Float16 v, uint32_t n) {
hvx_splat_u(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16));
}
static inline void hvx_splat_u16_a(void * restrict dst, uint16_t v, uint32_t n) {
hvx_splat_a(dst, Q6_Vh_vsplat_R(v), n, sizeof(uint16_t));
}
static inline void hvx_splat_u16_u(void * restrict dst, uint16_t v, uint32_t n) {
hvx_splat_u(dst, Q6_Vh_vsplat_R(v), n, sizeof(uint16_t));
}
static inline void hvx_splat_u8_a(void * restrict dst, uint8_t v, uint32_t n) {
hvx_splat_a(dst, Q6_Vb_vsplat_R(v), n, 1);
}
static inline void hvx_splat_u8_u(void * restrict dst, uint8_t v, uint32_t n) {
hvx_splat_u(dst, Q6_Vb_vsplat_R(v), n, 1);
}
#define hvx_copy_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { vdst[i] = vsrc[i]; } \
if (nloe) { \
vec_store((void *) &vdst[i], nloe * elem_size, vsrc[i]); \
} \
} while(0)
// Generic copy routines
static inline void hvx_copy_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n, uint32_t elem_size) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_copy_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_copy_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n, uint32_t elem_size) {
assert((unsigned long) dst % 128 == 0);
hvx_copy_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_copy_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n, uint32_t elem_size) {
assert((unsigned long) src % 128 == 0);
hvx_copy_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_copy_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n, uint32_t elem_size) {
hvx_copy_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
// copy n fp16 elements : destination and source are aligned to HVX Vector (128)
static inline void hvx_copy_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_aa(dst, src, n, sizeof(__fp16));
}
// copy n fp16 elements : destination is aligned, source is unaligned
static inline void hvx_copy_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_au(dst, src, n, sizeof(__fp16));
}
// copy n fp16 elements : destination is unaligned, source is aligned
static inline void hvx_copy_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_ua(dst, src, n, sizeof(__fp16));
}
// copy n fp16 elements : destination and source are unaligned
static inline void hvx_copy_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_uu(dst, src, n, sizeof(__fp16));
}
// copy n fp32 elements : destination and source are aligned to HVX Vector (128)
static inline void hvx_copy_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_aa(dst, src, n, sizeof(float));
}
// copy n fp32 elements : destination is aligned, source is unaligned
static inline void hvx_copy_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_au(dst, src, n, sizeof(float));
}
// copy n fp32 elements : destination is unaligned, source is aligned
static inline void hvx_copy_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_ua(dst, src, n, sizeof(float));
}
// copy n fp32 elements : destination and source are unaligned
static inline void hvx_copy_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_uu(dst, src, n, sizeof(float));
}
//// fp32 -> fp16
#define hvx_copy_f16_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(__fp16); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_f32_to_f16(vsrc[i*2+0], vsrc[i*2+1]); \
} \
if (nloe) { \
HVX_Vector v = hvx_vec_f32_to_f16(vsrc[i*2+0], vsrc[i*2+1]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
// copy/convert n fp32 elements into n fp16 elements : destination and source are aligned
static inline void hvx_copy_f16_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
// copy/convert n fp32 elements into n fp16 elements : destination is aligned, source is unaligned
static inline void hvx_copy_f16_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
// copy/convert n fp32 elements into n fp16 elements : destination is unaligned, source is aligned
static inline void hvx_copy_f16_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
// copy/convert n fp32 elements into n fp16 elements : destination and source are unaligned
static inline void hvx_copy_f16_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
//// fp16 -> fp32
#define hvx_copy_f32_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const HVX_Vector one = hvx_vec_splat_f16(1.0); \
\
const uint32_t elem_size = sizeof(__fp16); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (i = 0; i < nvec; ++i) { \
HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vsrc[i]), one); \
vdst[i*2] = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(p)); \
vdst[i*2+1] = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(p)); \
} \
\
if (nloe) { \
HVX_VectorPair p = Q6_Wqf32_vmpy_VhfVhf(Q6_Vh_vshuff_Vh(vsrc[i]), one); \
\
HVX_Vector vd = Q6_V_lo_W(p); \
i = 2 * i; \
\
if (nloe >= 32) { \
vdst[i] = Q6_Vsf_equals_Vqf32(vd); \
nloe -= 32; ++i; vd = Q6_V_hi_W(p); \
} \
\
if (nloe) { \
vd = Q6_Vsf_equals_Vqf32(vd); \
hvx_vec_store_u(&vdst[i], nloe * sizeof(float), vd); \
} \
} \
} while(0)
// copy/convert n fp16 elements into n fp32 elements : destination and source are aligned
static inline void hvx_copy_f32_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
// copy/convert n fp16 elements into n fp32 elements : destination is aligned, source is unaligned
static inline void hvx_copy_f32_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
// copy/convert n fp16 elements into n fp32 elements : destination is unaligned, source is aligned
static inline void hvx_copy_f32_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
// copy/convert n fp16 elements into n fp32 elements : destination and source are unaligned
static inline void hvx_copy_f32_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
//// fp32 -> int32
#define hvx_copy_i32_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(int32_t); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = Q6_Vw_equals_Vsf(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = Q6_Vw_equals_Vsf(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
// copy/convert n fp32 elements into n int32 elements : destination and source are aligned
static inline void hvx_copy_i32_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_copy_i32_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
// copy/convert n fp32 elements into n int32 elements : destination and source are unaligned
static inline void hvx_copy_i32_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_i32_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
//// int32 -> fp32
#define hvx_copy_f32_i32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(float); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = Q6_Vsf_equals_Vw(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = Q6_Vsf_equals_Vw(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
// copy/convert n int32 elements into n fp32 elements : destination and source are aligned
static inline void hvx_copy_f32_i32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_copy_f32_i32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
// copy/convert n int32 elements into n fp32 elements : destination and source are unaligned
static inline void hvx_copy_f32_i32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_f32_i32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
#endif // HVX_COPY_H
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-div.h
+291
View File
@@ -0,0 +1,291 @@
#ifndef HVX_DIV_H
#define HVX_DIV_H
#include <HAP_farf.h>
#include <math.h>
#include <string.h>
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include "hvx-base.h"
#include "hex-utils.h"
#include "hvx-inverse.h"
#include "hvx-arith.h"
#if __HVX_ARCH__ < 79
#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b))
#define HVX_OP_MUL_F16(a, b) Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(a, b))
#else
#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b)
#define HVX_OP_MUL_F16(a, b) Q6_Vhf_vmpy_VhfVhf(a, b)
#endif
// Compute div by scaler in f32. Requires first by expanding fp32 to fp16 and converting the result back to fp32.
static inline HVX_Vector hvx_div_mul_f16_const_using_f32(HVX_Vector vec1_hf, HVX_Vector vec2_sf_const, HVX_Vector vec_hf_one_1_0) {
#if __HVX_ARCH__ < 79
HVX_VectorPair src_to_f32 = Q6_Wqf32_vmpy_VhfVhf(vec1_hf, vec_hf_one_1_0);
HVX_Vector src_to_f32_0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(src_to_f32));
HVX_Vector src_to_f32_1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(src_to_f32));
#else
HVX_VectorPair src_to_f32 = Q6_Wsf_vmpy_VhfVhf(vec1_hf, vec_hf_one_1_0);
HVX_Vector src_to_f32_0 = Q6_V_lo_W(src_to_f32);
HVX_Vector src_to_f32_1 = Q6_V_hi_W(src_to_f32);
#endif
HVX_Vector div_f32_0 = HVX_OP_MUL_F32(src_to_f32_0, vec2_sf_const);
HVX_Vector div_f32_1 = HVX_OP_MUL_F32(src_to_f32_1, vec2_sf_const);
#if __HVX_ARCH__ < 79
HVX_Vector res = hvx_vec_f32_to_f16(div_f32_0, div_f32_1);
#else
HVX_Vector res = Q6_Vhf_vcvt_VsfVsf(div_f32_0, div_f32_1);
#endif
return res;
}
// Variant for <v79: Use pre-computed f16 reciprocal constant
static inline HVX_Vector hvx_div_mul_f16_const_using_f16(HVX_Vector vec1_hf, HVX_Vector const_inv_hf) {
// Multiply by pre-computed f16 reciprocal constant
return HVX_OP_MUL_F16(vec1_hf, const_inv_hf);
}
#define hvx_div_scaler_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
HVX_Vector hf_one = Q6_Vh_vsplat_R(0x3C00); \
\
const uint32_t nvec = n / VLEN_FP16; \
const uint32_t nloe = n % VLEN_FP16; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
HVX_Vector res; \
if (__HVX_ARCH__ < 79) { \
res = hvx_div_mul_f16_const_using_f16(vsrc[i], val_vec_f16); \
} else { \
res = hvx_div_mul_f16_const_using_f32(vsrc[i], val_vec_f32, hf_one); \
} \
vdst[i] = res; \
} \
if (nloe) { \
HVX_Vector res; \
if (__HVX_ARCH__ < 79) { \
res = hvx_div_mul_f16_const_using_f16(vsrc[i], val_vec_f16); \
} else { \
res = hvx_div_mul_f16_const_using_f32(vsrc[i], val_vec_f32, hf_one); \
} \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, res); \
} \
} while(0)
static inline void hvx_div_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec_f32 = hvx_vec_splat_f32(1.0f/((float)val));
const HVX_Vector val_vec_f16 = hvx_vec_splat_f16(1.0f / val);
assert((uintptr_t) dst % 128 == 0);
assert((uintptr_t) src % 128 == 0);
hvx_div_scaler_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_div_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec_f32 = hvx_vec_splat_f32(1.0f/((float)val));
const HVX_Vector val_vec_f16 = hvx_vec_splat_f16(1.0f / val);
assert((uintptr_t) dst % 128 == 0);
hvx_div_scaler_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_div_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec_f32 = hvx_vec_splat_f32(1.0f/((float)val));
const HVX_Vector val_vec_f16 = hvx_vec_splat_f16(1.0f / val);
assert((uintptr_t) src % 128 == 0);
hvx_div_scaler_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_div_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec_f32 = hvx_vec_splat_f32(1.0f/((float)val));
const HVX_Vector val_vec_f16 = hvx_vec_splat_f16(1.0f / val);
hvx_div_scaler_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
// Compute div by using hvx_vec_inverse_f32_guard. Requires first by exapnding fp32 to fp16 and convert the result back to fp32.
static inline HVX_Vector hvx_vec_div_f16_using_f32(HVX_Vector vec1, HVX_Vector vec2, HVX_Vector f32_nan_inf_mask, HVX_Vector vec_hf_one_1_0) {
#if __HVX_ARCH__ < 79
// Convert first input to fp32
HVX_VectorPair vec1_to_f32 = Q6_Wqf32_vmpy_VhfVhf(vec1, vec_hf_one_1_0); // *1.0
HVX_Vector vec1_to_f32_0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vec1_to_f32));
HVX_Vector vec1_to_f32_1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vec1_to_f32));
// Convert second input to fp32
HVX_VectorPair vec2_to_f32 = Q6_Wqf32_vmpy_VhfVhf(vec2, vec_hf_one_1_0); // *1.0
HVX_Vector vec2_to_f32_0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vec2_to_f32));
HVX_Vector vec2_to_f32_1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(vec2_to_f32));
#else
// Convert first input to fp32
HVX_VectorPair vec1_to_f32 = Q6_Wsf_vmpy_VhfVhf(vec1, vec_hf_one_1_0); // *1.0
HVX_Vector vec1_to_f32_0 = Q6_V_lo_W(vec1_to_f32);
HVX_Vector vec1_to_f32_1 = Q6_V_hi_W(vec1_to_f32);
// Convert second input to fp32
HVX_VectorPair vec2_to_f32 = Q6_Wsf_vmpy_VhfVhf(vec2, vec_hf_one_1_0); // *1.0
HVX_Vector vec2_to_f32_0 = Q6_V_lo_W(vec2_to_f32);
HVX_Vector vec2_to_f32_1 = Q6_V_hi_W(vec2_to_f32);
#endif
// Inverse second input in fp32
HVX_Vector vec2_inv_f32_0 = hvx_vec_inverse_f32_guard(vec2_to_f32_0, f32_nan_inf_mask);
HVX_Vector vec2_inv_f32_1 = hvx_vec_inverse_f32_guard(vec2_to_f32_1, f32_nan_inf_mask);
// Multiply first input by inverse of second, in fp32
HVX_Vector div_f32_0 = HVX_OP_MUL_F32(vec1_to_f32_0, vec2_inv_f32_0);
HVX_Vector div_f32_1 = HVX_OP_MUL_F32(vec1_to_f32_1, vec2_inv_f32_1);
// Convert back to fp16
#if __HVX_ARCH__ < 79
HVX_Vector recip = hvx_vec_f32_to_f16(div_f32_0, div_f32_1);
#else
HVX_Vector recip = Q6_Vhf_vcvt_VsfVsf(div_f32_0, div_f32_1);
#endif
return recip;
}
// Hybrid approach: f16 reciprocal for <v79, f32 precision for >=v79
static inline HVX_Vector hvx_vec_hybrid_div_f16(HVX_Vector vec1, HVX_Vector vec2, HVX_Vector f32_nan_inf_mask, HVX_Vector f16_nan_inf_mask, HVX_Vector vec_hf_one_1_0) {
#if __HVX_ARCH__ < 79
// For older architectures, use f16 reciprocal to avoid NaN/-inf issues
HVX_Vector vec2_inv = hvx_vec_inverse_f16_guard(vec2, f16_nan_inf_mask);
return HVX_OP_MUL_F16(vec1, vec2_inv);
#else
return hvx_vec_div_f16_using_f32(vec1, vec2, f32_nan_inf_mask, vec_hf_one_1_0);
#endif
}
#define hvx_div_f16_loop_body(dst_type, src0_type, src1_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src0_type * restrict vsrc0 = (src0_type *) src0; \
src1_type * restrict vsrc1 = (src1_type *) src1; \
\
const HVX_Vector f32_nan_inf_mask = Q6_V_vsplat_R(0x7f800000); \
const HVX_Vector f16_nan_inf_mask = Q6_Vh_vsplat_R(0x7c00); \
const HVX_Vector hf_one = Q6_Vh_vsplat_R(0x3C00); \
\
const uint32_t nvec = n / VLEN_FP16; \
const uint32_t nloe = n % VLEN_FP16; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
HVX_Vector res = hvx_vec_hybrid_div_f16(vsrc0[i], vsrc1[i], \
f32_nan_inf_mask, f16_nan_inf_mask, \
hf_one); \
vdst[i] = res; \
} \
if (nloe) { \
HVX_Vector res = hvx_vec_hybrid_div_f16(vsrc0[i], vsrc1[i], \
f32_nan_inf_mask, f16_nan_inf_mask, \
hf_one); \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, res); \
} \
} while(0)
#define hvx_div_f32_loop_body(dst_type, src0_type, src1_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src0_type * restrict vsrc0 = (src0_type *) src0; \
src1_type * restrict vsrc1 = (src1_type *) src1; \
\
const HVX_Vector nan_inf_mask = Q6_V_vsplat_R(0x7f800000); \
\
const uint32_t nvec = n / VLEN_FP32; \
const uint32_t nloe = n % VLEN_FP32; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
HVX_Vector inv_src1 = hvx_vec_inverse_f32_guard(vsrc1[i], nan_inf_mask); \
HVX_Vector res = HVX_OP_MUL_F32(vsrc0[i], inv_src1); \
vdst[i] = res; \
} \
if (nloe) { \
HVX_Vector inv_src1 = hvx_vec_inverse_f32_guard(vsrc1[i], nan_inf_mask); \
HVX_Vector res = HVX_OP_MUL_F32(vsrc0[i], inv_src1); \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP32, res); \
} \
} while(0)
// Generic macro to define alignment permutations for an op
#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \
static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \
} \
static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \
} \
static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \
} \
static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \
} \
// Dispatcher logic
#define HVX_DIV_DISPATCHER(OP_NAME) \
static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \
if (hex_is_aligned((void *) dst, 128)) { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \
else OP_NAME##_aau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \
else OP_NAME##_auu(dst, src0, src1, num_elems); \
} \
} else { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \
else OP_NAME##_uau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \
else OP_NAME##_uuu(dst, src0, src1, num_elems); \
} \
} \
}
DEFINE_HVX_DIV_OP_VARIANTS(hvx_div_f32, hvx_div_f32_loop_body)
DEFINE_HVX_DIV_OP_VARIANTS(hvx_div_f16, hvx_div_f16_loop_body)
HVX_DIV_DISPATCHER(hvx_div_f32)
HVX_DIV_DISPATCHER(hvx_div_f16)
#undef HVX_OP_MUL_F32
#undef HVX_OP_MUL_F16
#endif // HVX_DIV_H
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-dump.h
+129
View File
@@ -0,0 +1,129 @@
#ifndef HVX_DUMP_H
#define HVX_DUMP_H
#include <HAP_farf.h>
#include <stdbool.h>
#include <stdint.h>
#include "hex-utils.h"
#include "hvx-types.h"
static void hvx_vec_dump_f16_n(char * pref, HVX_Vector v, uint32_t n) {
HVX_VectorAlias u = { .v = v };
const uint32_t n0 = n / 16;
const uint32_t n1 = n % 16;
int i = 0;
for (; i < n0; i++) {
hex_dump_f16_line(pref, u.fp16 + (16 * i), 16);
}
if (n1) {
hex_dump_f16_line(pref, u.fp16 + (16 * i), n1);
}
}
static void hvx_vec_dump_f16(char * pref, HVX_Vector v) {
hvx_vec_dump_f16_n(pref, v, 64);
}
static void hvx_vec_dump_f32_n(char * pref, HVX_Vector v, uint32_t n) {
HVX_VectorAlias u = { .v = v };
const uint32_t n0 = n / 16;
const uint32_t n1 = n % 16;
int i = 0;
for (; i < n0; i++) {
hex_dump_f32_line(pref, u.fp32 + (16 * i), 16);
}
if (n1) {
hex_dump_f32_line(pref, u.fp32 + (16 * i), n1);
}
}
static void hvx_vec_dump_f32_hmt(char * pref, HVX_Vector v) {
union {
HVX_Vector v;
float d[32];
} u = { .v = v };
FARF(HIGH, "%s: %.6f %.6f %.6f %.6f ... %.6f %.6f %.6f %.6f ... %.6f %.6f %.6f %.6f\n", pref, u.d[0], u.d[1],
u.d[2], u.d[3], u.d[12], u.d[13], u.d[14], u.d[15], u.d[28], u.d[29], u.d[30], u.d[31]);
}
static void hvx_vec_dump_f32(char * pref, HVX_Vector v) {
hvx_vec_dump_f32_n(pref, v, 32);
}
static void hvx_vec_dump_int32(char * pref, HVX_Vector v) {
union {
HVX_Vector v;
int32_t d[32];
} u = { .v = v };
for (int i = 0; i < 32 / 16; i++) {
hex_dump_int32_line(pref, u.d + (16 * i), 16);
}
}
static void hvx_vec_dump_int32_hmt(char * pref, HVX_Vector v) {
union {
HVX_Vector v;
int32_t d[32];
} u = { .v = v };
FARF(HIGH, "%s: %d %d %d %d ... %d %d %d %d ... %d %d %d %d\n", pref, u.d[0], u.d[1], u.d[2], u.d[3], u.d[12],
u.d[13], u.d[14], u.d[15], u.d[28], u.d[29], u.d[30], u.d[31]);
}
static void hvx_vec_dump_int8_hmt(char * pref, HVX_Vector v) {
union {
HVX_Vector v;
int8_t d[128];
} u = { .v = v };
FARF(HIGH, "%s: %d %d %d %d ... %d %d %d %d ... %d %d %d %d\n", pref, u.d[0], u.d[1], u.d[2], u.d[3], u.d[60],
u.d[61], u.d[62], u.d[63], u.d[124], u.d[125], u.d[126], u.d[127]);
}
static void hvx_vec_dump_int8(char * pref, HVX_Vector v) {
union {
HVX_Vector v;
int8_t d[128];
} u = { .v = v };
for (int i = 0; i < 128 / 16; i++) {
hex_dump_int8_line(pref, u.d + (16 * i), 16);
}
}
static void hvx_vec_dump_uint8(char * pref, HVX_Vector v) {
union {
HVX_Vector v;
uint8_t d[128];
} u = { .v = v };
for (int i = 0; i < 128 / 16; i++) {
hex_dump_uint8_line(pref, u.d + (16 * i), 16);
}
}
static bool hvx_vec_eq(HVX_Vector v0, HVX_Vector v1, size_t n) {
typedef union {
HVX_Vector v;
int8_t d[128];
} U;
U u0 = { .v = v0 };
U u1 = { .v = v1 };
for (int i = 0; i < n; i++) {
if (u0.d[i] != u1.d[i]) {
return false;
}
}
return true;
}
#endif /* HVX_DUMP_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-erf.h
+74
View File
@@ -0,0 +1,74 @@
#ifndef HVX_ERF_H
#define HVX_ERF_H
#include "hvx-base.h"
#include "hvx-exp.h"
#include "hvx-inverse.h"
// Maximum error is about 1.5e-7 for the Abramowitz-Stegun approximation.
static __attribute__((noinline)) HVX_Vector hvx_vec_erf_f32(HVX_Vector x) {
const HVX_Vector zero = hvx_vec_splat_f32(0.0f);
const HVX_Vector ax = hvx_vec_abs_f32(x);
HVX_Vector t = hvx_vec_inverse_f32(hvx_vec_add_f32_f32(
hvx_vec_splat_f32(1.0f), hvx_vec_mul_f32_f32(hvx_vec_splat_f32(0.3275911f), ax)));
HVX_Vector poly = hvx_vec_splat_f32(1.061405429f);
poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(-1.453152027f), hvx_vec_mul_f32_f32(poly, t));
poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(1.421413741f), hvx_vec_mul_f32_f32(poly, t));
poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(-0.284496736f), hvx_vec_mul_f32_f32(poly, t));
poly = hvx_vec_add_f32_f32(hvx_vec_splat_f32(0.254829592f), hvx_vec_mul_f32_f32(poly, t));
const HVX_Vector exp_term = hvx_vec_exp_f32(hvx_vec_neg_f32(hvx_vec_mul_f32_f32(ax, ax)));
HVX_Vector result = hvx_vec_sub_f32_f32(hvx_vec_splat_f32(1.0f),
hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(poly, t), exp_term));
const HVX_VectorPred neg = Q6_Q_vcmp_gt_VsfVsf(zero, x);
result = Q6_V_vmux_QVV(neg, hvx_vec_neg_f32(result), result);
return result;
}
// GELU_ERF uses the erf definition.
static inline HVX_Vector hvx_vec_gelu_erf_f32(HVX_Vector x) {
const HVX_Vector scale = hvx_vec_splat_f32(0.7071067811865475f);
const HVX_Vector half = hvx_vec_splat_f32(0.5f);
const HVX_Vector one = hvx_vec_splat_f32(1.0f);
const HVX_Vector max_x = hvx_vec_splat_f32(10.0f);
const HVX_Vector min_x = hvx_vec_splat_f32(-10.0f);
const HVX_VectorPred neg_large = Q6_Q_vcmp_gt_VsfVsf(min_x, x);
const HVX_VectorPred pos_large = Q6_Q_vcmp_gt_VsfVsf(x, max_x);
HVX_Vector x_calc = Q6_V_vmux_QVV(neg_large, min_x, x);
x_calc = Q6_V_vmux_QVV(pos_large, max_x, x_calc);
const HVX_Vector erf = hvx_vec_erf_f32(hvx_vec_mul_f32_f32(x_calc, scale));
HVX_Vector result = hvx_vec_mul_f32_f32(hvx_vec_mul_f32_f32(half, x_calc), hvx_vec_add_f32_f32(one, erf));
result = Q6_V_vmux_QVV(neg_large, hvx_vec_splat_f32(0.0f), result);
result = Q6_V_vmux_QVV(pos_large, x, result);
return result;
}
static inline void hvx_gelu_erf_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(float);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
vdst[i] = hvx_vec_gelu_erf_f32(vsrc[i]);
}
if (nloe) {
HVX_Vector v = hvx_vec_gelu_erf_f32(vsrc[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
#endif /* HVX_ERF_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-exp.h
+255
View File
@@ -0,0 +1,255 @@
#ifndef HVX_EXP_H
#define HVX_EXP_H
#include <stdbool.h>
#include <stdint.h>
#include <math.h>
#include "hvx-base.h"
#include "hvx-floor.h"
#define EXP_COEFF_5 (0x39506967) // 0.000198757 = 1/(7!)
#define EXP_COEFF_4 (0x3AB743CE) // 0.0013982 = 1/(6!)
#define EXP_COEFF_3 (0x3C088908) // 0.00833345 = 1/(5!)
#define EXP_COEFF_2 (0x3D2AA9C1) // 0.416658 = 1/(4!)
#define EXP_COEFF_1 (0x3E2AAAAA) // 0.16666667 = 1/(3!)
#define EXP_COEFF_0 (0x3F000000) // 0.5 = 1/(2!)
#define EXP_LOGN2 (0x3F317218) // ln(2) = 0.6931471805
#define EXP_LOG2E (0x3FB8AA3B) // log2(e) = 1/ln(2) = 1.4426950408
#define EXP_LOG2E_F 1.44269504f
#define EXP_ONE (0x3f800000) // 1.0
#define EXP_RANGE_R (0x42B17218) // ln(FLT_MAX) approx = 88.7228
#define EXP_RANGE_L (0xC2B00000) // -88.0 (approx log(FLT_MIN))
static inline HVX_Vector hvx_vec_exp_f32(HVX_Vector in_vec) {
HVX_Vector z_qf32_v;
HVX_Vector x_v;
HVX_Vector x_qf32_v;
HVX_Vector y_v;
HVX_Vector k_v;
HVX_Vector f_v;
HVX_Vector epsilon_v;
HVX_Vector log2e = Q6_V_vsplat_R(EXP_LOG2E);
HVX_Vector logn2 = Q6_V_vsplat_R(EXP_LOGN2);
HVX_Vector E_const;
HVX_Vector zero_v = Q6_V_vzero();
// exp(x) is approximated as follows:
// f = floor(x/ln(2)) = floor(x*log2(e))
// epsilon = x - f*ln(2)
// exp(x) = exp(epsilon+f*ln(2))
// = exp(epsilon)*exp(f*ln(2))
// = exp(epsilon)*2^f
//
// Since epsilon is close to zero, it can be approximated with its Taylor series:
// exp(x) ~= 1+x+x^2/2!+x^3/3!+...+x^n/n!+...
// Preserving the first eight elements, we get:
// exp(x) ~= 1+x+e0*x^2+e1*x^3+e2*x^4+e3*x^5+e4*x^6+e5*x^7
// = 1+x+(E0+(E1+(E2+(E3+(E4+E5*x)*x)*x)*x)*x)*x^2
HVX_Vector temp_v = in_vec;
// Clamp inputs to (-88.0, 88.0) to avoid overflow/underflow
HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VsfVsf(in_vec, Q6_V_vsplat_R(EXP_RANGE_R));
HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VsfVsf(Q6_V_vsplat_R(EXP_RANGE_L), in_vec);
in_vec = Q6_V_vmux_QVV(pred_cap_right, Q6_V_vsplat_R(EXP_RANGE_R), temp_v);
in_vec = Q6_V_vmux_QVV(pred_cap_left, Q6_V_vsplat_R(EXP_RANGE_L), in_vec);
epsilon_v = Q6_Vqf32_vmpy_VsfVsf(log2e, in_vec);
epsilon_v = Q6_Vsf_equals_Vqf32(epsilon_v);
// f_v is the floating point result and k_v is the integer result
f_v = hvx_vec_floor_f32(epsilon_v);
k_v = hvx_vec_truncate_f32(f_v);
x_qf32_v = Q6_Vqf32_vadd_VsfVsf(in_vec, zero_v);
// x = x - f_v * logn2;
epsilon_v = Q6_Vqf32_vmpy_VsfVsf(f_v, logn2);
x_qf32_v = Q6_Vqf32_vsub_Vqf32Vqf32(x_qf32_v, epsilon_v);
// normalize before every QFloat's vmpy
x_qf32_v = Q6_Vqf32_vadd_Vqf32Vsf(x_qf32_v, zero_v);
x_v = Q6_Vsf_equals_Vqf32(x_qf32_v);
// z = x * x;
z_qf32_v = Q6_Vqf32_vmpy_Vqf32Vqf32(x_qf32_v, x_qf32_v);
z_qf32_v = Q6_Vqf32_vadd_Vqf32Vsf(z_qf32_v, zero_v);
// y = E4 + E5 * x;
E_const = Q6_V_vsplat_R(EXP_COEFF_5);
y_v = Q6_Vqf32_vmpy_VsfVsf(E_const, x_v);
E_const = Q6_V_vsplat_R(EXP_COEFF_4);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);
// y = E3 + y * x;
E_const = Q6_V_vsplat_R(EXP_COEFF_3);
y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, x_qf32_v);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);
// y = E2 + y * x;
E_const = Q6_V_vsplat_R(EXP_COEFF_2);
y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, x_qf32_v);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);
// y = E1 + y * x;
E_const = Q6_V_vsplat_R(EXP_COEFF_1);
y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, x_qf32_v);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);
// y = E0 + y * x;
E_const = Q6_V_vsplat_R(EXP_COEFF_0);
y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, x_qf32_v);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, E_const);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);
// y = x + y * z;
y_v = Q6_Vqf32_vmpy_Vqf32Vqf32(y_v, z_qf32_v);
y_v = Q6_Vqf32_vadd_Vqf32Vqf32(y_v, x_qf32_v);
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, zero_v);
// y = y + 1.0;
y_v = Q6_Vqf32_vadd_Vqf32Vsf(y_v, Q6_V_vsplat_R(EXP_ONE));
// insert exponents
// y = ldexpf(y, k);
// y_v += k_v; // qf32
// modify exponent
y_v = Q6_Vsf_equals_Vqf32(y_v);
// add k_v to the exponent of y_v
HVX_Vector y_v_exponent = Q6_Vw_vasl_VwR(y_v, 1);
y_v_exponent = Q6_Vuw_vlsr_VuwR(y_v_exponent, IEEE_VSF_MANTLEN + 1);
y_v_exponent = Q6_Vw_vadd_VwVw(k_v, y_v_exponent);
// exponent cannot be negative; if overflow is detected, result is set to zero
HVX_VectorPred qy_v_negative_exponent = Q6_Q_vcmp_gt_VwVw(zero_v, y_v_exponent);
y_v = Q6_Vw_vaslacc_VwVwR(y_v, k_v, IEEE_VSF_MANTLEN);
y_v = Q6_V_vmux_QVV(qy_v_negative_exponent, zero_v, y_v);
return y_v;
}
static inline HVX_Vector hvx_vec_exp_f32_guard(HVX_Vector in_vec, HVX_Vector max_exp, HVX_Vector inf) {
const HVX_VectorPred pred0 = Q6_Q_vcmp_gt_VsfVsf(in_vec, max_exp);
HVX_Vector out = hvx_vec_exp_f32(in_vec);
return Q6_V_vmux_QVV(pred0, inf, out);
}
static inline void hvx_exp_f32(uint8_t * restrict dst, const uint8_t * restrict src, const int num_elems, bool negate) {
int left_over = num_elems & (VLEN_FP32 - 1);
int num_elems_whole = num_elems - left_over;
int unaligned_addr = 0;
int unaligned_loop = 0;
if ((0 == hex_is_aligned((void *) src, VLEN)) || (0 == hex_is_aligned((void *) dst, VLEN))) {
unaligned_addr = 1;
}
// assert((0 == unaligned_addr) || (0 == num_elems_whole));
if ((1 == unaligned_addr) && (num_elems_whole != 0)) {
unaligned_loop = 1;
}
HVX_Vector vec_out = Q6_V_vzero();
static const float kInf = INFINITY;
static const float kMaxExp = 88.7228f;
const HVX_Vector max_exp = hvx_vec_splat_f32(kMaxExp);
const HVX_Vector inf = hvx_vec_splat_f32(kInf);
if (0 == unaligned_loop) {
HVX_Vector * p_vec_in1 = (HVX_Vector *) src;
HVX_Vector * p_vec_out = (HVX_Vector *) dst;
#pragma unroll(2)
for (int i = 0; i < num_elems_whole; i += VLEN_FP32) {
if (true == negate) {
HVX_Vector neg_vec_in = hvx_vec_neg_f32(*p_vec_in1++);
*p_vec_out++ = hvx_vec_exp_f32_guard(neg_vec_in, max_exp, inf);
} else {
*p_vec_out++ = hvx_vec_exp_f32_guard(*p_vec_in1++, max_exp, inf);
}
}
} else {
#pragma unroll(2)
for (int i = 0; i < num_elems_whole; i += VLEN_FP32) {
HVX_Vector in = *(HVX_UVector *) (src + i * SIZEOF_FP32);
if (true == negate) {
HVX_Vector neg_vec_in = hvx_vec_neg_f32(in);
*(HVX_UVector *) (dst + i * SIZEOF_FP32) = hvx_vec_exp_f32_guard(neg_vec_in, max_exp, inf);
} else {
*(HVX_UVector *) (dst + i * SIZEOF_FP32) = hvx_vec_exp_f32_guard(in, max_exp, inf);
}
}
}
if (left_over > 0) {
const float * srcf = (float *) src + num_elems_whole;
float * dstf = (float *) dst + num_elems_whole;
HVX_Vector in = *(HVX_UVector *) srcf;
if (true == negate) {
HVX_Vector neg_vec_in = hvx_vec_neg_f32(in);
vec_out = hvx_vec_exp_f32_guard(neg_vec_in, max_exp, inf);
} else {
vec_out = hvx_vec_exp_f32_guard(in, max_exp, inf);
}
hvx_vec_store_u((void *) dstf, left_over * SIZEOF_FP32, vec_out);
}
}
static inline HVX_Vector hvx_vec_exp2_f16(HVX_Vector x_v) {
const HVX_Vector zero_v = Q6_V_vzero();
const HVX_Vector half_hf_v = Q6_Vh_vsplat_R(0x3800); // fp16 0.5
// Clamp input to prevent integer underflow in FP16-to-INT16 conversion
const HVX_Vector v_clamp_min = hvx_vec_splat_f16(-24.0f);
x_v = Q6_Vhf_vmax_VhfVhf(v_clamp_min, x_v);
// k = round_toward_neg_inf(x); f = (float)k; frac = x - f
HVX_Vector x_minus_half = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vsub_VhfVhf(x_v, half_hf_v));
HVX_Vector k_v = Q6_Vh_equals_Vhf(x_minus_half); // truncate to int16
HVX_Vector f_v = Q6_Vhf_equals_Vh(k_v); // back to fp16
HVX_Vector x_qf16 = Q6_Vqf16_vsub_VhfVhf(x_v, f_v); // fractional part in qf16
// Horner: y = ((((E5*x + E4)*x + E3)*x + E2)*x + E1)*x + E0
HVX_Vector y = Q6_Vqf16_vmpy_Vqf16Vqf16(Q6_Vh_vsplat_R(0x5082), x_qf16); // E5*x
y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x157d)); // + E4
y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16);
y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x20ed)); // + E3
y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16);
y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x2b1b)); // + E2
y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16);
y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x33b0)); // + E1
y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16);
y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x398c)); // + E0
y = Q6_Vqf16_vmpy_Vqf16Vqf16(y, x_qf16); // y = y * x
y = Q6_Vqf16_vadd_Vqf16Vhf(y, Q6_Vh_vsplat_R(0x3c00)); // + 1.0
// Combine polynomial (mantissa) with integer part (exponent): result = y * 2^k
y = Q6_Vhf_equals_Vqf16(y);
HVX_Vector y_exp = Q6_Vuh_vlsr_VuhR(Q6_Vh_vasl_VhR(y, 1), 11);
y_exp = Q6_Vh_vadd_VhVh(k_v, y_exp);
HVX_VectorPred q_underflow = Q6_Q_vcmp_gt_VhVh(zero_v, y_exp);
y = Q6_Vh_vaslacc_VhVhR(y, k_v, 10);
return Q6_V_vmux_QVV(q_underflow, zero_v, y);
}
#endif /* HVX_EXP_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-fa-kernels.h
+303
View File
@@ -0,0 +1,303 @@
#ifndef HVX_FA_KERNELS_H
#define HVX_FA_KERNELS_H
#include <assert.h>
#include <math.h>
#include "hvx-utils.h"
// Little inner kernels for HVX
#if __HVX_ARCH__ < 79
#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b))
#define HVX_OP_SUB_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a, b))
#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b))
#else
#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b)
#define HVX_OP_SUB_F32(a, b) Q6_Vsf_vsub_VsfVsf(a, b)
#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b)
#endif
// This is a bit of a hack because the compiler is struggling to properly inline
// the default hvx_vec_f32_to_f16 with output into the local array.
static __attribute__((unused)) __attribute__((noinline)) void hvx_vec_f32_to_f16_a(void *ptr, HVX_Vector v0, HVX_Vector v1)
{
*(HVX_Vector *) ptr = hvx_vec_f32_to_f16(v0, v1);
}
// Dot product of two F16 vectors, accumulating to float
static inline void hvx_dot_f16_f16_aa(float * restrict r, const void * restrict x, const void * restrict y, unsigned int n, float s) {
const HVX_Vector * restrict vx = (const HVX_Vector * restrict) x; // fp16
const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16
uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors
uint32_t nloe = n % VLEN_FP16; // leftover elements
HVX_VectorPair rsum_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));
uint32_t i = 0;
#pragma unroll(4)
for (i = 0; i < nvec; i++) {
rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, vx[i], vy[i]);
}
if (nloe) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);
HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]);
HVX_Vector x_hf = Q6_V_vand_QV(bmask, vx[i]);
rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf);
}
HVX_Vector rsum = HVX_OP_ADD_F32(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p));
rsum = HVX_OP_MUL_F32(hvx_vec_splat_f32(s), hvx_vec_reduce_sum_f32(rsum));
hvx_vec_store_u(r, 4, rsum);
}
static inline HVX_Vector hvx_dot_f16_f16_aa_rx4(const void * restrict y,
const uint8_t * restrict x,
const size_t stride_x,
const size_t nvec,
const size_t nloe) {
const HVX_Vector * restrict vx0 = (const HVX_Vector * restrict) x; // fp16
const HVX_Vector * restrict vx1 = (const HVX_Vector * restrict) (x + stride_x); // fp16
const HVX_Vector * restrict vx2 = (const HVX_Vector * restrict) (x + stride_x * 2); // fp16
const HVX_Vector * restrict vx3 = (const HVX_Vector * restrict) (x + stride_x * 3); // fp16
const HVX_Vector * restrict vy = (const HVX_Vector * restrict) y; // fp16
HVX_VectorPair rsum0_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));
HVX_VectorPair rsum1_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));
HVX_VectorPair rsum2_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));
HVX_VectorPair rsum3_p = Q6_W_vcombine_VV(Q6_V_vsplat_R(0), Q6_V_vsplat_R(0));
uint32_t i = 0;
for (i = 0; i < nvec; i++) {
HVX_Vector y_hf = vy[i];
HVX_Vector x0_hf = vx0[i];
HVX_Vector x1_hf = vx1[i];
HVX_Vector x2_hf = vx2[i];
HVX_Vector x3_hf = vx3[i];
rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf);
rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf);
rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf);
rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf);
}
if (nloe) {
// Load x (fp16) and zero-out unused elements
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);
HVX_Vector y_hf = Q6_V_vand_QV(bmask, vy[i]);
HVX_Vector x0_hf = Q6_V_vand_QV(bmask, vx0[i]);
HVX_Vector x1_hf = Q6_V_vand_QV(bmask, vx1[i]);
HVX_Vector x2_hf = Q6_V_vand_QV(bmask, vx2[i]);
HVX_Vector x3_hf = Q6_V_vand_QV(bmask, vx3[i]);
rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf);
rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf);
rsum2_p = hvx_vec_mpyacc_f32_f16(rsum2_p, x2_hf, y_hf);
rsum3_p = hvx_vec_mpyacc_f32_f16(rsum3_p, x3_hf, y_hf);
}
HVX_Vector rsum0 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p));
HVX_Vector rsum1 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p));
HVX_Vector rsum2 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum2_p), Q6_V_hi_W(rsum2_p));
HVX_Vector rsum3 = HVX_OP_ADD_F32(Q6_V_lo_W(rsum3_p), Q6_V_hi_W(rsum3_p));
HVX_Vector_x4 rsum0123 = { .v = { rsum0, rsum1, rsum2, rsum3 } };
return hvx_vec_reduce_sum_f32x4(rsum0123);
}
static inline HVX_Vector hvx_dot_f16_f16_aa_rx32(const void * restrict y,
const uint8_t * restrict x,
const size_t stride_x,
const size_t n,
float s) {
const size_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors
const size_t nloe = n % VLEN_FP16; // leftover elements
HVX_Vector sums = Q6_V_vzero();
const size_t stride_x_4 = stride_x * 4;
for (uint32_t j = 0; j < VLEN_FP32; j += 4) {
HVX_Vector sums_x4 = hvx_dot_f16_f16_aa_rx4(y, x, stride_x, nvec, nloe);
HVX_VectorPred pred = Q6_Q_vsetq_R(j * SIZEOF_FP32);
sums = Q6_V_vmux_QVV(pred, sums, sums_x4);
x += stride_x_4;
}
return HVX_OP_MUL_F32(hvx_vec_splat_f32(s), sums);
}
// MAD: y (F32) += x (F16) * s (F16)
static inline void hvx_mad_f32_f16_aa(float * restrict y, const void * restrict x, const __fp16 * restrict s, uint32_t n) {
const HVX_Vector * restrict vx0 = (const HVX_Vector *) x;
HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y;
HVX_Vector * restrict vy = (HVX_Vector *) y;
uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors
uint32_t nloe = n % VLEN_FP16; // leftover elements
HVX_Vector S0 = hvx_vec_splat_f16(*s);
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; ++i) {
vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0);
}
if (nloe) {
HVX_VectorPair xy_p = vy_p[i];
xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0);
HVX_Vector xy = Q6_V_lo_W(xy_p);
i = 2 * i; // index for vy
if (nloe >= VLEN_FP32) {
vy[i] = xy;
nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p);
}
if (nloe) {
hvx_vec_store_a(&vy[i], nloe * 4, xy);
}
}
}
// MAD: y (F32) += x0 (F16) * s0 (F16) + x1 (F16) * s1 (F16)
static inline void hvx_mad_f32_f16_aa_rx2(float * restrict y, const void * restrict x0, const void * restrict x1,
const __fp16 * restrict s0, const __fp16 * restrict s1, uint32_t n) {
const HVX_Vector * restrict vx0 = (const HVX_Vector *) x0;
const HVX_Vector * restrict vx1 = (const HVX_Vector *) x1;
HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y;
HVX_Vector * restrict vy = (HVX_Vector *) y;
uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors
uint32_t nloe = n % VLEN_FP16; // leftover elements
HVX_Vector S0 = hvx_vec_splat_f16(*s0);
HVX_Vector S1 = hvx_vec_splat_f16(*s1);
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; ++i) {
vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0);
vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx1[i]), S1);
}
if (nloe) {
HVX_VectorPair xy_p = vy_p[i];
xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0);
xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx1[i]), S1);
HVX_Vector xy = Q6_V_lo_W(xy_p);
i = 2 * i; // index for vy
if (nloe >= VLEN_FP32) {
vy[i] = xy;
nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p);
}
if (nloe) {
hvx_vec_store_a(&vy[i], nloe * 4, xy);
}
}
}
static inline void hvx_mad_f32_f16_aa_vec(float * restrict y, const void * restrict x, HVX_Vector S0, uint32_t n) {
const HVX_Vector * restrict vx0 = (const HVX_Vector *) x;
HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y;
HVX_Vector * restrict vy = (HVX_Vector *) y;
uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors
uint32_t nloe = n % VLEN_FP16; // leftover elements
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; ++i) {
vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0);
}
if (nloe) {
HVX_VectorPair xy_p = vy_p[i];
xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0);
HVX_Vector xy = Q6_V_lo_W(xy_p);
i = 2 * i; // index for vy
if (nloe >= VLEN_FP32) {
vy[i] = xy;
nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p);
}
if (nloe) {
hvx_vec_store_a(&vy[i], nloe * 4, xy);
}
}
}
static inline void hvx_mad_f32_f16_aa_rx2_vec(float * restrict y, const void * restrict x0, const void * restrict x1,
HVX_Vector S0, HVX_Vector S1, uint32_t n) {
const HVX_Vector * restrict vx0 = (const HVX_Vector *) x0;
const HVX_Vector * restrict vx1 = (const HVX_Vector *) x1;
HVX_VectorPair * restrict vy_p = (HVX_VectorPair *) y;
HVX_Vector * restrict vy = (HVX_Vector *) y;
uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors
uint32_t nloe = n % VLEN_FP16; // leftover elements
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; ++i) {
vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx0[i]), S0);
vy_p[i] = hvx_vec_mpyacc_f32_f16(vy_p[i], Q6_Vh_vshuff_Vh(vx1[i]), S1);
}
if (nloe) {
HVX_VectorPair xy_p = vy_p[i];
xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx0[i]), S0);
xy_p = hvx_vec_mpyacc_f32_f16(xy_p, Q6_Vh_vshuff_Vh(vx1[i]), S1);
HVX_Vector xy = Q6_V_lo_W(xy_p);
i = 2 * i; // index for vy
if (nloe >= VLEN_FP32) {
vy[i] = xy;
nloe -= VLEN_FP32; ++i; xy = Q6_V_hi_W(xy_p);
}
if (nloe) {
hvx_vec_store_a(&vy[i], nloe * 4, xy);
}
}
}
static inline void hvx_scale_vec_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t n, HVX_Vector vs) {
assert((size_t) dst % 128 == 0);
assert((size_t) src % 128 == 0);
const HVX_Vector * restrict vsrc = (const HVX_Vector * restrict) src;
HVX_Vector * restrict vdst = (HVX_Vector * restrict) dst;
const uint32_t nvec = n / VLEN_FP32;
const uint32_t nloe = n % VLEN_FP32;
uint32_t i = 0;
#pragma unroll(4)
for (; i < nvec; ++i) {
vdst[i] = HVX_OP_MUL_F32(vsrc[i], vs);
}
if (nloe) {
hvx_vec_store_a(&vdst[i], nloe * sizeof(float), HVX_OP_MUL_F32(vsrc[i], vs));
}
}
#endif /* HVX_FA_KERNELS_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-flash-attn.h
+47
View File
@@ -0,0 +1,47 @@
#ifndef HVX_FLASH_ATTN_H
#define HVX_FLASH_ATTN_H
#include <math.h>
#include "hvx-utils.h"
// Scalar helper to compute a single ALiBi slope.
static inline float alibi_slope(uint32_t h, uint32_t n_head_log2, float m0, float m1) {
return (h < n_head_log2) ? powf(m0, h + 1) : powf(m1, 2 * (h - n_head_log2) + 1);
}
// Vectorized helper to compute 32 ALiBi slopes starting from (kv_head * G).
static inline HVX_Vector hvx_alibi_slopes(
uint32_t kv_head,
uint32_t G,
uint32_t n_head_log2,
float m0,
float m1
) {
static const float ramp_32[32] __attribute__((aligned(128))) = {
0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f,
8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f,
16.0f, 17.0f, 18.0f, 19.0f, 20.0f, 21.0f, 22.0f, 23.0f,
24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0f, 31.0f
};
HVX_Vector v_ramp = hvx_vmem(ramp_32);
HVX_Vector v_h_base = hvx_vec_splat_f32((float)(kv_head * G));
HVX_Vector v_h = hvx_vec_add_f32_f32(v_h_base, v_ramp);
// Compute exponent_m0: h + 1
HVX_Vector v_exp_m0 = hvx_vec_add_f32_f32(v_h, hvx_vec_splat_f32(1.0f));
// Compute exponent_m1: 2 * (h - n_head_log2) + 1
HVX_Vector v_n_head_log2 = hvx_vec_splat_f32((float)n_head_log2);
HVX_Vector v_h_minus = hvx_vec_sub_f32_f32(v_h, v_n_head_log2);
HVX_Vector v_exp_m1 = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(hvx_vec_splat_f32(2.0f), v_h_minus), hvx_vec_splat_f32(1.0f));
// Compute powers
HVX_Vector v_pow_m0 = hvx_vec_pow_const_base_f32(m0, v_exp_m0);
HVX_Vector v_pow_m1 = hvx_vec_pow_const_base_f32(m1, v_exp_m1);
// Select based on h < n_head_log2
HVX_VectorPred p_cond = Q6_Q_vcmp_gt_VsfVsf(v_n_head_log2, v_h); // v_n_head_log2 > v_h <=> h < n_head_log2
return Q6_V_vmux_QVV(p_cond, v_pow_m0, v_pow_m1);
}
#endif /* HVX_FLASH_ATTN_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-floor.h
+108
View File
@@ -0,0 +1,108 @@
#ifndef HVX_FLOOR_H
#define HVX_FLOOR_H
#include <stdbool.h>
#include <stdint.h>
#include "hvx-base.h"
#define IEEE_VSF_EXPLEN (8)
#define IEEE_VSF_EXPBIAS (127)
#define IEEE_VSF_EXPMASK (0xFF)
#define IEEE_VSF_MANTLEN (23)
#define IEEE_VSF_MANTMASK (0x7FFFFF)
#define IEEE_VSF_MIMPMASK (0x800000)
static inline HVX_Vector hvx_vec_truncate_f32(HVX_Vector in_vec) {
HVX_Vector mask_mant_v = Q6_V_vsplat_R(IEEE_VSF_MANTMASK);
HVX_Vector mask_impl_v = Q6_V_vsplat_R(IEEE_VSF_MIMPMASK);
HVX_Vector const_zero_v = Q6_V_vzero();
HVX_VectorPred q_negative = Q6_Q_vcmp_gt_VwVw(const_zero_v, in_vec);
HVX_Vector expval_v = in_vec >> IEEE_VSF_MANTLEN;
expval_v &= IEEE_VSF_EXPMASK;
expval_v -= IEEE_VSF_EXPBIAS;
// negative exp == fractional value
HVX_VectorPred q_negexp = Q6_Q_vcmp_gt_VwVw(const_zero_v, expval_v);
HVX_Vector rshift_v = IEEE_VSF_MANTLEN - expval_v; // fractional bits - exp shift
HVX_Vector mant_v = in_vec & mask_mant_v; // obtain mantissa
HVX_Vector vout = Q6_Vw_vadd_VwVw(mant_v, mask_impl_v); // add implicit 1.0
vout = Q6_Vw_vasr_VwVw(vout, rshift_v); // shift to obtain truncated integer
vout = Q6_V_vmux_QVV(q_negexp, const_zero_v, vout); // expval<0 -> 0
HVX_Vector neg_vout = -vout;
vout = Q6_V_vmux_QVV(q_negative, neg_vout, vout); // handle negatives
return (vout);
}
static inline HVX_Vector hvx_vec_floor_f32(HVX_Vector in_vec) {
HVX_Vector mask_mant_v = Q6_V_vsplat_R(IEEE_VSF_MANTMASK);
HVX_Vector mask_impl_v = Q6_V_vsplat_R(IEEE_VSF_MIMPMASK);
HVX_Vector const_mnlen_v = Q6_V_vsplat_R(IEEE_VSF_MANTLEN);
HVX_Vector const_zero_v = Q6_V_vzero();
HVX_Vector const_negone_v = Q6_V_vsplat_R(0xbf800000); // -1 IEEE vsf
HVX_VectorPred q_negative = Q6_Q_vcmp_gt_VwVw(const_zero_v, in_vec);
HVX_Vector expval_v = in_vec >> IEEE_VSF_MANTLEN;
expval_v &= IEEE_VSF_EXPMASK;
expval_v -= IEEE_VSF_EXPBIAS;
HVX_VectorPred q_negexp = Q6_Q_vcmp_gt_VwVw(const_zero_v, expval_v);
HVX_VectorPred q_expltmn = Q6_Q_vcmp_gt_VwVw(const_mnlen_v, expval_v);
HVX_VectorPred q_negexp_pos = Q6_Q_vcmp_gtand_QVwVw(q_negexp, in_vec, const_zero_v);
HVX_VectorPred q_negexp_neg = Q6_Q_vcmp_gtand_QVwVw(q_negexp, const_zero_v, in_vec);
// if expval < 0 (q_negexp) // <0, floor is 0
// if vin > 0
// floor = 0
// if vin < 0
// floor = -1
// if expval < mant_len (q_expltmn) // >0, but fraction may exist
// get sign (q_negative)
// mask >> expval // fraction bits to mask off
// vout = ~(mask) // apply mask to remove fraction
// if (qneg) // negative floor is one less (more, sign bit for neg)
// vout += ((impl_mask) >> expval)
// if (mask && vin)
// vout = vin
// else // already an integer
// ; // no change
// compute floor
mask_mant_v >>= expval_v;
HVX_Vector neg_addin_v = mask_impl_v >> expval_v;
HVX_Vector vout_neg_addin = Q6_Vw_vadd_VwVw(in_vec, neg_addin_v);
HVX_Vector vout = Q6_V_vmux_QVV(q_negative, vout_neg_addin, in_vec);
HVX_Vector mask_chk_v = Q6_V_vand_VV(in_vec, mask_mant_v); // chk if bits set
HVX_VectorPred q_integral = Q6_Q_vcmp_eq_VwVw(const_zero_v, mask_chk_v);
HVX_Vector not_mask_v = Q6_V_vnot_V(mask_mant_v); // frac bits to clear
HVX_Vector vfrfloor_v = Q6_V_vand_VV(vout, not_mask_v); // clear frac bits
vout = in_vec;
vout = Q6_V_vmux_QVV(q_expltmn, vfrfloor_v, vout); // expval<mant
vout = Q6_V_vmux_QVV(q_integral, in_vec, vout); // integral values
vout = Q6_V_vmux_QVV(q_negexp_pos, const_zero_v, vout); // expval<0 x>0 -> 0
vout = Q6_V_vmux_QVV(q_negexp_neg, const_negone_v, vout); // expval<0 x<0 -> -1
return vout;
}
// ceil(x) = -floor(-x); keep +-0 as is since floor(-0) returns -1
static inline HVX_Vector hvx_vec_ceil_f32(HVX_Vector in_vec) {
HVX_Vector vout = hvx_vec_neg_f32(hvx_vec_floor_f32(hvx_vec_neg_f32(in_vec)));
HVX_Vector vabs = Q6_V_vand_VV(in_vec, Q6_V_vsplat_R(0x7fffffff));
HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(vabs, Q6_V_vzero());
return Q6_V_vmux_QVV(q_zero, in_vec, vout);
}
#endif /* HVX_FLOOR_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-inverse.h
+210
View File
@@ -0,0 +1,210 @@
#ifndef HVX_INVERSE_H
#define HVX_INVERSE_H
#include <HAP_farf.h>
#include <math.h>
#include <string.h>
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include "hvx-base.h"
// ====================================================
// FUNCTION: 1/(x+1) y(0) = 1, y(0.5) = 0.6667, y(1) = 0.5
// Order:3; continuity: True; Ends forced: True
// Mode: unsigned; Result fractional bits: 14
// Peak Error: 1.1295e-04 Rms Error: 2.8410e-05 Mean Error: 1.1370e-05
// 32769 -32706 31252 -10589
// 32590 -30635 22793 -4493
// 32066 -27505 16481 -2348
// 31205 -24054 11849 -1306
static inline HVX_Vector hvx_vec_recip_xp1_O3_unsigned(HVX_Vector vx) {
// input is 0..0xffff representing 0.0 .. 1.0
HVX_Vector p;
p = Q6_Vh_vlut4_VuhPh(vx, 0xFAE6F6D4EE73D6A3ull);
p = Q6_Vh_vmpa_VhVhVuhPuh_sat(p, vx, 0x2E49406159097A14ull);
p = Q6_Vh_vmps_VhVhVuhPuh_sat(p, vx, 0x5DF66B7177AB7FC2ull);
p = Q6_Vh_vmpa_VhVhVuhPuh_sat(p, vx, 0x79E57D427F4E8001ull);
return p; // signed result, 14 fractional bits
}
// Find reciprocal of fp16.
// (1) first, convert to fp32, multiplying by 1.0; this is done to
// handle denormals. Ignoring sign and zero, result should be at
// least 5.9604645e-08 (32-bit code 0x33800000) and at most 131008 (0x47ffe000)
// (exponent in range [103,143])
// (2) extract the mantissa into 16-bit unsigned; find reciprocal using a fitted poly
// (3) put this, along with '253-exp' (exp from (1)) together to make an qf32
// (4) convert that to fp16
// (5) put sign back in. Also, if the original value (w/o sign) was <0x81, replace
// the result with the max value.
static inline HVX_Vector hvx_vec_inverse_f16(HVX_Vector vals) {
HVX_Vector em_mask = Q6_Vh_vsplat_R(0x7FFF);
HVX_Vector avals = Q6_V_vand_VV(vals, em_mask);
HVX_VectorPred is_neg = Q6_Q_vcmp_gt_VhVh(avals, vals);
// is too small to 1/x ? for 'standard' fp16, this would be 0x101
HVX_VectorPred is_small = Q6_Q_vcmp_gt_VhVh(Q6_Vh_vsplat_R(0x101), avals);
HVX_VectorPair to_qf32 = Q6_Wqf32_vmpy_VhfVhf(avals, Q6_Vh_vsplat_R(0x3C00)); // *1.0
HVX_Vector to_f32_0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(to_qf32));
HVX_Vector to_f32_1 = Q6_Vsf_equals_Vqf32(Q6_V_hi_W(to_qf32));
// bits 22..13 contain the mantissa now (w/o hidden bit); move to bit 14..5 of a 16-bit vector
HVX_Vector mant_u16 = Q6_Vh_vshuffo_VhVh(Q6_Vw_vasl_VwR(to_f32_1, 9), Q6_Vw_vasl_VwR(to_f32_0, 9));
// likewise extract the upper 16 from each, containing the exponents in range 103..142
HVX_Vector exp_u16 = Q6_Vh_vshuffo_VhVh(to_f32_1, to_f32_0);
//Get exponent in IEEE 32-bit representation
exp_u16 = Q6_Vuh_vlsr_VuhR(exp_u16, 7);
// so, mant_u16 contains an unbiased mantissa in upper 10 bits of each u16 lane
// We can consider it to be x-1.0, with 16 fractional bits, where 'x' is in range [1.0,2.0)
// Use poly to transform to 1/x, with 14 fractional bits
//
HVX_Vector rm = hvx_vec_recip_xp1_O3_unsigned(mant_u16);
HVX_Vector vcl0 = Q6_Vuh_vcl0_Vuh(rm); //count leading zeros
// Get mantissa for 16-bit representation
HVX_Vector mant_recip = Q6_V_vand_VV(Q6_Vh_vasr_VhR(Q6_Vh_vasl_VhVh(rm, vcl0), 5), Q6_Vh_vsplat_R(0x03FF));
//Compute Reciprocal Exponent
HVX_Vector exp_recip =
Q6_Vh_vsub_VhVh(Q6_Vh_vsub_VhVh(Q6_Vh_vsplat_R(254), exp_u16), Q6_Vh_vsub_VhVh(vcl0, Q6_Vh_vsplat_R(1)));
//Convert it for 16-bit representation
exp_recip = Q6_Vh_vadd_VhVh_sat(Q6_Vh_vsub_VhVh(exp_recip, Q6_Vh_vsplat_R(127)), Q6_Vh_vsplat_R(15));
exp_recip = Q6_Vh_vasl_VhR(exp_recip, 10);
//Merge exponent and mantissa for reciprocal
HVX_Vector recip = Q6_V_vor_VV(exp_recip, mant_recip);
// map 'small' inputs to standard largest value 0x7bff
recip = Q6_V_vmux_QVV(is_small, Q6_Vh_vsplat_R(0x7bff), recip);
// add sign back
recip = Q6_V_vandor_VQR(recip, is_neg, 0x80008000);
return recip;
}
static inline HVX_Vector hvx_vec_inverse_f32(HVX_Vector v_sf) {
HVX_Vector inv_aprox_sf = Q6_V_vsplat_R(0x7EEEEBB3);
HVX_Vector two_sf = hvx_vec_splat_f32(2.0);
// First approximation
HVX_Vector i_sf = Q6_Vw_vsub_VwVw(inv_aprox_sf, v_sf);
HVX_Vector r_qf;
// Refine
r_qf = Q6_Vqf32_vmpy_VsfVsf(
i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v_sf)))));
r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v_sf))));
r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32(
r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v_sf))));
return Q6_Vsf_equals_Vqf32(r_qf);
}
static inline HVX_Vector hvx_vec_inverse_f32_guard(HVX_Vector v_sf, HVX_Vector nan_inf_mask) {
HVX_Vector out = hvx_vec_inverse_f32(v_sf);
HVX_Vector masked_out = Q6_V_vand_VV(out, nan_inf_mask);
const HVX_VectorPred pred = Q6_Q_vcmp_eq_VwVw(nan_inf_mask, masked_out);
return Q6_V_vmux_QVV(pred, Q6_V_vzero(), out);
}
#define hvx_inverse_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const HVX_Vector nan_inf_mask = Q6_V_vsplat_R(0x7f800000); \
\
const uint32_t nvec = n / VLEN_FP32; \
const uint32_t nloe = n % VLEN_FP32; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_inverse_f32_guard(vsrc[i], nan_inf_mask); \
} \
if (nloe) { \
HVX_Vector v = hvx_vec_inverse_f32_guard(vsrc[i], nan_inf_mask); \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP32, v); \
} \
} while(0)
static inline HVX_Vector hvx_vec_inverse_f16_guard(HVX_Vector v_sf, HVX_Vector nan_inf_mask) {
HVX_Vector out = hvx_vec_inverse_f16(v_sf);
HVX_Vector masked_out = Q6_V_vand_VV(out, nan_inf_mask);
const HVX_VectorPred pred = Q6_Q_vcmp_eq_VhVh(nan_inf_mask, masked_out);
return Q6_V_vmux_QVV(pred, Q6_V_vzero(), out);
}
#define hvx_inverse_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const HVX_Vector nan_inf_mask = Q6_Vh_vsplat_R(0x7c00); \
\
const uint32_t nvec = n / VLEN_FP16; \
const uint32_t nloe = n % VLEN_FP16; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_inverse_f16_guard(vsrc[i], nan_inf_mask); \
} \
if (nloe) { \
HVX_Vector v = hvx_vec_inverse_f16_guard(vsrc[i], nan_inf_mask); \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \
} \
} while(0)
// Generic macro to define alignment permutations for an op
#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \
static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \
assert((uintptr_t) src % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \
} \
static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \
OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \
} \
// Dispatcher logic
#define HVX_INV_DISPATCHER(OP_NAME) \
static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { \
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \
OP_NAME##_aa(dst, src, num_elems); \
} else if (hex_is_aligned((void *) dst, 128)) { \
OP_NAME##_au(dst, src, num_elems); \
} else if (hex_is_aligned((void *) src, 128)) { \
OP_NAME##_ua(dst, src, num_elems); \
} else { \
OP_NAME##_uu(dst, src, num_elems); \
} \
}
DEFINE_HVX_INV_OP_VARIANTS(hvx_inverse_f32, hvx_inverse_f32_loop_body)
DEFINE_HVX_INV_OP_VARIANTS(hvx_inverse_f16, hvx_inverse_f16_loop_body)
HVX_INV_DISPATCHER(hvx_inverse_f32)
HVX_INV_DISPATCHER(hvx_inverse_f16)
#endif // HVX_INVERSE_H
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-log.h
+118
View File
@@ -0,0 +1,118 @@
#ifndef HVX_LOG_H
#define HVX_LOG_H
#include "hvx-base.h"
// Approximates ln(x) element-wise for float vectors.
// x must contain positive float elements.
// Uses Abramowitz & Stegun polynomial approximation 4.1.44 for ln(1+y) over [0, 1].
static inline HVX_Vector hvx_vec_log_f32(HVX_Vector x) {
// x = m * 2^e, where m in [1, 2)
HVX_Vector biased_e = Q6_Vuw_vlsr_VuwR(x, 23);
HVX_Vector e_int = Q6_Vw_vsub_VwVw(biased_e, Q6_V_vsplat_R(127));
HVX_Vector e_float = Q6_Vsf_equals_Vw(e_int);
// Extract mantissa and set exponent to 127 (which represents float value in [1.0, 2.0))
HVX_Vector mant_mask = Q6_V_vsplat_R(0x007FFFFF);
HVX_Vector exp_127 = Q6_V_vsplat_R(0x3F800000);
HVX_Vector m = Q6_V_vor_VV(Q6_V_vand_VV(x, mant_mask), exp_127);
// y = m - 1.0f, y in [0, 1)
HVX_Vector y = hvx_vec_sub_f32_f32(m, hvx_vec_splat_f32(1.0f));
// Abramowitz & Stegun 4.1.44 polynomial approximation of ln(1+y)
HVX_Vector c;
HVX_Vector res;
c = hvx_vec_splat_f32(-0.0064535442f);
res = hvx_vec_mul_f32_f32(y, c);
c = hvx_vec_splat_f32(0.0360884937f);
res = hvx_vec_add_f32_f32(res, c);
res = hvx_vec_mul_f32_f32(y, res);
c = hvx_vec_splat_f32(-0.0953293897f);
res = hvx_vec_add_f32_f32(res, c);
res = hvx_vec_mul_f32_f32(y, res);
c = hvx_vec_splat_f32(0.1676540711f);
res = hvx_vec_add_f32_f32(res, c);
res = hvx_vec_mul_f32_f32(y, res);
c = hvx_vec_splat_f32(-0.2407338084f);
res = hvx_vec_add_f32_f32(res, c);
res = hvx_vec_mul_f32_f32(y, res);
c = hvx_vec_splat_f32(0.3317990258f);
res = hvx_vec_add_f32_f32(res, c);
res = hvx_vec_mul_f32_f32(y, res);
c = hvx_vec_splat_f32(-0.4998741238f);
res = hvx_vec_add_f32_f32(res, c);
res = hvx_vec_mul_f32_f32(y, res);
c = hvx_vec_splat_f32(0.9999964239f);
res = hvx_vec_add_f32_f32(res, c);
res = hvx_vec_mul_f32_f32(y, res);
// ln(x) = e * ln(2) + ln(1+y)
HVX_Vector ln2 = hvx_vec_splat_f32(0.69314718056f);
HVX_Vector term_e = hvx_vec_mul_f32_f32(e_float, ln2);
return hvx_vec_add_f32_f32(term_e, res);
}
static inline void hvx_log_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t elem_size = sizeof(float);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
vdst[i] = hvx_vec_log_f32(vsrc[i]);
}
if (nloe) {
HVX_Vector v = hvx_vec_log_f32(vsrc[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v);
}
}
// Compute log(x) for f16 by promoting to f32, applying hvx_vec_log_f32, and narrowing back.
static inline void hvx_log_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc = (HVX_Vector *) src;
const uint32_t nvec = n / VLEN_FP16;
const uint32_t nloe = n % VLEN_FP16;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p));
HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p));
vdst[i] = hvx_vec_f32_to_f16(r0, r1);
}
if (nloe) {
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]);
HVX_Vector r0 = hvx_vec_log_f32(Q6_V_lo_W(p));
HVX_Vector r1 = hvx_vec_log_f32(Q6_V_hi_W(p));
HVX_Vector v = hvx_vec_f32_to_f16(r0, r1);
hvx_vec_store_a((void *) &vdst[i], nloe * SIZEOF_FP16, v);
}
}
#endif /* HVX_LOG_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-float.h
+575
View File
@@ -0,0 +1,575 @@
#ifndef HVX_MM_KERNELS_FLOAT_H
#define HVX_MM_KERNELS_FLOAT_H
#include "hvx-utils.h"
#include "htp-tensor.h"
// Float activation copy/quantization kernels (DDR -> VTCM)
static inline void quantize_f32_f32_kernel(
const uint8_t * restrict src_data,
uint8_t * restrict dst_data,
uint8_t * restrict tmp_data,
uint32_t ne0,
uint32_t nrows,
size_t src_stride,
size_t dst_stride
) {
(void) tmp_data;
const size_t src_row_size = ne0 * sizeof(float);
for (uint32_t i = 0; i < nrows; ++i) {
hex_l2fetch(src_data, src_row_size, src_stride, 2);
hvx_copy_f32_au(dst_data, src_data, ne0);
dst_data += dst_stride;
src_data += src_stride;
}
}
static inline void quantize_f32_f16_kernel(
const uint8_t * restrict src_data,
uint8_t * restrict dst_data,
uint8_t * restrict tmp_data,
uint32_t ne0,
uint32_t nrows,
size_t src_stride,
size_t dst_stride
) {
(void) tmp_data;
const size_t src_row_size = ne0 * sizeof(float);
for (uint32_t i = 0; i < nrows; ++i) {
hex_l2fetch(src_data, src_row_size, src_stride, 2);
hvx_copy_f16_f32_au(dst_data, src_data, ne0);
dst_data += dst_stride;
src_data += src_stride;
}
}
static inline void quantize_f16_f16_kernel(
const uint8_t * restrict src_data,
uint8_t * restrict dst_data,
uint8_t * restrict tmp_data,
uint32_t ne0,
uint32_t nrows,
size_t src_stride,
size_t dst_stride
) {
(void) tmp_data;
const size_t src_row_size = ne0 * sizeof(float);
for (uint32_t i = 0; i < nrows; ++i) {
hex_l2fetch(src_data, src_row_size, src_stride, 2);
hvx_copy_f16_au(dst_data, src_data, ne0);
dst_data += dst_stride;
src_data += src_stride;
}
}
// Float dot product kernels (HVX)
#if __HVX_ARCH__ < 79
#define HVX_OP_ADD_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a, b))
#define HVX_OP_MUL_F32(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b))
#else
#define HVX_OP_ADD_F32(a, b) Q6_Vsf_vadd_VsfVsf(a, b)
#define HVX_OP_MUL_F32(a, b) Q6_Vsf_vmpy_VsfVsf(a, b)
#endif
static inline void flat_vec_dot_f32_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) {
const HVX_Vector * restrict x = (const HVX_Vector *) vx;
const HVX_Vector * restrict y = (const HVX_Vector *) vy;
uint32_t nvec = n / VLEN_FP32; // num full fp32 hvx vectors
uint32_t nloe = n % VLEN_FP32; // leftover elements
HVX_Vector rsum = Q6_V_vzero();
uint32_t i = 0;
#pragma unroll(4)
for (i = 0; i < nvec; i++) {
HVX_Vector prod = HVX_OP_MUL_F32(x[i], y[i]);
rsum = HVX_OP_ADD_F32(rsum, prod);
}
if (nloe) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector x_sf = Q6_V_vand_QV(bmask, x[i]);
HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]);
HVX_Vector prod = HVX_OP_MUL_F32(x_sf, y_sf);
rsum = HVX_OP_ADD_F32(rsum, prod);
}
*s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum));
}
static inline void flat_vec_dot_f32_f32_aa_2x1(const uint32_t n, float * restrict s0,
const void * restrict vx0, const void * restrict vx1,
const void * restrict vy0) {
const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;
const HVX_Vector * restrict y = (const HVX_Vector *) vy0;
uint32_t nvec = n / VLEN_FP32;
uint32_t nloe = n % VLEN_FP32;
HVX_Vector rsum0 = Q6_V_vzero();
HVX_Vector rsum1 = Q6_V_vzero();
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; i++) {
HVX_Vector y_sf = y[i];
HVX_Vector prod0 = HVX_OP_MUL_F32(x0[i], y_sf);
HVX_Vector prod1 = HVX_OP_MUL_F32(x1[i], y_sf);
rsum0 = HVX_OP_ADD_F32(rsum0, prod0);
rsum1 = HVX_OP_ADD_F32(rsum1, prod1);
}
if (nloe) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector y_sf = Q6_V_vand_QV(bmask, y[i]);
HVX_Vector x0_sf = Q6_V_vand_QV(bmask, x0[i]);
HVX_Vector x1_sf = Q6_V_vand_QV(bmask, x1[i]);
HVX_Vector prod0 = HVX_OP_MUL_F32(x0_sf, y_sf);
HVX_Vector prod1 = HVX_OP_MUL_F32(x1_sf, y_sf);
rsum0 = HVX_OP_ADD_F32(rsum0, prod0);
rsum1 = HVX_OP_ADD_F32(rsum1, prod1);
}
HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1);
hvx_vec_store_u(s0, 8, rsum);
}
static inline void flat_vec_dot_f32_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1,
const void * restrict vx0, const void * restrict vx1,
const void * restrict vy0, const void * restrict vy1) {
const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;
const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0;
const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1;
uint32_t nvec = n / VLEN_FP32;
uint32_t nloe = n % VLEN_FP32;
HVX_Vector r0_c0_sum = Q6_V_vzero();
HVX_Vector r0_c1_sum = Q6_V_vzero();
HVX_Vector r1_c0_sum = Q6_V_vzero();
HVX_Vector r1_c1_sum = Q6_V_vzero();
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; i++) {
HVX_Vector r0_sf = x0[i];
HVX_Vector r1_sf = x1[i];
HVX_Vector c0_sf = y0[i];
HVX_Vector c1_sf = y1[i];
r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf));
r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf));
r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf));
r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf));
}
if (nloe) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector r0_sf = Q6_V_vand_QV(bmask, x0[i]);
HVX_Vector r1_sf = Q6_V_vand_QV(bmask, x1[i]);
HVX_Vector c0_sf = Q6_V_vand_QV(bmask, y0[i]);
HVX_Vector c1_sf = Q6_V_vand_QV(bmask, y1[i]);
r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(r0_sf, c0_sf));
r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(r0_sf, c1_sf));
r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(r1_sf, c0_sf));
r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(r1_sf, c1_sf));
}
// Reduce and store results
HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum);
HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum);
hvx_vec_store_u(s0, 8, r0_r1_c0_sum);
hvx_vec_store_u(s1, 8, r0_r1_c1_sum);
}
static inline void flat_vec_dot_bf16_f32_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) {
const HVX_Vector * restrict x = (const HVX_Vector *) vx;
const HVX_Vector * restrict y = (const HVX_Vector *) vy;
uint32_t nvec = n / 64;
uint32_t rem = n % 64;
HVX_Vector rsum0 = Q6_V_vzero();
HVX_Vector rsum1 = Q6_V_vzero();
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; i++) {
HVX_Vector v_bf16 = x[i];
HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16);
HVX_Vector w0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16);
HVX_Vector w1_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p), 16);
rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y[2 * i + 0]));
rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w1_f32, y[2 * i + 1]));
}
if (rem) {
HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2);
HVX_Vector v_bf16 = Q6_V_vand_QV(bmask, x[i]);
HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16);
HVX_Vector w0_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16);
HVX_Vector w1_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p), 16);
if (rem <= 32) {
HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4);
HVX_Vector y0 = Q6_V_vand_QV(a_mask, y[2 * i + 0]);
rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y0));
} else {
HVX_Vector y0 = y[2 * i + 0];
rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w0_f32, y0));
HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4);
HVX_Vector y1 = Q6_V_vand_QV(a_mask, y[2 * i + 1]);
rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w1_f32, y1));
}
}
HVX_Vector rsum = HVX_OP_ADD_F32(rsum0, rsum1);
*s = hvx_vec_get_f32(hvx_vec_reduce_sum_f32(rsum));
}
static inline void flat_vec_dot_bf16_f32_aa_2x1(const uint32_t n, float * restrict s0,
const void * restrict vx0, const void * restrict vx1,
const void * restrict vy0) {
const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;
const HVX_Vector * restrict y = (const HVX_Vector *) vy0;
uint32_t nvec = n / 64;
uint32_t rem = n % 64;
HVX_Vector rsum0 = Q6_V_vzero();
HVX_Vector rsum1 = Q6_V_vzero();
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; i++) {
HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(x0[i]);
HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(x1[i]);
HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16);
HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16);
HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16);
HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16);
HVX_Vector y0 = y[2 * i + 0];
HVX_Vector y1 = y[2 * i + 1];
rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0));
rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0));
rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w01_f32, y1));
rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w11_f32, y1));
}
if (rem) {
HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2);
HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x0[i]));
HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x1[i]));
HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16);
HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16);
HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16);
HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16);
if (rem <= 32) {
HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4);
HVX_Vector y0 = Q6_V_vand_QV(a_mask, y[2 * i + 0]);
rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0));
rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0));
} else {
HVX_Vector y0 = y[2 * i + 0];
rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w00_f32, y0));
rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w10_f32, y0));
HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4);
HVX_Vector y1 = Q6_V_vand_QV(a_mask, y[2 * i + 1]);
rsum0 = HVX_OP_ADD_F32(rsum0, HVX_OP_MUL_F32(w01_f32, y1));
rsum1 = HVX_OP_ADD_F32(rsum1, HVX_OP_MUL_F32(w11_f32, y1));
}
}
HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1);
hvx_vec_store_u(s0, 8, rsum);
}
static inline void flat_vec_dot_bf16_f32_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1,
const void * restrict vx0, const void * restrict vx1,
const void * restrict vy0, const void * restrict vy1) {
const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;
const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0;
const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1;
uint32_t nvec = n / 64;
uint32_t rem = n % 64;
HVX_Vector r0_c0_sum = Q6_V_vzero();
HVX_Vector r0_c1_sum = Q6_V_vzero();
HVX_Vector r1_c0_sum = Q6_V_vzero();
HVX_Vector r1_c1_sum = Q6_V_vzero();
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; i++) {
HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(x0[i]);
HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(x1[i]);
HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16);
HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16);
HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16);
HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16);
HVX_Vector c0_0 = y0[2 * i + 0];
HVX_Vector c0_1 = y0[2 * i + 1];
HVX_Vector c1_0 = y1[2 * i + 0];
HVX_Vector c1_1 = y1[2 * i + 1];
r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0));
r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0));
r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0));
r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0));
r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w01_f32, c0_1));
r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w01_f32, c1_1));
r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w11_f32, c0_1));
r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w11_f32, c1_1));
}
if (rem) {
HVX_VectorPred bmask = Q6_Q_vsetq2_R(rem * 2);
HVX_VectorPair p0 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x0[i]));
HVX_VectorPair p1 = Q6_Ww_vunpack_Vh(Q6_V_vand_QV(bmask, x1[i]));
HVX_Vector w00_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p0), 16);
HVX_Vector w01_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p0), 16);
HVX_Vector w10_f32 = Q6_Vw_vasl_VwR(Q6_V_lo_W(p1), 16);
HVX_Vector w11_f32 = Q6_Vw_vasl_VwR(Q6_V_hi_W(p1), 16);
if (rem <= 32) {
HVX_VectorPred a_mask = Q6_Q_vsetq2_R(rem * 4);
HVX_Vector c0_0 = Q6_V_vand_QV(a_mask, y0[2 * i + 0]);
HVX_Vector c1_0 = Q6_V_vand_QV(a_mask, y1[2 * i + 0]);
r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0));
r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0));
r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0));
r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0));
} else {
HVX_Vector c0_0 = y0[2 * i + 0];
HVX_Vector c1_0 = y1[2 * i + 0];
r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w00_f32, c0_0));
r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w00_f32, c1_0));
r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w10_f32, c0_0));
r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w10_f32, c1_0));
HVX_VectorPred a_mask = Q6_Q_vsetq2_R((rem - 32) * 4);
HVX_Vector c0_1 = Q6_V_vand_QV(a_mask, y0[2 * i + 1]);
HVX_Vector c1_1 = Q6_V_vand_QV(a_mask, y1[2 * i + 1]);
r0_c0_sum = HVX_OP_ADD_F32(r0_c0_sum, HVX_OP_MUL_F32(w01_f32, c0_1));
r0_c1_sum = HVX_OP_ADD_F32(r0_c1_sum, HVX_OP_MUL_F32(w01_f32, c1_1));
r1_c0_sum = HVX_OP_ADD_F32(r1_c0_sum, HVX_OP_MUL_F32(w11_f32, c0_1));
r1_c1_sum = HVX_OP_ADD_F32(r1_c1_sum, HVX_OP_MUL_F32(w11_f32, c1_1));
}
}
HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum);
HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum);
hvx_vec_store_u(s0, 8, r0_r1_c0_sum);
hvx_vec_store_u(s1, 8, r0_r1_c1_sum);
}
#undef HVX_OP_ADD_F32
#undef HVX_OP_MUL_F32
static inline void flat_vec_dot_f16_f16_aa_1x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy) {
const HVX_Vector * restrict x = (const HVX_Vector *) vx;
const HVX_Vector * restrict y = (const HVX_Vector *) vy;
uint32_t nvec = n / VLEN_FP16; // num full fp16 hvx vectors
uint32_t nloe = n % VLEN_FP16; // leftover elements
HVX_VectorPair rsum_p = Q6_W_vzero();
uint32_t i = 0;
#pragma unroll(4)
for (i = 0; i < nvec; i++) {
rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x[i], y[i]);
}
if (nloe) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);
HVX_Vector x_hf = Q6_V_vand_QV(bmask, x[i]);
HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]);
rsum_p = hvx_vec_mpyacc_f32_f16(rsum_p, x_hf, y_hf);
}
HVX_Vector rsum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum_p), Q6_V_hi_W(rsum_p)));
hvx_vec_store_u(s, 4, hvx_vec_reduce_sum_f32(rsum));
}
static inline void flat_vec_dot_f16_f16_aa_2x1(const uint32_t n, float * restrict s0,
const void * restrict vx0, const void * restrict vx1,
const void * restrict vy0) {
const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;
const HVX_Vector * restrict y = (const HVX_Vector *) vy0;
uint32_t nvec = n / VLEN_FP16;
uint32_t nloe = n % VLEN_FP16;
HVX_VectorPair rsum0_p = Q6_W_vzero();
HVX_VectorPair rsum1_p = Q6_W_vzero();
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; i++) {
HVX_Vector y_hf = y[i];
rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0[i], y_hf);
rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1[i], y_hf);
}
if (nloe) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);
HVX_Vector y_hf = Q6_V_vand_QV(bmask, y[i]);
HVX_Vector x0_hf = Q6_V_vand_QV(bmask, x0[i]);
HVX_Vector x1_hf = Q6_V_vand_QV(bmask, x1[i]);
rsum0_p = hvx_vec_mpyacc_f32_f16(rsum0_p, x0_hf, y_hf);
rsum1_p = hvx_vec_mpyacc_f32_f16(rsum1_p, x1_hf, y_hf);
}
HVX_Vector rsum0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum0_p), Q6_V_hi_W(rsum0_p)));
HVX_Vector rsum1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(rsum1_p), Q6_V_hi_W(rsum1_p)));
HVX_Vector rsum = hvx_vec_reduce_sum_f32x2(rsum0, rsum1);
hvx_vec_store_u(s0, 8, rsum);
}
static inline void flat_vec_dot_f16_f16_aa_2x2(const uint32_t n, float * restrict s0, float * restrict s1,
const void * restrict vx0, const void * restrict vx1,
const void * restrict vy0, const void * restrict vy1) {
const HVX_Vector * restrict x0 = (const HVX_Vector *) vx0;
const HVX_Vector * restrict x1 = (const HVX_Vector *) vx1;
const HVX_Vector * restrict y0 = (const HVX_Vector *) vy0;
const HVX_Vector * restrict y1 = (const HVX_Vector *) vy1;
uint32_t nvec = n / VLEN_FP16;
uint32_t nloe = n % VLEN_FP16;
// Row sums (sf) - 4 accumulators for 2x2 tile
HVX_VectorPair r0_c0_sum_p = Q6_W_vzero();
HVX_VectorPair r0_c1_sum_p = Q6_W_vzero();
HVX_VectorPair r1_c0_sum_p = Q6_W_vzero();
HVX_VectorPair r1_c1_sum_p = Q6_W_vzero();
uint32_t i = 0;
#pragma unroll(2)
for (i = 0; i < nvec; i++) {
HVX_Vector r0_hf = x0[i];
HVX_Vector r1_hf = x1[i];
HVX_Vector c0_hf = y0[i];
HVX_Vector c1_hf = y1[i];
// Compute 4 dot products: r0xc0, r0xc1, r1xc0, r1xc1
r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf);
r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf);
r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf);
r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf);
}
if (nloe) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 2);
HVX_Vector r0_hf = Q6_V_vand_QV(bmask, x0[i]);
HVX_Vector r1_hf = Q6_V_vand_QV(bmask, x1[i]);
HVX_Vector c0_hf = Q6_V_vand_QV(bmask, y0[i]);
HVX_Vector c1_hf = Q6_V_vand_QV(bmask, y1[i]);
r0_c0_sum_p = hvx_vec_mpyacc_f32_f16(r0_c0_sum_p, r0_hf, c0_hf);
r0_c1_sum_p = hvx_vec_mpyacc_f32_f16(r0_c1_sum_p, r0_hf, c1_hf);
r1_c0_sum_p = hvx_vec_mpyacc_f32_f16(r1_c0_sum_p, r1_hf, c0_hf);
r1_c1_sum_p = hvx_vec_mpyacc_f32_f16(r1_c1_sum_p, r1_hf, c1_hf);
}
HVX_Vector r0_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c0_sum_p), Q6_V_hi_W(r0_c0_sum_p)));
HVX_Vector r0_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r0_c1_sum_p), Q6_V_hi_W(r0_c1_sum_p)));
HVX_Vector r1_c0_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c0_sum_p), Q6_V_hi_W(r1_c0_sum_p)));
HVX_Vector r1_c1_sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(r1_c1_sum_p), Q6_V_hi_W(r1_c1_sum_p)));
// Reduce and store results
HVX_Vector r0_r1_c0_sum = hvx_vec_reduce_sum_f32x2(r0_c0_sum, r1_c0_sum);
HVX_Vector r0_r1_c1_sum = hvx_vec_reduce_sum_f32x2(r0_c1_sum, r1_c1_sum);
hvx_vec_store_u(&s0[0], 8, r0_r1_c0_sum); // row0,col0 row1,col0
hvx_vec_store_u(&s1[0], 8, r0_r1_c1_sum); // row0,col1 row1,col1
}
static inline void hvx_tensor_add_f32_grid(
const struct htp_tensor * restrict dst,
const struct htp_tensor * restrict src2,
uint32_t start_row,
uint32_t end_row,
uint32_t start_col,
uint32_t end_col,
const struct fastdiv_values * div_ne11_12,
const struct fastdiv_values * div_ne11
) {
if (start_row >= end_row || start_col >= end_col) return;
const uint32_t nb1 = dst->nb[1]; // row stride in bytes
const uint32_t ne11 = dst->ne[1];
const uint32_t ne12 = dst->ne[2];
const uint32_t ne11_12 = ne11 * ne12;
const bool is_broadcast1 = (src2->ne[1] == 1);
const bool is_broadcast2 = (src2->ne[2] == 1);
const bool is_broadcast3 = (src2->ne[3] == 1);
for (uint32_t r = start_row; r < end_row; r++) {
float * dst_row = (float *) ((uint8_t *) dst->data + (size_t) r * nb1);
uint32_t i13 = fastdiv(r, div_ne11_12);
uint32_t i12 = fastdiv(r - i13 * ne11_12, div_ne11);
uint32_t i11 = r - i13 * ne11_12 - i12 * ne11;
uint32_t i23 = is_broadcast3 ? 0 : i13;
uint32_t i22 = is_broadcast2 ? 0 : i12;
uint32_t i21 = is_broadcast1 ? 0 : i11;
const float * src2_row = (const float *) ((const uint8_t *) src2->data +
(size_t) i21 * src2->nb[1] + (size_t) i22 * src2->nb[2] + (size_t) i23 * src2->nb[3]);
float * dst_ptr = &dst_row[start_col];
const float * src2_ptr = &src2_row[start_col];
int remaining = end_col - start_col;
while (remaining >= 32) {
HVX_Vector v_out = hvx_vmemu(dst_ptr);
HVX_Vector v_z = hvx_vmemu(src2_ptr);
hvx_vmemu(dst_ptr) = hvx_vec_add_f32_f32(v_out, v_z);
dst_ptr += 32;
src2_ptr += 32;
remaining -= 32;
}
if (remaining > 0) {
HVX_Vector v_out = hvx_vmemu(dst_ptr);
HVX_Vector v_z = hvx_vmemu(src2_ptr);
hvx_vec_store_u(dst_ptr, remaining * sizeof(float), hvx_vec_add_f32_f32(v_out, v_z));
}
}
}
#endif // HVX_MM_KERNELS_FLOAT_H
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
+2101
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-norm.h
+462
View File
@@ -0,0 +1,462 @@
#ifndef HVX_NORM_H
#define HVX_NORM_H
#include <stdint.h>
#include "hvx-base.h"
#include "hvx-reduce.h"
#include "hvx-inverse.h"
#include "hvx-sqrt.h"
#include "hvx-repl.h"
static inline void hvx_fast_rms_norm_f32(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
float epsilon,
float scale) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
const int nvec = num_elems / VLEN_FP32; // number of full vectors
const int nloe = num_elems % VLEN_FP32; // leftover elements
// Compute sum of squares for full vectors
HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_Vector v1 = v_src[i];
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);
}
// Handle tail elements using vectorized ops with masking
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);
}
// Reduce HVX sum
sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);
HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);
HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);
HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);
// Scale full vectors
HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));
if (scale != 0.0f) {
scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale)));
}
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_Vector v1 = v_src[i];
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);
v_dst[i] = Q6_Vsf_equals_Vqf32(v2);
}
// Handle tail elements using vectorized ops with masking
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);
HVX_Vector result = Q6_Vsf_equals_Vqf32(v2);
// Store with masking to avoid overwriting memory beyond the tensor
hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);
}
}
static inline void hvx_fast_rms_norm_mul_f32(const uint8_t * restrict src,
const uint8_t * restrict weight,
uint8_t * restrict dst,
const int num_elems,
float epsilon,
float scale) {
const HVX_Vector * restrict v_src = (const HVX_Vector *) src;
const HVX_Vector * restrict v_weight = (const HVX_Vector *) weight;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
const int nvec = num_elems / VLEN_FP32; // number of full vectors
const int nloe = num_elems % VLEN_FP32; // leftover elements
// Compute sum of squares for full vectors
HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_Vector v1 = v_src[i];
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);
}
// Handle tail elements using vectorized ops with masking
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, v2);
}
// Reduce HVX sum
sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);
HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);
HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);
HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);
// Scale and multiply
HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));
if (scale != 0.0f) {
scale_v = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(scale_v, hvx_vec_splat_f32(scale)));
}
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_Vector v1 = v_src[i];
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);
HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2);
HVX_Vector result = Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[i]);
v_dst[i] = Q6_Vsf_equals_Vqf32(result);
}
// Handle tail elements using vectorized ops with masking
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, scale_v);
HVX_Vector v3 = Q6_Vsf_equals_Vqf32(v2);
HVX_Vector result = Q6_Vqf32_vmpy_VsfVsf(v3, v_weight[nvec]);
HVX_Vector res_v = Q6_Vsf_equals_Vqf32(result);
// Store with masking to avoid overwriting memory beyond the tensor
hvx_vec_store_a(&v_dst[nvec], nloe * 4, res_v);
}
}
static inline void hvx_fast_norm_f32(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
float epsilon) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
const int nvec = num_elems / VLEN_FP32; // number of full vectors
const int nloe = num_elems % VLEN_FP32; // leftover elements
// Compute sum of squares and sum of values for full vectors
HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_Vector v1 = v_src[i];
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, v2);
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(v1, Q6_V_vzero()));
}
// Handle tail elements using vectorized ops with masking
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, v2);
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(v1, Q6_V_vzero()));
}
// Reduce HVX sums
sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v));
sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v));
HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);
HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);
HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v);
HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v);
HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v));
HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v);
HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v);
// scale = rsqrt(variance + epsilon), mean_x broadcast for subtraction
HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v));
HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v));
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_Vector v1 = v_src[i];
HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, mean_x_b);
HVX_Vector v3 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v2), scale_v);
v_dst[i] = Q6_Vsf_equals_Vqf32(v3);
}
// Handle tail elements using vectorized ops with masking
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_Vector v2 = Q6_Vqf32_vsub_VsfVsf(v1, mean_x_b);
HVX_Vector v3 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v2), scale_v);
HVX_Vector result = Q6_Vsf_equals_Vqf32(v3);
// Store with masking to avoid overwriting memory beyond the tensor
hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);
}
}
static inline void hvx_fast_l2_norm_f32(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
float epsilon) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
HVX_Vector sum_v = hvx_vec_splat_f32(0.0f);
const int nvec = num_elems / VLEN_FP32;
const int nloe = num_elems % VLEN_FP32;
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_Vector v1 = v_src[i];
HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq);
}
// Include tail elements in the sum-of-squares using a predicate mask
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_Vector sq = Q6_Vqf32_vmpy_VsfVsf(v1, v1);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, sq);
}
// Compute scale = 1/fmax(sqrt(sum), epsilon) entirely in HVX registers.
// hvx_vec_rsqrt_f32 + hvx_vec_inverse_f32 avoids scalar extraction.
HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf); // 1/sqrt(sum)
HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v); // sqrt(sum)
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v); // fmax(sqrt(sum), epsilon)
HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v); // 1/fmax(sqrt(sum), epsilon)
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_Vector v1 = v_src[i];
v_dst[i] = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v));
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * 4);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_Vector result = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v1, scale_v));
hvx_vec_store_a(&v_dst[nvec], nloe * 4, result);
}
}
// F16 norm kernels: reduce and scale in f32 (via promote/narrow), matching the
// precision-preserving pattern used by the flash-attn f16 kernels.
static inline void hvx_fast_rms_norm_f16(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
float epsilon) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
const int nvec = num_elems / VLEN_FP16; // number of full f16 vectors
const int nloe = num_elems % VLEN_FP16; // leftover elements
HVX_Vector sum_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
}
sum_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);
HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);
HVX_Vector mean_v = Q6_Vqf32_vmpy_VsfVsf(sum_v, denom_v);
HVX_Vector mean_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(mean_v, epsilon_v);
HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(mean_epsilon_v));
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
v_dst[i] = hvx_vec_f32_to_f16(r0, r1);
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
HVX_Vector result = hvx_vec_f32_to_f16(r0, r1);
hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result);
}
}
static inline void hvx_fast_norm_f16(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
float epsilon) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
const int nvec = num_elems / VLEN_FP16;
const int nloe = num_elems % VLEN_FP16;
HVX_Vector sum_sq_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector sum_x_v = Q6_V_vsplat_R(0x00000000);
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero()));
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero()));
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_sq_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_sq_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p0, Q6_V_vzero()));
sum_x_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_x_v, Q6_Vqf32_vadd_VsfVsf(p1, Q6_V_vzero()));
}
sum_sq_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_sq_v));
sum_x_v = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_x_v));
HVX_Vector t_v = hvx_vec_splat_f32((float) num_elems);
HVX_Vector denom_v = hvx_vec_inverse_f32(t_v);
HVX_Vector mean_sq_v = Q6_Vqf32_vmpy_VsfVsf(sum_sq_v, denom_v);
HVX_Vector mean_x_v = Q6_Vqf32_vmpy_VsfVsf(sum_x_v, denom_v);
HVX_Vector mean_x_sq_v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(mean_x_v), Q6_Vsf_equals_Vqf32(mean_x_v));
HVX_Vector var_v = Q6_Vqf32_vsub_Vqf32Vqf32(mean_sq_v, mean_x_sq_v);
HVX_Vector var_epsilon_v = Q6_Vqf32_vadd_Vqf32Vsf(var_v, epsilon_v);
HVX_Vector scale_v = hvx_vec_rsqrt_f32(Q6_Vsf_equals_Vqf32(var_epsilon_v));
HVX_Vector mean_x_b = hvx_vec_repl_f32(Q6_Vsf_equals_Vqf32(mean_x_v));
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b);
HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v));
v_dst[i] = hvx_vec_f32_to_f16(r0, r1);
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector d0 = Q6_Vqf32_vsub_VsfVsf(Q6_V_lo_W(p), mean_x_b);
HVX_Vector d1 = Q6_Vqf32_vsub_VsfVsf(Q6_V_hi_W(p), mean_x_b);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d0), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(d1), scale_v));
HVX_Vector result = hvx_vec_f32_to_f16(r0, r1);
hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result);
}
}
static inline void hvx_fast_l2_norm_f16(const uint8_t * restrict src,
uint8_t * restrict dst,
const int num_elems,
float epsilon) {
const HVX_Vector * restrict v_src = (HVX_Vector *) src;
HVX_Vector * restrict v_dst = (HVX_Vector *) dst;
const int nvec = num_elems / VLEN_FP16;
const int nloe = num_elems % VLEN_FP16;
HVX_Vector sum_v = hvx_vec_splat_f32(0.0f);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector p0 = Q6_V_lo_W(p);
HVX_Vector p1 = Q6_V_hi_W(p);
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p0, p0));
sum_v = Q6_Vqf32_vadd_Vqf32Vqf32(sum_v, Q6_Vqf32_vmpy_VsfVsf(p1, p1));
}
HVX_Vector sum_sf = hvx_vec_reduce_sum_f32(Q6_Vsf_equals_Vqf32(sum_v));
HVX_Vector rsqrt_v = hvx_vec_rsqrt_f32(sum_sf);
HVX_Vector sqrt_v = hvx_vec_inverse_f32(rsqrt_v);
HVX_Vector epsilon_v = hvx_vec_splat_f32(epsilon);
HVX_Vector denom_v = Q6_Vsf_vmax_VsfVsf(sqrt_v, epsilon_v);
HVX_Vector scale_v = hvx_vec_inverse_f32(denom_v);
#pragma unroll(4)
for (int i = 0; i < nvec; i++) {
HVX_VectorPair p = hvx_vec_f16_to_f32(v_src[i]);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
v_dst[i] = hvx_vec_f32_to_f16(r0, r1);
}
if (nloe > 0) {
HVX_VectorPred bmask = Q6_Q_vsetq_R(nloe * SIZEOF_FP16);
HVX_Vector v1 = Q6_V_vand_QV(bmask, v_src[nvec]);
HVX_VectorPair p = hvx_vec_f16_to_f32(v1);
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), scale_v));
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), scale_v));
HVX_Vector result = hvx_vec_f32_to_f16(r0, r1);
hvx_vec_store_a(&v_dst[nvec], nloe * SIZEOF_FP16, result);
}
}
#endif // HVX_NORM_H
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-pow.h
+42
View File
@@ -0,0 +1,42 @@
#ifndef HVX_POW_H
#define HVX_POW_H
#include <math.h>
#include "hvx-base.h"
#include "hvx-exp.h"
#include "hvx-log.h"
// Approximates base^exponent element-wise for float vectors.
// base must be a positive constant. exponent is an HVX f32 vector.
// Uses base^x = exp(x * ln(base)).
static inline HVX_Vector hvx_vec_pow_const_base_f32(float base, HVX_Vector exponent) {
float ln_base = logf(base);
HVX_Vector ln_base_v = hvx_vec_splat_f32(ln_base);
HVX_Vector x = hvx_vec_mul_f32_f32(exponent, ln_base_v);
static const float kInf = INFINITY;
static const float kMaxExp = 88.7228f;
const HVX_Vector max_exp = hvx_vec_splat_f32(kMaxExp);
const HVX_Vector inf = hvx_vec_splat_f32(kInf);
return hvx_vec_exp_f32_guard(x, max_exp, inf);
}
// Approximates base^exponent element-wise for float vectors.
// base and exponent are HVX f32 vectors. base elements must be positive.
// Uses base^exponent = exp(exponent * ln(base)).
static inline HVX_Vector hvx_vec_pow_f32(HVX_Vector base, HVX_Vector exponent) {
HVX_Vector ln_base = hvx_vec_log_f32(base);
HVX_Vector x = hvx_vec_mul_f32_f32(exponent, ln_base);
static const float kInf = INFINITY;
static const float kMaxExp = 88.7228f;
const HVX_Vector max_exp = hvx_vec_splat_f32(kMaxExp);
const HVX_Vector inf = hvx_vec_splat_f32(kInf);
return hvx_vec_exp_f32_guard(x, max_exp, inf);
}
#endif /* HVX_POW_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-quant.h
+726
View File
@@ -0,0 +1,726 @@
#ifndef HVX_QUANT_H
#define HVX_QUANT_H
#include <math.h>
#include <stdint.h>
#include <string.h>
#include "htp-tensor.h"
#include "hvx-arith.h"
#include "hvx-base.h"
#include "hvx-reduce.h"
#include "hvx-repl.h"
#include "hvx-utils.h"
#ifndef GGML_COMMON_DECL_C
#define GGML_COMMON_DECL_C
#endif
#include "ggml-common.h"
#include "ggml-impl.h"
static inline void hvx_quantize_row_q8_0_f32(void * restrict dst_ptr, const float * restrict src_ptr, int n) {
const int nb = n / QK8_0;
block_q8_0 * dst = (block_q8_0 *) dst_ptr;
HVX_Vector zero = Q6_V_vzero();
int i = 0;
for (; i + 3 < nb; i += 4) {
HVX_Vector * vx = (HVX_Vector *) (src_ptr + i * QK8_0);
HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0]));
HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1]));
HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2]));
HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3]));
HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero);
HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero);
HVX_Vector vx2_qf = Q6_Vqf32_vsub_VsfVsf(vx[2], zero);
HVX_Vector vx3_qf = Q6_Vqf32_vsub_VsfVsf(vx[3], zero);
HVX_Vector vmax0_qf = Q6_Vqf32_vsub_VsfVsf(vmax0_sf, zero);
HVX_Vector vmax1_qf = Q6_Vqf32_vsub_VsfVsf(vmax1_sf, zero);
HVX_Vector vmax2_qf = Q6_Vqf32_vsub_VsfVsf(vmax2_sf, zero);
HVX_Vector vmax3_qf = Q6_Vqf32_vsub_VsfVsf(vmax3_sf, zero);
HVX_Vector vmax01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax1_qf, vmax0_qf)));
HVX_Vector vmax23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vmax3_qf, vmax2_qf)));
HVX_Vector vx01_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx1_qf, vx0_qf)));
HVX_Vector vx23_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(vx3_qf, vx2_qf)));
HVX_Vector vd01_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax01_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
HVX_Vector vd23_qf16 = Q6_Vqf16_vmpy_VhfVhf(vmax23_hf, Q6_Vh_vsplat_R(0x2008)); // 1.0 / 127.0
HVX_Vector vd01_hf = Q6_Vhf_equals_Vqf16(vd01_qf16);
HVX_Vector vd23_hf = Q6_Vhf_equals_Vqf16(vd23_qf16);
HVX_Vector vd01_inv_hf = hvx_vec_inverse_f16(vd01_hf);
HVX_Vector vd23_inv_hf = hvx_vec_inverse_f16(vd23_hf);
vx01_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx01_hf, vd01_inv_hf));
vx23_hf = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(vx23_hf, vd23_inv_hf));
HVX_Vector vx01_i16 = hvx_vec_i16_from_hf_rnd_sat(vx01_hf);
HVX_Vector vx23_i16 = hvx_vec_i16_from_hf_rnd_sat(vx23_hf);
HVX_Vector vx_i8 = Q6_Vb_vpack_VhVh_sat(vx23_i16, vx01_i16);
hvx_vec_store_u(&dst[i + 0].d, 2, vd01_hf);
hvx_vec_store_u(dst[i + 0].qs, 32, vx_i8);
hvx_vec_store_u(&dst[i + 1].d, 2, Q6_V_vror_VR(vd01_hf, 64));
hvx_vec_store_u(dst[i + 1].qs, 32, Q6_V_vror_VR(vx_i8, 32));
hvx_vec_store_u(&dst[i + 2].d, 2, vd23_hf);
hvx_vec_store_u(dst[i + 2].qs, 32, Q6_V_vror_VR(vx_i8, 64));
hvx_vec_store_u(&dst[i + 3].d, 2, Q6_V_vror_VR(vd23_hf, 64));
hvx_vec_store_u(dst[i + 3].qs, 32, Q6_V_vror_VR(vx_i8, 96));
}
for (; i < nb; i++) {
const float * block_src = src_ptr + i * QK8_0;
HVX_Vector vx = *(const HVX_UVector *) block_src;
HVX_Vector v_abs = hvx_vec_abs_f32(vx);
HVX_Vector v_max = hvx_vec_reduce_max_f32(v_abs);
float amax = hvx_vec_get_f32(v_max);
const float d = amax / 127.0f;
const float id = d ? (1.0f / d) : 0.0f;
dst[i].d = GGML_FP32_TO_FP16(d);
HVX_Vector vid = hvx_vec_splat_f32(id);
HVX_Vector v_scaled = hvx_vec_mul_f32_f32(vx, vid);
HVX_Vector v_scaled_qf = Q6_Vqf32_vsub_VsfVsf(v_scaled, zero);
HVX_Vector v_scaled_hf = Q6_Vh_vdeal_Vh(Q6_Vhf_equals_Wqf32(Q6_W_vcombine_VV(zero, v_scaled_qf)));
HVX_Vector v_i16 = hvx_vec_i16_from_hf_rnd_sat(v_scaled_hf);
HVX_Vector v_i8 = Q6_Vb_vpack_VhVh_sat(zero, v_i16);
hvx_vec_store_u(dst[i].qs, 32, v_i8);
}
}
static inline void hvx_dequantize_row_flat_q8_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const int nb = n / QK8_0;
const block_q8_0 * src = (const block_q8_0 *) src_ptr;
for (int i = 0; i < nb; i++) {
HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
HVX_Vector vd = Q6_V_lo_W(vp_f32);
HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs;
HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8);
HVX_Vector v_i16 = Q6_V_lo_W(p16);
HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16);
HVX_Vector v_i32 = Q6_V_lo_W(p32);
HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
HVX_Vector res = hvx_vec_mul_f32_f32(v_f32, vd);
float * block_dst = dst_ptr + i * QK8_0;
hvx_vmem(block_dst) = res;
}
}
static inline void hvx_dequantize_row_flat_q8_0_f16(__fp16 * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const int nb = n / QK8_0;
const block_q8_0 * src = (const block_q8_0 *) src_ptr;
for (int i = nb - 1; i >= 0; i--) {
HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
HVX_Vector vd = Q6_V_lo_W(vp_f32);
HVX_Vector vq_i8 = *(const HVX_UVector *) src[i].qs;
HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq_i8);
HVX_Vector v_i16 = Q6_V_lo_W(p16);
HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16);
HVX_Vector v_i32 = Q6_V_lo_W(p32);
HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
HVX_Vector res_f32 = hvx_vec_mul_f32_f32(v_f32, vd);
HVX_Vector res_f16 = hvx_vec_f32_to_f16(res_f32, Q6_V_vzero());
__fp16 * block_dst = dst_ptr + i * QK8_0;
hvx_vec_store_u(block_dst, QK8_0 * sizeof(__fp16), res_f16);
}
}
static inline void hvx_dequantize_row_flat_f16_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const uint32_t nvec = n / VLEN_FP32;
const uint32_t nloe = n % VLEN_FP32;
const _Float16 * src = (const _Float16 *) src_ptr;
for (uint32_t i = 0; i < nvec; i++) {
HVX_Vector v_f16 = *(const HVX_UVector *) (src + i * VLEN_FP32);
HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16);
HVX_Vector res = Q6_V_lo_W(vp_f32);
hvx_vmem(dst_ptr + i * VLEN_FP32) = res;
}
if (nloe) {
HVX_Vector v_f16 = *(const HVX_UVector *) (src + nvec * VLEN_FP32);
HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(v_f16);
HVX_Vector res = Q6_V_lo_W(vp_f32);
hvx_vec_store_a(dst_ptr + nvec * VLEN_FP32, nloe * sizeof(float), res);
}
}
static inline void hvx_dequantize_row_flat_bf16_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const uint32_t nvec = n / VLEN_FP32;
const uint32_t nloe = n % VLEN_FP32;
const uint16_t * src = (const uint16_t *) src_ptr;
for (uint32_t i = 0; i < nvec; i++) {
HVX_Vector v_bf16 = *(const HVX_UVector *) (src + i * VLEN_FP32);
HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16);
HVX_Vector res = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16);
hvx_vmem(dst_ptr + i * VLEN_FP32) = res;
}
if (nloe) {
HVX_Vector v_bf16 = *(const HVX_UVector *) (src + nvec * VLEN_FP32);
HVX_VectorPair p = Q6_Ww_vunpack_Vh(v_bf16);
HVX_Vector res = Q6_Vw_vasl_VwR(Q6_V_lo_W(p), 16);
hvx_vec_store_a(dst_ptr + nvec * VLEN_FP32, nloe * sizeof(float), res);
}
}
static inline void hvx_dequantize_row_flat_q1_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
static const uint32_t __attribute__((aligned(128))) bit_mask[32] = {
1u << 0, 1u << 1, 1u << 2, 1u << 3, 1u << 4, 1u << 5, 1u << 6, 1u << 7,
1u << 8, 1u << 9, 1u << 10, 1u << 11, 1u << 12, 1u << 13, 1u << 14, 1u << 15,
1u << 16, 1u << 17, 1u << 18, 1u << 19, 1u << 20, 1u << 21, 1u << 22, 1u << 23,
1u << 24, 1u << 25, 1u << 26, 1u << 27, 1u << 28, 1u << 29, 1u << 30, 1u << 31,
};
const int nb = n / QK1_0;
const block_q1_0 * src = (const block_q1_0 *) src_ptr;
const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask;
const HVX_Vector v_zero = Q6_V_vzero();
const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000);
for (int i = 0; i < nb; i++) {
HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16));
HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit);
HVX_Vector v_qs = *(const HVX_UVector *) src[i].qs;
float * block_dst = dst_ptr + i * QK1_0;
#pragma unroll
for (int q = 0; q < 4; q++) {
HVX_Vector v_bits = hvx_vec_repl_u32(q == 0 ? v_qs : Q6_V_vror_VR(v_qs, 4 * q));
HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero);
hvx_vmem(block_dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd);
}
}
}
static inline void hvx_dequantize_row_flat_q2_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const int nb = n / QK2_0;
const block_q2_0 * src = (const block_q2_0 *) src_ptr;
const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
const HVX_Vector v_one = Q6_Vb_vsplat_R(1);
for (int i = 0; i < nb; i++) {
HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(vd_f16));
HVX_Vector vq = *(const HVX_UVector *) src[i].qs;
HVX_Vector v0 = Q6_V_vand_VV(vq, mask_03);
HVX_Vector v1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 2), mask_03);
HVX_Vector v2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vq, 4), mask_03);
HVX_Vector v3 = Q6_Vub_vlsr_VubR(vq, 6);
HVX_Vector v01 = Q6_V_lo_W(Q6_W_vshuff_VVR(v1, v0, -1));
HVX_Vector v23 = Q6_V_lo_W(Q6_W_vshuff_VVR(v3, v2, -1));
HVX_Vector v64 = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(v23, v01, -2)), v_one);
HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v64);
HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
float * block_dst = dst_ptr + i * QK2_0;
hvx_vmem(block_dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), vd);
hvx_vmem(block_dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), vd);
}
}
static inline void hvx_dequantize_row_flat_q4_0_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const int nb = n / QK4_0;
const block_q4_0 * src = (const block_q4_0 *) src_ptr;
const HVX_VectorPred q16 = Q6_Q_vsetq_R(16);
const HVX_Vector v_eight = Q6_Vb_vsplat_R(8);
const HVX_Vector v_zero = Q6_V_vzero();
for (int i = 0; i < nb; i++) {
HVX_Vector vd_f16 = Q6_Vh_vsplat_R(*(const int16_t *) &src[i].d);
HVX_VectorPair vp_f32 = hvx_vec_f16_to_f32(vd_f16);
HVX_Vector vd = Q6_V_lo_W(vp_f32);
HVX_Vector vq = *(const HVX_UVector *) src[i].qs;
HVX_Vector v_lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
HVX_Vector v_hi = Q6_Vub_vlsr_VubR(vq, 4);
HVX_Vector v_lo_s8 = Q6_Vb_vsub_VbVb(v_lo, v_eight);
HVX_Vector v_hi_s8 = Q6_Vb_vsub_VbVb(v_hi, v_eight);
HVX_Vector v32 = Q6_V_vmux_QVV(q16, v_lo_s8, Q6_V_vlalign_VVR(v_hi_s8, v_zero, 16));
HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(v32);
HVX_Vector v_i16 = Q6_V_lo_W(p16);
HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(v_i16);
HVX_Vector v_i32 = Q6_V_lo_W(p32);
HVX_Vector v_f32 = Q6_Vsf_equals_Vw(v_i32);
HVX_Vector res = hvx_vec_mul_f32_f32(v_f32, vd);
hvx_vmem(dst_ptr + i * QK4_0) = res;
}
}
static inline void hvx_dequantize_row_flat_q4_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const int nb = n / QK_K;
const block_q4_K * src = (const block_q4_K *) src_ptr;
for (int i = 0; i < nb; i++) {
const float d = GGML_FP16_TO_FP32(src[i].d);
const float min = GGML_FP16_TO_FP32(src[i].dmin);
const uint8_t * sc = src[i].scales;
float * y = dst_ptr + i * QK_K;
for (int step = 0; step < 4; ++step) {
uint8_t sc1, m1_u, sc2, m2_u;
if (step < 2) {
sc1 = sc[2 * step + 0] & 63;
m1_u = sc[2 * step + 4] & 63;
sc2 = sc[2 * step + 1] & 63;
m2_u = sc[2 * step + 5] & 63;
} else {
const int s = step - 2;
sc1 = (sc[8 + 2 * s + 0] & 0xF) | ((sc[2 * s + 0] >> 6) << 4);
m1_u = (sc[8 + 2 * s + 0] >> 4) | ((sc[4 + 2 * s + 0] >> 6) << 4);
sc2 = (sc[8 + 2 * s + 1] & 0xF) | ((sc[2 * s + 1] >> 6) << 4);
m2_u = (sc[8 + 2 * s + 1] >> 4) | ((sc[4 + 2 * s + 1] >> 6) << 4);
}
HVX_Vector vq = *(const HVX_UVector *) (src[i].qs + step * 32);
HVX_Vector v_lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
HVX_Vector v_hi = Q6_Vub_vlsr_VubR(vq, 4);
HVX_VectorPair p16_lo = Q6_Wuh_vunpack_Vub(v_lo);
HVX_VectorPair p32_lo = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_lo));
HVX_Vector v_i32_lo = Q6_V_lo_W(p32_lo);
HVX_Vector v_f32_lo = Q6_Vsf_equals_Vw(v_i32_lo);
HVX_Vector vd1 = hvx_vec_splat_f32(d * sc1);
HVX_Vector vm1 = hvx_vec_splat_f32(min * m1_u);
HVX_Vector v_out1 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_lo, vd1), vm1);
hvx_vmem(y + 0) = v_out1;
HVX_VectorPair p16_hi = Q6_Wuh_vunpack_Vub(v_hi);
HVX_VectorPair p32_hi = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_hi));
HVX_Vector v_i32_hi = Q6_V_lo_W(p32_hi);
HVX_Vector v_f32_hi = Q6_Vsf_equals_Vw(v_i32_hi);
HVX_Vector vd2 = hvx_vec_splat_f32(d * sc2);
HVX_Vector vm2 = hvx_vec_splat_f32(min * m2_u);
HVX_Vector v_out2 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_hi, vd2), vm2);
hvx_vmem(y + 32) = v_out2;
y += 64;
}
}
}
static inline void hvx_dequantize_row_flat_q5_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const int nb = n / QK_K;
const block_q5_K * src = (const block_q5_K *) src_ptr;
for (int i = 0; i < nb; i++) {
const float d = GGML_FP16_TO_FP32(src[i].d);
const float min = GGML_FP16_TO_FP32(src[i].dmin);
const uint8_t * sc = src[i].scales;
HVX_Vector v_qh = *(const HVX_UVector *) src[i].qh;
float * y = dst_ptr + i * QK_K;
for (int step = 0; step < 4; ++step) {
uint8_t sc1, m1_u, sc2, m2_u;
if (step < 2) {
sc1 = sc[2 * step + 0] & 63;
m1_u = sc[2 * step + 4] & 63;
sc2 = sc[2 * step + 1] & 63;
m2_u = sc[2 * step + 5] & 63;
} else {
const int s = step - 2;
sc1 = (sc[8 + 2 * s + 0] & 0xF) | ((sc[2 * s + 0] >> 6) << 4);
m1_u = (sc[8 + 2 * s + 0] >> 4) | ((sc[4 + 2 * s + 0] >> 6) << 4);
sc2 = (sc[8 + 2 * s + 1] & 0xF) | ((sc[2 * s + 1] >> 6) << 4);
m2_u = (sc[8 + 2 * s + 1] >> 4) | ((sc[4 + 2 * s + 1] >> 6) << 4);
}
HVX_Vector v_ql = *(const HVX_UVector *) (src[i].qs + step * 32);
HVX_Vector v_lo = Q6_V_vand_VV(v_ql, Q6_Vb_vsplat_R(0x0F));
HVX_Vector v_hi = Q6_Vub_vlsr_VubR(v_ql, 4);
HVX_Vector v_h1 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2 * step), Q6_Vb_vsplat_R(1)), 4);
HVX_Vector v_val1 = Q6_V_vor_VV(v_lo, v_h1);
HVX_VectorPair p16_1 = Q6_Wuh_vunpack_Vub(v_val1);
HVX_VectorPair p32_1 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_1));
HVX_Vector v_i32_1 = Q6_V_lo_W(p32_1);
HVX_Vector v_f32_1 = Q6_Vsf_equals_Vw(v_i32_1);
HVX_Vector vd1 = hvx_vec_splat_f32(d * sc1);
HVX_Vector vm1 = hvx_vec_splat_f32(min * m1_u);
HVX_Vector v_out1 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_1, vd1), vm1);
hvx_vmem(y + 0) = v_out1;
HVX_Vector v_h2 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2 * step + 1), Q6_Vb_vsplat_R(1)), 4);
HVX_Vector v_val2 = Q6_V_vor_VV(v_hi, v_h2);
HVX_VectorPair p16_2 = Q6_Wuh_vunpack_Vub(v_val2);
HVX_VectorPair p32_2 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_2));
HVX_Vector v_i32_2 = Q6_V_lo_W(p32_2);
HVX_Vector v_f32_2 = Q6_Vsf_equals_Vw(v_i32_2);
HVX_Vector vd2 = hvx_vec_splat_f32(d * sc2);
HVX_Vector vm2 = hvx_vec_splat_f32(min * m2_u);
HVX_Vector v_out2 = hvx_vec_sub_f32_f32(hvx_vec_mul_f32_f32(v_f32_2, vd2), vm2);
hvx_vmem(y + 32) = v_out2;
y += 64;
}
}
}
static inline void hvx_dequantize_row_flat_q6_k_f32(float * restrict dst_ptr, const void * restrict src_ptr, uint32_t n) {
const int nb = n / QK_K;
const block_q6_K * src = (const block_q6_K *) src_ptr;
const HVX_VectorPred q64 = Q6_Q_vsetq_R(64);
const HVX_Vector v32_b = Q6_Vb_vsplat_R(32);
const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);
const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
for (int i = 0; i < nb; i++) {
const float d = GGML_FP16_TO_FP32(src[i].d);
const uint8_t * restrict ql = src[i].ql;
const uint8_t * restrict qh = src[i].qh;
const int8_t * restrict sc = src[i].scales;
float * y = dst_ptr + i * QK_K;
for (int half = 0; half < 2; ++half) {
float s[8];
for (int j = 0; j < 8; ++j) {
s[j] = d * sc[j];
}
HVX_Vector vscale1 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[0]), hvx_vec_splat_f32(s[1]));
HVX_Vector vscale2 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[2]), hvx_vec_splat_f32(s[3]));
HVX_Vector vscale3 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[4]), hvx_vec_splat_f32(s[5]));
HVX_Vector vscale4 = Q6_V_vmux_QVV(q64, hvx_vec_splat_f32(s[6]), hvx_vec_splat_f32(s[7]));
HVX_Vector v_ql0 = *(const HVX_UVector *) (ql);
HVX_Vector v_ql1 = *(const HVX_UVector *) (ql + 32);
HVX_Vector v_qh = *(const HVX_UVector *) (qh);
HVX_Vector ql0_lo = Q6_V_vand_VV(v_ql0, mask_0f);
HVX_Vector qh_0 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(v_qh, mask_03), 4);
HVX_Vector v_q1 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql0_lo, qh_0), v32_b);
HVX_Vector ql1_lo = Q6_V_vand_VV(v_ql1, mask_0f);
HVX_Vector qh_1 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 2), mask_03), 4);
HVX_Vector v_q2 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql1_lo, qh_1), v32_b);
HVX_Vector ql0_hi = Q6_Vub_vlsr_VubR(v_ql0, 4);
HVX_Vector qh_2 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 4), mask_03), 4);
HVX_Vector v_q3 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql0_hi, qh_2), v32_b);
HVX_Vector ql1_hi = Q6_Vub_vlsr_VubR(v_ql1, 4);
HVX_Vector qh_3 = Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_qh, 6), mask_03), 4);
HVX_Vector v_q4 = Q6_Vb_vsub_VbVb(Q6_V_vor_VV(ql1_hi, qh_3), v32_b);
HVX_VectorPair p16_1 = Q6_Wh_vunpack_Vb(v_q1);
HVX_VectorPair p32_1 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_1));
HVX_Vector v_f32_1 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_1));
hvx_vmem(y + 0) = hvx_vec_mul_f32_f32(v_f32_1, vscale1);
HVX_VectorPair p16_2 = Q6_Wh_vunpack_Vb(v_q2);
HVX_VectorPair p32_2 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_2));
HVX_Vector v_f32_2 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_2));
hvx_vmem(y + 32) = hvx_vec_mul_f32_f32(v_f32_2, vscale2);
HVX_VectorPair p16_3 = Q6_Wh_vunpack_Vb(v_q3);
HVX_VectorPair p32_3 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_3));
HVX_Vector v_f32_3 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_3));
hvx_vmem(y + 64) = hvx_vec_mul_f32_f32(v_f32_3, vscale3);
HVX_VectorPair p16_4 = Q6_Wh_vunpack_Vb(v_q4);
HVX_VectorPair p32_4 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16_4));
HVX_Vector v_f32_4 = Q6_Vsf_equals_Vw(Q6_V_lo_W(p32_4));
hvx_vmem(y + 96) = hvx_vec_mul_f32_f32(v_f32_4, vscale4);
y += 128;
ql += 64;
qh += 32;
sc += 8;
}
}
}
// bytes row, row + 32, row + 64 and row + 96 of v, to bytes 0..3
static inline HVX_Vector hvx_dequantize_gather4(HVX_Vector v, uint32_t row) {
v = Q6_V_vror_VR(v, row);
HVX_Vector packed = Q6_V_vmux_QVV(Q6_Q_vsetq_R(1), v, Q6_V_vror_VR(v, 31));
packed = Q6_V_vmux_QVV(Q6_Q_vsetq_R(2), packed, Q6_V_vror_VR(v, 62));
return Q6_V_vmux_QVV(Q6_Q_vsetq_R(3), packed, Q6_V_vror_VR(v, 93));
}
static inline void hvx_dequantize_row_tiled_q8_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
const HVX_VectorPred first2 = Q6_Q_vsetq_R(2);
const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
const uint8_t * tile = src_spad + k_tile * tile_stride;
float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
HVX_Vector vq = Q6_V_vzero();
for (int group = 7; group >= 0; --group) {
const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row);
const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62));
vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
}
const HVX_Vector scales = hvx_vmem(tile + 1024);
const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
*(HVX_Vector *) dst = values;
}
}
static inline void hvx_dequantize_row_tiled_q4_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
const uint8_t * tile = src_spad + k_tile * tile_stride;
float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
HVX_Vector vq = Q6_V_vzero();
for (int group = 3; group >= 0; --group) {
vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4));
}
const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);
vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));
const HVX_Vector scales = hvx_vmem(tile + 512);
const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
*(HVX_Vector *) dst = values;
}
}
static inline void hvx_dequantize_row_tiled_q4_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
const uint8_t * tile = src_spad + k_tile * tile_stride;
float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
HVX_Vector vq = Q6_V_vzero();
for (int group = 3; group >= 0; --group) {
vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4));
}
const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);
vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
const HVX_Vector scales = hvx_vmem(tile + 512);
const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row));
const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2));
const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf));
const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
*(HVX_Vector *) dst = hvx_vec_add_f32_f32(values, offset);
}
}
static inline void hvx_dequantize_row_tiled_q5_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
const HVX_Vector one = Q6_Vb_vsplat_R(1);
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
const uint8_t * tile = src_spad + k_tile * tile_stride;
float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
const HVX_Vector p5 = hvx_dequantize_gather4(hvx_vmem(tile + 640), row);
HVX_Vector vq = Q6_V_vzero();
HVX_Vector vh = Q6_V_vzero();
for (int group = 3; group >= 0; --group) {
vq = Q6_V_vmux_QVV(first4, hvx_dequantize_gather4(hvx_vmem(tile + group * VLEN), row), Q6_V_vror_VR(vq, VLEN - 4));
vh = Q6_V_vmux_QVV(first4, Q6_Vub_vlsr_VubR(p5, 2 * group), Q6_V_vror_VR(vh, VLEN - 4));
}
const HVX_Vector lo = Q6_V_vor_VV(Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)), Q6_Vw_vasl_VwR(Q6_V_vand_VV(vh, one), 4));
const HVX_Vector hi = Q6_V_vor_VV(Q6_Vub_vlsr_VubR(vq, 4), Q6_Vw_vasl_VwR(Q6_V_vand_VV(Q6_Vub_vlsr_VubR(vh, 1), one), 4));
vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
const HVX_Vector scales = hvx_vmem(tile + 512);
const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row));
const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
const HVX_Vector offset_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 4 * row + 2));
const HVX_Vector offset = Q6_V_lo_W(hvx_vec_f16_to_f32(offset_hf));
const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
*(HVX_Vector *) dst = hvx_vec_add_f32_f32(values, offset);
}
}
static inline void hvx_dequantize_row_tiled_q6_k_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
const HVX_VectorPred first16 = Q6_Q_vsetq_R(16 * sizeof(float));
const HVX_Vector mask_0f = Q6_Vb_vsplat_R(0x0F);
const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
const uint8_t * tile = src_spad + k_tile * tile_stride;
float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
HVX_Vector vq = Q6_V_vzero();
for (int group = 7; group >= 0; --group) {
const HVX_Vector lo_plane = Q6_V_vror_VR(hvx_vmem(tile + (group >> 1) * VLEN), 4 * row);
const HVX_Vector hi_plane = Q6_V_vror_VR(hvx_vmem(tile + 512 + (group >> 2) * VLEN), 4 * row);
const HVX_Vector lo = (group & 1) ? Q6_Vub_vlsr_VubR(lo_plane, 4) : Q6_V_vand_VV(lo_plane, mask_0f);
const HVX_Vector hi = Q6_Vub_vlsr_VubR(hi_plane, 2 * (group & 3));
const HVX_Vector packed = Q6_V_vor_VV(lo, Q6_Vw_vasl_VwR(Q6_V_vand_VV(hi, mask_03), 4));
vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
}
const HVX_Vector scales = hvx_vmem(tile + 768);
const HVX_Vector scale_lo_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
const HVX_Vector scale_hi_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 64 + 2 * row));
const HVX_Vector scale_lo = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_lo_hf));
const HVX_Vector scale_hi = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hi_hf));
const HVX_Vector scale = Q6_V_vmux_QVV(first16, scale_lo, scale_hi);
const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(32)));
const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
*(HVX_Vector *) dst = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
}
}
static inline HVX_Vector hvx_dequantize_tiled_f16_gather4(const uint8_t * p, uint32_t r_byte) {
HVX_Vector w0 = Q6_V_vror_VR(hvx_vmem(p + 0 * VLEN), r_byte);
HVX_Vector w1 = Q6_V_vror_VR(hvx_vmem(p + 1 * VLEN), r_byte);
HVX_Vector w01 = Q6_V_lo_W(Q6_W_vshuff_VVR(w1, w0, 4));
HVX_Vector w2 = Q6_V_vror_VR(hvx_vmem(p + 2 * VLEN), r_byte);
HVX_Vector w3 = Q6_V_vror_VR(hvx_vmem(p + 3 * VLEN), r_byte);
HVX_Vector w23 = Q6_V_lo_W(Q6_W_vshuff_VVR(w3, w2, 4));
return Q6_V_lo_W(Q6_W_vshuff_VVR(w23, w01, 8));
}
static inline void hvx_dequantize_row_tiled_f16_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
const uint32_t r_byte = 4 * row;
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
const uint8_t * tile = src_spad + k_tile * tile_stride;
float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_N_COLS;
HVX_Vector g0 = hvx_dequantize_tiled_f16_gather4(tile + 0 * VLEN, r_byte);
HVX_Vector g1 = hvx_dequantize_tiled_f16_gather4(tile + 4 * VLEN, r_byte);
HVX_Vector g01 = Q6_V_lo_W(Q6_W_vshuff_VVR(g1, g0, 16));
HVX_Vector g2 = hvx_dequantize_tiled_f16_gather4(tile + 8 * VLEN, r_byte);
HVX_Vector g3 = hvx_dequantize_tiled_f16_gather4(tile + 12 * VLEN, r_byte);
HVX_Vector g23 = Q6_V_lo_W(Q6_W_vshuff_VVR(g3, g2, 16));
HVX_Vector g = Q6_V_lo_W(Q6_W_vshuff_VVR(g23, g01, 32));
*(HVX_Vector *) dst = Q6_V_lo_W(hvx_vec_f16_to_f32(g));
}
}
static inline void hvx_dequantize_row_tiled_q1_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
static const uint32_t __attribute__((aligned(128))) bit_mask[32] = {
1u << 0, 1u << 8, 1u << 16, 1u << 24,
1u << 1, 1u << 9, 1u << 17, 1u << 25,
1u << 2, 1u << 10, 1u << 18, 1u << 26,
1u << 3, 1u << 11, 1u << 19, 1u << 27,
1u << 4, 1u << 12, 1u << 20, 1u << 28,
1u << 5, 1u << 13, 1u << 21, 1u << 29,
1u << 6, 1u << 14, 1u << 22, 1u << 30,
1u << 7, 1u << 15, 1u << 23, 1u << 31,
};
const HVX_Vector v_bit_mask = *(const HVX_Vector *) bit_mask;
const HVX_Vector v_zero = Q6_V_vzero();
const HVX_Vector v_sign_bit = Q6_V_vsplat_R(0x80000000);
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
const uint8_t * tile = src_spad + k_tile * tile_stride;
float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q1_0;
const HVX_Vector scales = hvx_vmem(tile + 512);
const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
const HVX_Vector vd = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
const HVX_Vector neg_vd = Q6_V_vxor_VV(vd, v_sign_bit);
#pragma unroll
for (int q = 0; q < 4; ++q) {
const HVX_Vector v_bits = hvx_vec_repl_u32(Q6_V_vror_VR(hvx_vmem(tile + q * VLEN), 4 * row));
const HVX_VectorPred q_zero = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(v_bits, v_bit_mask), v_zero);
hvx_vmem(dst + q * 32) = Q6_V_vmux_QVV(q_zero, neg_vd, vd);
}
}
}
static inline void hvx_dequantize_row_tiled_q2_0_f32(float * dst_spad, const uint8_t * src_spad, uint32_t n_k_tiles, uint32_t tile_stride, uint32_t row) {
const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
const HVX_Vector v_one = Q6_Vb_vsplat_R(1);
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
const uint8_t * tile = src_spad + k_tile * tile_stride;
float * dst = dst_spad + k_tile * HTP_TENSOR_TILE_K_Q2_0;
const HVX_Vector v_packed = hvx_dequantize_tiled_f16_gather4(tile, 4 * row);
const HVX_Vector p0 = Q6_V_vand_VV(v_packed, mask_03);
const HVX_Vector p1 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 2), mask_03);
const HVX_Vector p2 = Q6_V_vand_VV(Q6_Vub_vlsr_VubR(v_packed, 4), mask_03);
const HVX_Vector p3 = Q6_Vub_vlsr_VubR(v_packed, 6);
const HVX_Vector p01 = Q6_V_lo_W(Q6_W_vshuff_VVR(p1, p0, -4));
const HVX_Vector p23 = Q6_V_lo_W(Q6_W_vshuff_VVR(p3, p2, -4));
const HVX_Vector vq = Q6_Vb_vsub_VbVb(Q6_V_lo_W(Q6_W_vshuff_VVR(p23, p01, -8)), v_one);
const HVX_Vector scales = hvx_vmem(tile + 512);
const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
hvx_vmem(dst + 0) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
hvx_vmem(dst + 32) = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_hi_W(p32)), scale);
}
}
#endif // HVX_QUANT_H
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-reduce.h
+409
View File
@@ -0,0 +1,409 @@
#ifndef HVX_REDUCE_H
#define HVX_REDUCE_H
#include <math.h>
#include <stdbool.h>
#include <stdint.h>
#include <assert.h>
#include "hex-utils.h"
#include "hvx-base.h"
#include "hvx-types.h"
static inline HVX_Vector hvx_vec_reduce_sum_n_i32(HVX_Vector in, unsigned int n) {
unsigned int total = n * 4; // total vec nbytes
unsigned int width = 4; // int32
HVX_Vector sum = in, sum_t;
while (width < total) {
sum_t = Q6_V_vror_VR(sum, width); // rotate right
sum = Q6_Vw_vadd_VwVw(sum_t, sum); // elementwise sum
width = width << 1;
}
return sum;
}
static inline HVX_Vector hvx_vec_reduce_sum_i32(HVX_Vector in) {
return hvx_vec_reduce_sum_n_i32(in, 32);
}
static inline HVX_Vector hvx_vec_reduce_sum_n_qf32(HVX_Vector in, unsigned int n) {
unsigned int total = n * 4; // total vec nbytes
unsigned int width = 4; // fp32 nbytes
HVX_Vector sum = in, sum_t;
while (width < total) {
sum_t = Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum), width); // rotate right
sum = Q6_Vqf32_vadd_Vqf32Vsf(sum, sum_t); // elementwise sum
width = width << 1;
}
return sum;
}
static inline HVX_Vector hvx_vec_reduce_sum_qf32(HVX_Vector in) {
return hvx_vec_reduce_sum_n_qf32(in, 32);
}
#if __HVX_ARCH__ > 75
static inline HVX_Vector hvx_vec_reduce_sum_f32x4(HVX_Vector_x4 in) {
HVX_VectorPair sum_p01 = Q6_W_vshuff_VVR(in.v[1], in.v[0], 4);
HVX_VectorPair sum_p23 = Q6_W_vshuff_VVR(in.v[3], in.v[2], 4);
HVX_Vector sum_sf01 = Q6_Vsf_vadd_VsfVsf(Q6_V_lo_W(sum_p01), Q6_V_hi_W(sum_p01));
HVX_Vector sum_sf23 = Q6_Vsf_vadd_VsfVsf(Q6_V_lo_W(sum_p23), Q6_V_hi_W(sum_p23));
HVX_VectorPair sum_p0123 = Q6_W_vshuff_VVR(sum_sf23, sum_sf01, 8);
HVX_Vector sum_sf = Q6_Vsf_vadd_VsfVsf(Q6_V_lo_W(sum_p0123), Q6_V_hi_W(sum_p0123));
sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 2));
sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 4));
sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 8));
return sum_sf;
}
static inline HVX_Vector hvx_vec_reduce_sum_f32x2(HVX_Vector in0, HVX_Vector in1) {
HVX_VectorPair sump = Q6_W_vshuff_VVR(in1, in0, 4);
HVX_Vector sum_sf = Q6_Vsf_vadd_VsfVsf(Q6_V_lo_W(sump), Q6_V_hi_W(sump));
sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 2));
sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 4));
sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 8));
sum_sf = Q6_Vsf_vadd_VsfVsf(sum_sf, Q6_V_vror_VR(sum_sf, VLEN / 16));
return sum_sf;
}
static inline HVX_Vector hvx_vec_reduce_sum_n_f32(HVX_Vector in, unsigned int n) {
unsigned int total = n * 4; // total vec nbytes
unsigned int width = 4; // fp32 nbytes
HVX_Vector sum = in, sum_t;
while (width < total) {
sum_t = Q6_V_vror_VR(sum, width); // rotate right
sum = Q6_Vsf_vadd_VsfVsf(sum, sum_t); // elementwise sum
width = width << 1;
}
return sum;
}
#else
static inline HVX_Vector hvx_vec_reduce_sum_f32x4(HVX_Vector_x4 in) {
HVX_VectorPair sum_p01 = Q6_W_vshuff_VVR(in.v[1], in.v[0], 4);
HVX_VectorPair sum_p23 = Q6_W_vshuff_VVR(in.v[3], in.v[2], 4);
HVX_Vector sum_qf01 = Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(sum_p01), Q6_V_hi_W(sum_p01));
HVX_Vector sum_qf23 = Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(sum_p23), Q6_V_hi_W(sum_p23));
HVX_VectorPair sum_p0123 = Q6_W_vshuff_VVR(Q6_Vsf_equals_Vqf32(sum_qf23), Q6_Vsf_equals_Vqf32(sum_qf01), 8);
HVX_Vector sum_qf = Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(sum_p0123), Q6_V_hi_W(sum_p0123));
sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 2));
sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 4));
sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 8));
return Q6_Vsf_equals_Vqf32(sum_qf);
}
static inline HVX_Vector hvx_vec_reduce_sum_f32x2(HVX_Vector in0, HVX_Vector in1) {
HVX_VectorPair sump = Q6_W_vshuff_VVR(in1, in0, 4);
HVX_Vector sum_qf = Q6_Vqf32_vadd_VsfVsf(Q6_V_lo_W(sump), Q6_V_hi_W(sump));
sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 2));
sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 4));
sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 8));
sum_qf = Q6_Vqf32_vadd_Vqf32Vsf(sum_qf, Q6_V_vror_VR(Q6_Vsf_equals_Vqf32(sum_qf), VLEN / 16));
return Q6_Vsf_equals_Vqf32(sum_qf);
}
static inline HVX_Vector hvx_vec_reduce_sum_n_f32(HVX_Vector in, unsigned int n) {
unsigned int total = n * 4; // total vec nbytes
unsigned int width = 4; // fp32 nbytes
HVX_Vector sum = in, sum_t;
while (width < total) {
sum_t = Q6_V_vror_VR(sum, width); // rotate right
sum = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(sum, sum_t)); // elementwise sum
width = width << 1;
}
return sum;
}
#endif
static inline HVX_Vector hvx_vec_reduce_sum_f32(HVX_Vector in) {
return hvx_vec_reduce_sum_n_f32(in, 32);
}
static inline HVX_Vector hvx_vec_reduce_max_f16(HVX_Vector in) {
unsigned total = 128; // total vec nbytes
unsigned width = 2; // fp16 nbytes
HVX_Vector _max = in, _max_t;
while (width < total) {
_max_t = Q6_V_vror_VR(_max, width); // rotate right
_max = Q6_Vhf_vmax_VhfVhf(_max_t, _max); // elementwise max
width = width << 1;
}
return _max;
}
static inline HVX_Vector hvx_vec_reduce_max2_f16(HVX_Vector in, HVX_Vector _max) {
unsigned total = 128; // total vec nbytes
unsigned width = 2; // fp32 nbytes
HVX_Vector _max_t;
_max = Q6_Vhf_vmax_VhfVhf(in, _max);
while (width < total) {
_max_t = Q6_V_vror_VR(_max, width); // rotate right
_max = Q6_Vhf_vmax_VhfVhf(_max_t, _max); // elementwise max
width = width << 1;
}
return _max;
}
static inline HVX_Vector hvx_vec_reduce_max_f32(HVX_Vector in) {
unsigned total = 128; // total vec nbytes
unsigned width = 4; // fp32 nbytes
HVX_Vector _max = in, _max_t;
while (width < total) {
_max_t = Q6_V_vror_VR(_max, width); // rotate right
_max = Q6_Vsf_vmax_VsfVsf(_max_t, _max); // elementwise max
width = width << 1;
}
return _max;
}
static inline HVX_Vector hvx_vec_reduce_max2_f32(HVX_Vector in, HVX_Vector _max) {
unsigned total = 128; // total vec nbytes
unsigned width = 4; // fp32 nbytes
HVX_Vector _max_t;
_max = Q6_Vsf_vmax_VsfVsf(in, _max);
while (width < total) {
_max_t = Q6_V_vror_VR(_max, width); // rotate right
_max = Q6_Vsf_vmax_VsfVsf(_max_t, _max); // elementwise max
width = width << 1;
}
return _max;
}
#define hvx_reduce_loop_body(src_type, init_vec, pad_vec, vec_op, reduce_op, scalar_reduce) \
do { \
src_type * restrict vsrc = (src_type *) src; \
HVX_Vector acc = init_vec; \
\
const uint32_t elem_size = sizeof(float); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = num_elems / epv; \
const uint32_t nloe = num_elems % epv; \
\
uint32_t i = 0; \
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
acc = vec_op(acc, vsrc[i]); \
} \
if (nloe) { \
const float * srcf = (const float *) src + i * epv; \
HVX_Vector in = *(HVX_UVector *) srcf; \
HVX_Vector temp = Q6_V_valign_VVR(in, pad_vec, nloe * elem_size); \
acc = vec_op(acc, temp); \
} \
HVX_Vector v = reduce_op(acc); \
return scalar_reduce(v); \
} while(0)
#define HVX_REDUCE_MAX_OP(acc, val) Q6_Vsf_vmax_VsfVsf(acc, val)
#define HVX_REDUCE_SUM_OP(acc, val) Q6_Vqf32_vadd_VsfVsf(Q6_Vsf_equals_Vqf32(acc), val)
#define HVX_SUM_SQ_OP(acc, val) Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(val, val))
#define HVX_REDUCE_MAX_SCALAR(v) hvx_vec_get_f32(v)
#define HVX_REDUCE_SUM_SCALAR(v) hvx_vec_get_f32(Q6_Vsf_equals_Vqf32(v))
// Max variants
static inline float hvx_reduce_max_f32_a(const uint8_t * restrict src, const int num_elems) {
HVX_Vector init_vec = hvx_vec_splat_f32(((const float *) src)[0]);
assert((unsigned long) src % 128 == 0);
hvx_reduce_loop_body(HVX_Vector, init_vec, init_vec, HVX_REDUCE_MAX_OP, hvx_vec_reduce_max_f32, HVX_REDUCE_MAX_SCALAR);
}
static inline float hvx_reduce_max_f32_u(const uint8_t * restrict src, const int num_elems) {
HVX_Vector init_vec = hvx_vec_splat_f32(((const float *) src)[0]);
hvx_reduce_loop_body(HVX_UVector, init_vec, init_vec, HVX_REDUCE_MAX_OP, hvx_vec_reduce_max_f32, HVX_REDUCE_MAX_SCALAR);
}
static inline float hvx_reduce_max_f32(const uint8_t * restrict src, const int num_elems) {
if (hex_is_aligned((void *) src, 128)) {
return hvx_reduce_max_f32_a(src, num_elems);
} else {
return hvx_reduce_max_f32_u(src, num_elems);
}
}
// Sum variants
static inline float hvx_reduce_sum_f32_a(const uint8_t * restrict src, const int num_elems) {
HVX_Vector init_vec = Q6_V_vsplat_R(0);
assert((unsigned long) src % 128 == 0);
hvx_reduce_loop_body(HVX_Vector, init_vec, init_vec, HVX_REDUCE_SUM_OP, hvx_vec_reduce_sum_qf32, HVX_REDUCE_SUM_SCALAR);
}
static inline float hvx_reduce_sum_f32_u(const uint8_t * restrict src, const int num_elems) {
HVX_Vector init_vec = Q6_V_vsplat_R(0);
hvx_reduce_loop_body(HVX_UVector, init_vec, init_vec, HVX_REDUCE_SUM_OP, hvx_vec_reduce_sum_qf32, HVX_REDUCE_SUM_SCALAR);
}
static inline float hvx_reduce_sum_f32(const uint8_t * restrict src, const int num_elems) {
if (hex_is_aligned((void *) src, 128)) {
return hvx_reduce_sum_f32_a(src, num_elems);
} else {
return hvx_reduce_sum_f32_u(src, num_elems);
}
}
// Sum of squares variants
static inline float hvx_sum_of_squares_f32_a(const uint8_t * restrict src, const int num_elems) {
HVX_Vector init_vec = Q6_V_vsplat_R(0);
assert((uintptr_t) src % 128 == 0);
hvx_reduce_loop_body(HVX_Vector, init_vec, init_vec, HVX_SUM_SQ_OP, hvx_vec_reduce_sum_qf32, HVX_REDUCE_SUM_SCALAR);
}
static inline float hvx_sum_of_squares_f32_u(const uint8_t * restrict src, const int num_elems) {
HVX_Vector init_vec = Q6_V_vsplat_R(0);
hvx_reduce_loop_body(HVX_UVector, init_vec, init_vec, HVX_SUM_SQ_OP, hvx_vec_reduce_sum_qf32, HVX_REDUCE_SUM_SCALAR);
}
static inline float hvx_sum_of_squares_f32(const uint8_t * restrict src, const int num_elems) {
if (hex_is_aligned((void *) src, 128)) {
return hvx_sum_of_squares_f32_a(src, num_elems);
} else {
return hvx_sum_of_squares_f32_u(src, num_elems);
}
}
// Signed 32-bit Integer Max variants
static inline HVX_Vector hvx_vec_reduce_max_n_i32(HVX_Vector in, unsigned int n) {
unsigned int total = n * 4; // total vec nbytes
unsigned int width = 4; // int32 nbytes
HVX_Vector max_val = in, max_t;
while (width < total) {
max_t = Q6_V_vror_VR(max_val, width); // rotate right
max_val = Q6_Vw_vmax_VwVw(max_t, max_val); // elementwise signed max
width = width << 1;
}
return max_val;
}
static inline HVX_Vector hvx_vec_reduce_max_i32(HVX_Vector in) {
return hvx_vec_reduce_max_n_i32(in, 32);
}
static inline int32_t hvx_reduce_max_i32_a(const uint8_t * restrict src, const int num_elems) {
HVX_Vector init_vec = Q6_V_vsplat_R(((const int32_t *) src)[0]);
HVX_Vector pad_vec = Q6_V_vsplat_R(0x80000000);
assert((uintptr_t) src % 128 == 0);
hvx_reduce_loop_body(HVX_Vector, init_vec, pad_vec, Q6_Vw_vmax_VwVw, hvx_vec_reduce_max_i32, hvx_vec_get_i32);
}
static inline int32_t hvx_reduce_max_i32_u(const uint8_t * restrict src, const int num_elems) {
HVX_Vector init_vec = Q6_V_vsplat_R(((const int32_t *) src)[0]);
HVX_Vector pad_vec = Q6_V_vsplat_R(0x80000000);
hvx_reduce_loop_body(HVX_UVector, init_vec, pad_vec, Q6_Vw_vmax_VwVw, hvx_vec_reduce_max_i32, hvx_vec_get_i32);
}
static inline int32_t hvx_reduce_max_i32(const uint8_t * restrict src, const int num_elems) {
if (hex_is_aligned((void *) src, 128)) {
return hvx_reduce_max_i32_a(src, num_elems);
} else {
return hvx_reduce_max_i32_u(src, num_elems);
}
}
static inline void hvx_argmax_f32(
const float * restrict src,
uint32_t n,
uint32_t offset,
float * out_val,
int32_t * out_idx
) {
if (n == 0) {
*out_val = -INFINITY;
*out_idx = (int32_t) offset;
return;
}
if (n < 32 || !hex_is_aligned((void *) src, 128)) {
float best_val = src[0];
int32_t best_idx = (int32_t) offset;
for (uint32_t i = 1; i < n; i++) {
if (src[i] > best_val) {
best_val = src[i];
best_idx = (int32_t) (offset + i);
}
}
*out_val = best_val;
*out_idx = best_idx;
return;
}
static const int32_t c_lane_idx[32] __attribute__((aligned(128))) = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
};
const HVX_Vector v_init_idx = *(const HVX_Vector *) c_lane_idx;
const HVX_Vector v_step = Q6_V_vsplat_R(32);
HVX_Vector v_cur_idx = Q6_Vw_vadd_VwVw(v_init_idx, Q6_V_vsplat_R((int32_t) offset));
HVX_Vector v_max_val = hvx_vec_splat_f32(-INFINITY);
HVX_Vector v_max_idx = v_cur_idx;
const HVX_Vector * vsrc = (const HVX_Vector *) src;
const uint32_t nvec = n / 32;
for (uint32_t vi = 0; vi < nvec; vi++) {
HVX_Vector v = vsrc[vi];
HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(v, v_max_val);
v_max_val = Q6_V_vmux_QVV(pred, v, v_max_val);
v_max_idx = Q6_V_vmux_QVV(pred, v_cur_idx, v_max_idx);
v_cur_idx = Q6_Vw_vadd_VwVw(v_cur_idx, v_step);
}
HVX_VectorAlias u_val, u_idx;
u_val.v = v_max_val;
u_idx.v = v_max_idx;
float best_val = u_val.fp32[0];
int32_t best_idx = (int32_t) u_idx.w[0];
for (int i = 1; i < 32; i++) {
if (u_val.fp32[i] > best_val) {
best_val = u_val.fp32[i];
best_idx = (int32_t) u_idx.w[i];
}
}
for (uint32_t i = nvec * 32; i < n; i++) {
if (src[i] > best_val) {
best_val = src[i];
best_idx = (int32_t) (offset + i);
}
}
*out_val = best_val;
*out_idx = best_idx;
}
#undef hvx_reduce_loop_body
#undef HVX_REDUCE_MAX_OP
#undef HVX_REDUCE_SUM_OP
#undef HVX_REDUCE_MAX_SCALAR
#undef HVX_REDUCE_SUM_SCALAR
#undef HVX_SUM_SQ_OP
#endif /* HVX_REDUCE_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-repl.h
+74
View File
@@ -0,0 +1,74 @@
#ifndef HVX_REPL_H
#define HVX_REPL_H
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include "hvx-base.h"
static inline HVX_Vector hvx_vec_repl(HVX_Vector v, const uint8_t * ctrl) {
return Q6_V_vdelta_VV(v, hvx_vmem(ctrl));
}
static inline HVX_Vector hvx_vec_repl_u32(HVX_Vector v) {
// vdelta control to replicate first 4 bytes across all lanes
static const uint8_t __attribute__((aligned(128))) repl[128] = {
0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
};
return hvx_vec_repl(v, repl);
}
static inline HVX_Vector hvx_vec_repl_f32(HVX_Vector v) {
// vdelta control to replicate first 4 bytes across all lanes
static const uint8_t __attribute__((aligned(128))) repl[128] = {
0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x40, 0x40, 0x40, 0x40, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x20, 0x20, 0x20, 0x20, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
0x10, 0x10, 0x10, 0x10, 0x04, 0x04, 0x04, 0x04, 0x08, 0x08, 0x08, 0x08, 0x04, 0x04, 0x04, 0x04,
};
return hvx_vec_repl(v, repl);
}
static inline HVX_Vector hvx_vec_repl_f16(HVX_Vector v) {
// vdelta control to replicate first two bytes across all lanes
static const uint8_t __attribute__((aligned(128))) repl[128] = {
0x00, 0x00, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x20, 0x20, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x40, 0x40, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x20, 0x20, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
};
return hvx_vec_repl(v, repl);
}
static inline HVX_Vector hvx_vec_repl_2x_f16(HVX_Vector v) {
// vdelta control to splat a pair of f16s: first half = f16[0], second half = f16[1]
static const uint8_t __attribute__((aligned(128))) repl[128] = {
0x00, 0x00, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x20, 0x20, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02,
0x02, 0x02, 0x40, 0x40, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04,
0x02, 0x02, 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04,
0x02, 0x02, 0x20, 0x20, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04,
0x02, 0x02, 0x10, 0x10, 0x02, 0x02, 0x04, 0x04, 0x02, 0x02, 0x08, 0x08, 0x02, 0x02, 0x04, 0x04,
};
return hvx_vec_repl(v, repl);
}
#endif // HVX_REPL_H
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-scale.h
+199
View File
@@ -0,0 +1,199 @@
#ifndef HVX_SCALE_H
#define HVX_SCALE_H
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include "hvx-base.h"
#define hvx_scale_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
HVX_Vector vs = hvx_vec_splat_f32(scale); \
\
const uint32_t elem_size = sizeof(float); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; ++i) { \
HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs); \
vdst[i] = Q6_Vsf_equals_Vqf32(v); \
} \
if (nloe) { \
HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs); \
vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \
} \
} while(0)
static inline void hvx_scale_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {
assert((size_t) dst % 128 == 0);
assert((size_t) src % 128 == 0);
hvx_scale_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_scale_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {
assert((size_t) dst % 128 == 0);
hvx_scale_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_scale_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {
assert((size_t) src % 128 == 0);
hvx_scale_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_scale_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {
hvx_scale_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_scale_f32(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale) {
if (((size_t) dst & 127) == 0) {
if (((size_t) src & 127) == 0) {
hvx_scale_f32_aa(dst, src, n, scale);
} else {
hvx_scale_f32_au(dst, src, n, scale);
}
} else {
if (((size_t) src & 127) == 0) {
hvx_scale_f32_ua(dst, src, n, scale);
} else {
hvx_scale_f32_uu(dst, src, n, scale);
}
}
}
#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
HVX_Vector vs = hvx_vec_splat_f32(scale); \
HVX_Vector vo = hvx_vec_splat_f32(offset); \
\
const uint32_t elem_size = sizeof(float); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; ++i) { \
HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \
vdst[i] = Q6_Vsf_equals_Vqf32(v); \
} \
if (nloe) { \
HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \
vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \
} \
} while(0)
static inline void hvx_scale_offset_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
assert((size_t) dst % 128 == 0);
assert((size_t) src % 128 == 0);
hvx_scale_offset_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_scale_offset_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
assert((size_t) dst % 128 == 0);
hvx_scale_offset_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_scale_offset_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
assert((size_t) src % 128 == 0);
hvx_scale_offset_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_scale_offset_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
hvx_scale_offset_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_scale_offset_f32(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
if (((size_t) dst & 127) == 0) {
if (((size_t) src & 127) == 0) {
hvx_scale_offset_f32_aa(dst, src, n, scale, offset);
} else {
hvx_scale_offset_f32_au(dst, src, n, scale, offset);
}
} else {
if (((size_t) src & 127) == 0) {
hvx_scale_offset_f32_ua(dst, src, n, scale, offset);
} else {
hvx_scale_offset_f32_uu(dst, src, n, scale, offset);
}
}
}
// Scale+offset computed by promoting f16 -> f32, then narrowing the result back to f16.
#define hvx_scale_offset_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
HVX_Vector vs = hvx_vec_splat_f32(scale); \
HVX_Vector vo = hvx_vec_splat_f32(offset); \
\
const uint32_t nvec = n / VLEN_FP16; \
const uint32_t nloe = n % VLEN_FP16; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; ++i) { \
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \
vdst[i] = hvx_vec_f32_to_f16(r0, r1); \
} \
if (nloe) { \
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
HVX_Vector r0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(p), vs), vo)); \
HVX_Vector r1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(p), vs), vo)); \
HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \
} \
} while(0)
static inline void hvx_scale_offset_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
assert((size_t) dst % 128 == 0);
assert((size_t) src % 128 == 0);
hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_scale_offset_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
assert((size_t) dst % 128 == 0);
hvx_scale_offset_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_scale_offset_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
assert((size_t) src % 128 == 0);
hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_scale_offset_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
hvx_scale_offset_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_scale_offset_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
if (((size_t) dst & 127) == 0) {
if (((size_t) src & 127) == 0) {
hvx_scale_offset_f16_aa(dst, src, n, scale, offset);
} else {
hvx_scale_offset_f16_au(dst, src, n, scale, offset);
}
} else {
if (((size_t) src & 127) == 0) {
hvx_scale_offset_f16_ua(dst, src, n, scale, offset);
} else {
hvx_scale_offset_f16_uu(dst, src, n, scale, offset);
}
}
}
#endif // HVX_SCALE_H
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
+243
View File
@@ -0,0 +1,243 @@
#ifndef HVX_SIGMOID_H
#define HVX_SIGMOID_H
#include "hvx-base.h"
#include "hvx-inverse.h"
#include "hvx-exp.h"
#define FAST_SIGMOID_LOG2F (0x3fb8aa3b) // 1.442695022
#define FAST_SIGMOID_C1 (0x3d009076) // 0.03138777
#define FAST_SIGMOID_C2 (0x3e8d74bd) // 0.276281267
#define FAST_SIGMOID_C3 (0x3f000000) // 0.5
static inline HVX_Vector hvx_vec_fast_sigmoid_f32(HVX_Vector v) {
v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3));
HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v));
HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int));
HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x);
HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2));
v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F));
HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1));
v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx);
v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x);
HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1));
HVX_Vector v3_exponent = Q6_Vw_vasl_VwR(v3, 1);
v3_exponent = Q6_Vuw_vlsr_VuwR(v3_exponent, 24);
v3_exponent = Q6_Vw_vadd_VwVw(in_int, v3_exponent);
v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24);
HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1));
HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4));
HVX_Vector res = hvx_vec_inverse_f32(v5);
res = Q6_Vqf32_vmpy_VsfVsf(v3, res);
return Q6_Vsf_equals_Vqf32(res);
}
static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard(HVX_Vector v,
HVX_Vector one,
HVX_Vector max_exp,
HVX_Vector min_exp) {
const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v);
const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp);
HVX_Vector out = hvx_vec_fast_sigmoid_f32(v);
out = Q6_V_vmux_QVV(pred_max, out, one);
return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero());
}
static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) {
// tanh(x) = 2 * sigmoid(2x) - 1
HVX_Vector two = hvx_vec_splat_f32(2.0f);
HVX_Vector one = hvx_vec_splat_f32(1.0f);
HVX_Vector x2 = Q6_Vqf32_vmpy_VsfVsf(x, two);
HVX_Vector max_exp = hvx_vec_splat_f32(87.f);
HVX_Vector min_exp = hvx_vec_splat_f32(-87.f);
HVX_Vector sig2x = hvx_vec_fast_sigmoid_f32_guard(Q6_Vsf_equals_Vqf32(x2), one, max_exp, min_exp);
HVX_Vector res = Q6_Vqf32_vmpy_VsfVsf(sig2x, two);
res = Q6_Vqf32_vsub_Vqf32Vsf(res, one);
return Q6_Vsf_equals_Vqf32(res);
}
static inline HVX_Vector hvx_vec_silu_f32(HVX_Vector x) {
HVX_Vector one = hvx_vec_splat_f32(1.0f);
HVX_Vector max_exp = hvx_vec_splat_f32(87.f);
HVX_Vector min_exp = hvx_vec_splat_f32(-87.f);
HVX_Vector sig = hvx_vec_fast_sigmoid_f32_guard(x, one, max_exp, min_exp);
return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, sig));
}
#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const HVX_Vector one = hvx_vec_splat_f32(1.f); \
const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \
const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \
\
const uint32_t epv = 128 / sizeof(float); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \
} \
if (nloe) { \
HVX_Vector tmp = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \
vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \
} \
} while(0)
#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / sizeof(float); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \
} \
} while(0)
static inline void hvx_sigmoid_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_sigmoid_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_sigmoid_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_sigmoid_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_sigmoid_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_sigmoid_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_sigmoid_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_sigmoid_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_tanh_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_tanh_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline HVX_Vector hvx_vec_fast_sigmoid_f16(HVX_Vector x_v) {
const HVX_Vector v_one = hvx_vec_splat_f16(1.0f);
const HVX_Vector v_neg_log2e = hvx_vec_splat_f16(-EXP_LOG2E_F);
const HVX_Vector em_mask = Q6_Vh_vsplat_R(0x7FFF);
// Compute absolute value of x_v
HVX_Vector abs_x = Q6_V_vand_VV(x_v, em_mask);
// Compute u = -abs_x * log2(e) <= 0.
HVX_Vector u = hvx_vec_mul_f16_f16(abs_x, v_neg_log2e);
// Clamp input to prevent underflow in exp2
const HVX_Vector v_clamp_min = hvx_vec_splat_f16(-24.0f);
u = Q6_Vhf_vmax_VhfVhf(v_clamp_min, u);
HVX_Vector exp_val = hvx_vec_exp2_f16(u);
HVX_Vector denom = hvx_vec_add_f16_f16(v_one, exp_val);
HVX_Vector sig_abs = hvx_vec_inverse_f16(denom);
// check if x_v < 0 (using integer comparison on absolute value)
HVX_VectorPred is_neg = Q6_Q_vcmp_gt_VhVh(abs_x, x_v);
// If x_v < 0, return 1.0f - sig_abs
HVX_Vector sig_neg = Q6_Vhf_equals_Vqf16(Q6_Vqf16_vsub_VhfVhf(v_one, sig_abs));
return Q6_V_vmux_QVV(is_neg, sig_neg, sig_abs);
}
static inline HVX_Vector hvx_vec_tanh_f16(HVX_Vector x) {
// tanh(x) = 2 * sigmoid(2x) - 1
const HVX_Vector v_two = hvx_vec_splat_f16(2.0f);
HVX_Vector x2 = hvx_vec_mul_f16_f16(x, v_two);
HVX_Vector sig2x = hvx_vec_fast_sigmoid_f16(x2);
const HVX_Vector v_neg_one = hvx_vec_splat_f16(-1.0f);
return hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(sig2x, v_two), v_neg_one);
}
#define hvx_sigmoid_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / sizeof(_Float16); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_fast_sigmoid_f16(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector tmp = hvx_vec_fast_sigmoid_f16(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \
} \
} while(0)
#define hvx_tanh_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / sizeof(_Float16); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_tanh_f16(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector tmp = hvx_vec_tanh_f16(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \
} \
} while(0)
static inline void hvx_sigmoid_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_sigmoid_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_tanh_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_tanh_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
#endif /* HVX_SIGMOID_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h
+78
View File
@@ -0,0 +1,78 @@
#ifndef HVX_SIN_COS_H
#define HVX_SIN_COS_H
#include "hvx-base.h"
#include "hvx-floor.h"
// Range-reduce x to y in [-pi/2, pi/2] and the quadrant sign (-1)^n.
// Floor/truncate need IEEE bits, so convert qf32 back to sf before them.
static inline void hvx_vec_sincos_reduce_f32(HVX_Vector x, HVX_Vector * y, HVX_Vector * sign) {
HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);
HVX_Vector const_half = hvx_vec_splat_f32(0.5f);
HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);
HVX_Vector const_one = hvx_vec_splat_f32(1.0f);
HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f);
HVX_Vector const_one_i = Q6_V_vsplat_R(1);
HVX_Vector x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, const_inv_pi));
x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(x_over_pi, const_half));
HVX_Vector n_float = hvx_vec_floor_f32(x_over_pi);
HVX_Vector n_int = hvx_vec_truncate_f32(n_float);
HVX_Vector n_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(n_float, const_pi));
*y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(x, n_pi));
HVX_VectorPred is_odd = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(n_int, const_one_i), const_one_i);
*sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);
}
static inline void hvx_vec_sincos_f32(HVX_Vector x, HVX_Vector * vcos, HVX_Vector * vsin) {
HVX_Vector y;
HVX_Vector sign;
hvx_vec_sincos_reduce_f32(x, &y, &sign);
HVX_Vector z = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, y));
HVX_Vector c4 = hvx_vec_splat_f32(2.3557242013849433e-05f);
HVX_Vector c3 = hvx_vec_splat_f32(-0.0013871428263450528f);
HVX_Vector c2 = hvx_vec_splat_f32(0.041665895266688284f);
HVX_Vector c1 = hvx_vec_splat_f32(-0.4999999360426369f);
HVX_Vector c0 = hvx_vec_splat_f32(0.9999999999071725f);
HVX_Vector cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, c4))));
cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));
cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));
cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));
HVX_Vector s4 = hvx_vec_splat_f32(2.642186986152672e-06f);
HVX_Vector s3 = hvx_vec_splat_f32(-0.00019825318964070864f);
HVX_Vector s2 = hvx_vec_splat_f32(0.00833326283319605f);
HVX_Vector s1 = hvx_vec_splat_f32(-0.16666666082087775f);
HVX_Vector s0 = hvx_vec_splat_f32(0.999999999915155f);
HVX_Vector sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, s4))));
sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));
sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));
sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));
sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, sin_y));
*vcos = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(cos_y, sign));
*vsin = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(sin_y, sign));
}
static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) {
HVX_Vector vcos;
HVX_Vector vsin;
hvx_vec_sincos_f32(x, &vcos, &vsin);
return vcos;
}
static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) {
HVX_Vector vcos;
HVX_Vector vsin;
hvx_vec_sincos_f32(x, &vcos, &vsin);
return vsin;
}
#endif /* HVX_SIN_COS_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-sqrt.h
+189
View File
@@ -0,0 +1,189 @@
#ifndef HVX_SQRT_H
#define HVX_SQRT_H
#include <stdbool.h>
#include <stdint.h>
#include "hex-utils.h"
#include "hvx-base.h"
#define RSQRT_CONST 0x5f3759df // Constant for fast inverse square root calculation
#define RSQRT_ONE_HALF 0x3f000000 // 0.5
#define RSQRT_THREE_HALVES 0x3fc00000 // 1.5
#if __HVX_ARCH__ < 79
#define HVX_OP_MUL(a, b) Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a, b))
#else
#define HVX_OP_MUL(a, b) Q6_Vsf_vmpy_VsfVsf(a, b)
#endif
static inline HVX_Vector hvx_vec_rsqrt_f32(HVX_Vector in_vec) {
//Algorithm :
// x2 = input*0.5
// y = * (long *) &input
// y = 0x5f3759df - (y>>1)
// y = y*(threehalfs - x2*y*y)
HVX_Vector rsqrtconst = Q6_V_vsplat_R(RSQRT_CONST);
HVX_Vector onehalf = Q6_V_vsplat_R(RSQRT_ONE_HALF);
HVX_Vector threehalfs = Q6_V_vsplat_R(RSQRT_THREE_HALVES);
HVX_Vector x2, y, ypower2, temp;
x2 = Q6_Vqf32_vmpy_VsfVsf(in_vec, onehalf);
x2 = Q6_Vqf32_vadd_Vqf32Vsf(x2, Q6_V_vzero());
y = Q6_Vw_vasr_VwR(in_vec, 1);
y = Q6_Vw_vsub_VwVw(rsqrtconst, y);
// 1st iteration
ypower2 = Q6_Vqf32_vmpy_VsfVsf(y, y);
ypower2 = Q6_Vqf32_vadd_Vqf32Vsf(ypower2, Q6_V_vzero());
temp = Q6_Vqf32_vmpy_Vqf32Vqf32(x2, ypower2);
temp = Q6_Vqf32_vsub_VsfVsf(threehalfs, Q6_Vsf_equals_Vqf32(temp));
temp = Q6_Vqf32_vmpy_VsfVsf(y, Q6_Vsf_equals_Vqf32(temp));
// 2nd iteration
y = Q6_Vqf32_vadd_Vqf32Vsf(temp, Q6_V_vzero());
ypower2 = Q6_Vqf32_vmpy_Vqf32Vqf32(y, y);
ypower2 = Q6_Vqf32_vadd_Vqf32Vsf(ypower2, Q6_V_vzero());
temp = Q6_Vqf32_vmpy_Vqf32Vqf32(x2, ypower2);
temp = Q6_Vqf32_vsub_VsfVsf(threehalfs, Q6_Vsf_equals_Vqf32(temp));
temp = Q6_Vqf32_vmpy_Vqf32Vqf32(y, temp);
// 3rd iteration
y = Q6_Vqf32_vadd_Vqf32Vsf(temp, Q6_V_vzero());
ypower2 = Q6_Vqf32_vmpy_Vqf32Vqf32(y, y);
ypower2 = Q6_Vqf32_vadd_Vqf32Vsf(ypower2, Q6_V_vzero());
temp = Q6_Vqf32_vmpy_Vqf32Vqf32(x2, ypower2);
temp = Q6_Vqf32_vsub_VsfVsf(threehalfs, Q6_Vsf_equals_Vqf32(temp));
temp = Q6_Vqf32_vmpy_Vqf32Vqf32(y, temp);
return Q6_Vsf_equals_Vqf32(temp);
}
// Compute sqrt(x) as x*inv_sqrt(x)
#define hvx_sqrt_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t nvec = n / VLEN_FP32; \
const uint32_t nloe = n % VLEN_FP32; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
HVX_Vector inv_sqrt = hvx_vec_rsqrt_f32(vsrc[i]); \
HVX_Vector sqrt_res = HVX_OP_MUL(inv_sqrt, vsrc[i]); \
vdst[i] = sqrt_res; \
} \
if (nloe) { \
HVX_Vector inv_sqrt = hvx_vec_rsqrt_f32(vsrc[i]); \
HVX_Vector sqrt_res = HVX_OP_MUL(inv_sqrt, vsrc[i]); \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP32, sqrt_res); \
} \
} while(0)
static inline void hvx_sqrt_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_sqrt_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_sqrt_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_sqrt_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_sqrt_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_sqrt_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_sqrt_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_sqrt_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_sqrt_f32(uint8_t * restrict dst, const uint8_t * restrict src, const int num_elems) {
if ((unsigned long) dst % 128 == 0) {
if ((unsigned long) src % 128 == 0) {
hvx_sqrt_f32_aa(dst, src, num_elems);
} else {
hvx_sqrt_f32_au(dst, src, num_elems);
}
} else {
if ((unsigned long) src % 128 == 0) {
hvx_sqrt_f32_ua(dst, src, num_elems);
} else {
hvx_sqrt_f32_uu(dst, src, num_elems);
}
}
}
// Compute sqrt(x) for f16 by promoting to f32, applying hvx_vec_rsqrt_f32, and narrowing back.
#define hvx_sqrt_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t nvec = n / VLEN_FP16; \
const uint32_t nloe = n % VLEN_FP16; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \
HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \
vdst[i] = hvx_vec_f32_to_f16(r0, r1); \
} \
if (nloe) { \
HVX_VectorPair p = hvx_vec_f16_to_f32(vsrc[i]); \
HVX_Vector r0 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_lo_W(p)), Q6_V_lo_W(p)); \
HVX_Vector r1 = HVX_OP_MUL(hvx_vec_rsqrt_f32(Q6_V_hi_W(p)), Q6_V_hi_W(p)); \
HVX_Vector v = hvx_vec_f32_to_f16(r0, r1); \
vec_store((void *) &vdst[i], nloe * SIZEOF_FP16, v); \
} \
} while(0)
static inline void hvx_sqrt_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_sqrt_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_sqrt_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_sqrt_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
static inline void hvx_sqrt_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_sqrt_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
static inline void hvx_sqrt_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_sqrt_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
static inline void hvx_sqrt_f16(uint8_t * restrict dst, const uint8_t * restrict src, const int num_elems) {
if ((unsigned long) dst % 128 == 0) {
if ((unsigned long) src % 128 == 0) {
hvx_sqrt_f16_aa(dst, src, num_elems);
} else {
hvx_sqrt_f16_au(dst, src, num_elems);
}
} else {
if ((unsigned long) src % 128 == 0) {
hvx_sqrt_f16_ua(dst, src, num_elems);
} else {
hvx_sqrt_f16_uu(dst, src, num_elems);
}
}
}
#endif /* HVX_SQRT_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-types.h
+36
View File
@@ -0,0 +1,36 @@
#ifndef HVX_TYPES_H
#define HVX_TYPES_H
#include <stdbool.h>
#include <stdint.h>
#include <hexagon_types.h>
#define SIZEOF_FP32 (4)
#define SIZEOF_FP16 (2)
#define VLEN (128)
#define VLEN_FP32 (VLEN / SIZEOF_FP32)
#define VLEN_FP16 (VLEN / SIZEOF_FP16)
typedef union {
HVX_Vector v;
uint8_t b[VLEN];
uint16_t h[VLEN_FP16];
uint32_t w[VLEN_FP32];
__fp16 fp16[VLEN_FP16];
float fp32[VLEN_FP32];
} __attribute__((aligned(VLEN), packed)) HVX_VectorAlias;
typedef struct {
HVX_Vector v[2];
} HVX_Vector_x2;
typedef struct {
HVX_Vector v[4];
} HVX_Vector_x4;
typedef struct {
HVX_Vector v[8];
} HVX_Vector_x8;
#endif /* HVX_TYPES_H */
llama.cpp/ggml/src/ggml-hexagon/htp/hvx-utils.h
+25
View File
@@ -0,0 +1,25 @@
#ifndef HVX_UTILS_H
#define HVX_UTILS_H
#include "hex-utils.h"
#include "hvx-types.h"
#include "hvx-copy.h"
#include "hvx-repl.h"
#include "hvx-scale.h"
#include "hvx-exp.h"
#include "hvx-erf.h"
#include "hvx-inverse.h"
#include "hvx-reduce.h"
#include "hvx-sigmoid.h"
#include "hvx-sqrt.h"
#include "hvx-arith.h"
#include "hvx-div.h"
#include "hvx-floor.h"
#include "hvx-sin-cos.h"
#include "hvx-base.h"
#include "hvx-pow.h"
#include "hvx-log.h"
#include "hvx-norm.h"
#endif /* HVX_UTILS_H */
llama.cpp/ggml/src/ggml-hexagon/htp/im2col-ops.c
+932
View File
@@ -0,0 +1,932 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic ignored "-Wunused-function"
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <hexagon_protos.h>
#include <hexagon_types.h>
#include <string.h>
#include "hex-common.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "hvx-utils.h"
#include "dma-queue.h"
#include "hex-profile.h"
#include "htp-vtcm.h"
#include "htp-tensor.h"
#include "im2col-ops.h"
struct htp_im2col_context {
struct htp_ops_context * octx;
uint32_t pe_row_base; // first output row index assigned to this dev (DMA path)
uint32_t pe_nrows; // number of output rows assigned to this dev (DMA path)
uint32_t pe_rows_per_thread; // output rows per worker
uint32_t pe_src_row_bytes; // one output row's source, rounded 256
uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256
// Patch-embed DMA path VTCM ping-pong.
uint8_t * pe_vtcm_src; // base of the 2x src buffers region
uint8_t * pe_vtcm_dst; // base of the 2x dst buffers region
uint32_t pe_src_size_per_thread; // 2 * pe_src_row_bytes
uint32_t pe_dst_size_per_thread; // 2 * pe_dst_row_bytes
uint32_t pe_owb; // output-col block size
uint32_t pe_wb; // staged source window width
uint32_t pe_nbpr; // blocks per row
// Precomputed dimensions for worker threads
uint32_t N;
uint32_t IC;
uint32_t ID, IH, IW;
uint32_t KD, KH, KW;
uint32_t OD, OH, OW;
uint32_t patch_stride;
// Per-thread initial coordinates (removes division from thread prologue)
uint32_t thread_in[HTP_MAX_NTHREADS];
uint32_t thread_iod[HTP_MAX_NTHREADS];
uint32_t thread_ioh[HTP_MAX_NTHREADS];
void * compute;
uint32_t dst_elem;
};
typedef void (*im2col_blocked_compute_fn_t)(
const struct htp_im2col_context * ictx,
void * restrict dstb, const float * restrict srcb,
float * restrict scratch,
uint32_t nb, uint32_t c0, uint32_t ioh);
typedef void (*im2col_patchembed_compute_fn_t)(
const struct htp_im2col_context * ictx,
void * restrict dstb, const float * restrict srcb, uint32_t ioh);
typedef void (*im2col_3d_blocked_compute_fn_t)(
const struct htp_im2col_context * ictx,
void * restrict dstb, const float * restrict srcb,
float * restrict scratch,
uint32_t nb, uint32_t c0, uint32_t iod, uint32_t ioh);
typedef void (*im2col_3d_patchembed_compute_fn_t)(
const struct htp_im2col_context * ictx,
void * restrict dstb, const float * restrict srcb);
struct im2col_cursor {
uint32_t r;
uint32_t in;
uint32_t iod;
uint32_t ioh;
uint32_t c0;
};
static inline void im2col_step_row(struct im2col_cursor * c, const struct htp_im2col_context * ictx) {
c->r++;
c->ioh++;
if (c->ioh >= ictx->OH) {
c->ioh = 0;
c->in++;
}
}
static inline void im2col_step_block(struct im2col_cursor * c, const struct htp_im2col_context * ictx) {
c->c0 += ictx->pe_owb;
if (c->c0 >= ictx->OW) {
c->c0 = 0;
c->r++;
c->ioh++;
if (c->ioh >= ictx->OH) {
c->ioh = 0;
c->in++;
}
}
}
static void im2col_blocked_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q,
float * srcb, const struct im2col_cursor * c) {
const uint32_t in = c->in, ioh = c->ioh, c0 = c->c0;
const struct htp_ops_context * octx = ictx->octx;
const struct htp_tensor * src1 = octx->src[1];
const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1];
const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3];
const int32_t d1 = octx->op_params[5];
const bool is_2D = (octx->op_params[6] == 1);
const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW;
const uint32_t KH = ictx->KH;
const uint32_t Wb = ictx->pe_wb;
const dma_addr_t src_data = src1->data;
const int32_t win0 = (int32_t) c0 * s0 - p0;
const size_t src_channel_stride = is_2D ? src1->nb[2] : src1->nb[1];
const size_t batch_offset = is_2D ? (size_t) in * src1->nb[3] : (size_t) in * src1->nb[2];
for (uint32_t ikh = 0; ikh < KH; ikh++) {
const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1;
if (iih < 0 || iih >= (int32_t) IH) {
dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0);
continue;
}
const int32_t lo = win0 < 0 ? -win0 : 0;
int32_t hi = (int32_t) IW - win0;
if (hi > (int32_t) Wb) {
hi = (int32_t) Wb;
}
if (hi <= lo) {
dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0);
continue;
}
const uint32_t cpw = (uint32_t) (hi - lo);
float * vdst = srcb + (size_t) ikh * Wb + lo;
const dma_addr_t vsrc = src_data + batch_offset + (size_t) iih * src1->nb[1] + (size_t) (win0 + lo) * src1->nb[0];
dma_queue_push(dma_q, dma_make_data(vdst, vsrc),
(size_t) KH * Wb * sizeof(float), src_channel_stride,
cpw * sizeof(float), IC);
}
}
#define IM2COL_BLOCKED_COMPUTE_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN) \
static void FNAME( \
const struct htp_im2col_context * ictx, \
void * restrict dstb_raw, const float * restrict srcb, \
float * restrict scratch, \
uint32_t nb, uint32_t c0, uint32_t ioh) { \
DST_CTYPE * restrict dstb = (DST_CTYPE *) dstb_raw; \
const struct htp_ops_context * octx = ictx->octx; \
const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; \
const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; \
const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; \
const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW; \
const uint32_t KH = ictx->KH, KW = ictx->KW; \
const uint32_t Wb = ictx->pe_wb; \
const uint32_t patch_stride = ictx->patch_stride; \
const int32_t win0 = (int32_t) c0 * s0 - p0; \
\
for (uint32_t j = 0; j < nb; j++) { \
const uint32_t iow = c0 + j; \
DST_CTYPE * dst_patch = dstb + (size_t) j * patch_stride; \
const int32_t iiw0 = (int32_t) iow * s0 - p0; \
for (uint32_t ikh = 0; ikh < KH; ikh++) { \
const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \
const int okh = (iih >= 0 && iih < (int32_t) IH); \
for (uint32_t iic = 0; iic < IC; iic++) { \
DST_CTYPE * out_run = dst_patch + (size_t) (iic * KH + ikh) * KW; \
if (!okh) { \
SPLAT_FN(out_run, 0, KW); \
continue; \
} \
const float * vrow = srcb + (size_t) (iic * KH + ikh) * Wb; \
if (d0 == 1) { \
const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \
int32_t hi = (int32_t) IW - iiw0; \
if (hi > (int32_t) KW) { \
hi = (int32_t) KW; \
} \
if (hi <= lo) { \
SPLAT_FN(out_run, 0, KW); \
} else { \
if (lo > 0) { \
SPLAT_FN(out_run, 0, (uint32_t) lo); \
} \
COPY_FN((uint8_t *) (out_run + lo), \
(const uint8_t *) (vrow + (iiw0 + lo - win0)), \
(uint32_t) (hi - lo)); \
if (hi < (int32_t) KW) { \
SPLAT_FN(out_run + hi, 0, (uint32_t) (KW - hi)); \
} \
} \
continue; \
} \
for (uint32_t ikw = 0; ikw < KW; ikw += 64) { \
const uint32_t chunk = MIN(KW - ikw, 64); \
for (uint32_t c = 0; c < chunk; c++) { \
const int32_t iiw = iiw0 + (int32_t) (ikw + c) * d0; \
scratch[c] = (iiw < 0 || iiw >= (int32_t) IW) ? 0.0f : vrow[iiw - win0]; \
} \
COPY_FN((uint8_t *) (out_run + ikw), (const uint8_t *) scratch, chunk); \
} \
} \
} \
} \
}
IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u)
IM2COL_BLOCKED_COMPUTE_BODY(im2col_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u)
static void im2col_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
struct htp_ops_context * octx = ictx->octx;
struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith];
const uint32_t OW = ictx->OW;
const uint32_t KH = ictx->KH;
const uint32_t owb = ictx->pe_owb;
const uint32_t nbpr = ictx->pe_nbpr;
const uint32_t patch_stride = ictx->patch_stride;
const uint32_t dst_elem = ictx->dst_elem;
im2col_blocked_compute_fn_t compute = (im2col_blocked_compute_fn_t) ictx->compute;
const dma_addr_t dst_data = octx->dst->data;
dma_queue * dma_q = octx->ctx->dma[ith];
uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread;
uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread;
float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) };
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes);
const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows;
const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith;
const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max);
if (row_start >= row_end) {
return;
}
const uint32_t nrows_local = row_end - row_start;
const uint32_t total_blocks = nrows_local * nbpr;
if (total_blocks == 0) {
return;
}
struct im2col_cursor cur = {
.r = row_start,
.in = ictx->thread_in[ith],
.iod = 0,
.ioh = ictx->thread_ioh[ith],
.c0 = 0,
};
struct im2col_cursor stage = cur;
for (uint32_t i = 0; i < total_blocks && i < 2; i++) {
dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0);
im2col_blocked_dma_stage(ictx, dma_q, srcb2[i], &stage);
im2col_step_block(&stage, ictx);
}
for (uint32_t bi = 0; bi < total_blocks; bi++) {
const uint32_t buf = bi & 1u;
float * srcb = srcb2[buf];
void * dstb = dstb2[buf];
const uint32_t nb = MIN(owb, OW - cur.c0);
dma_queue_pop(dma_q);
for (uint32_t k = 0; k < KH; k++) {
dma_queue_pop(dma_q);
}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r);
compute(ictx, dstb, srcb, scratch, nb, cur.c0, cur.ioh);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r);
const dma_addr_t ddr = dst_data + (size_t) (cur.r * OW + cur.c0) * patch_stride * dst_elem;
dma_queue_push(dma_q, dma_make_data(ddr, dstb),
nb * patch_stride * dst_elem, nb * patch_stride * dst_elem,
nb * patch_stride * dst_elem, 1);
if (bi + 2 < total_blocks) {
im2col_blocked_dma_stage(ictx, dma_q, srcb, &stage);
im2col_step_block(&stage, ictx);
}
im2col_step_block(&cur, ictx);
}
dma_queue_flush(dma_q);
}
static void im2col_patchembed_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q,
float * srcb, const struct im2col_cursor * c) {
const struct htp_ops_context * octx = ictx->octx;
const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW;
const uint32_t KH = ictx->KH;
const uint32_t in = c->in, ioh = c->ioh;
const size_t src_channel_stride = (size_t) IH * IW * sizeof(float);
const size_t vtcm_channel_stride = (size_t) KH * IW * sizeof(float);
const dma_addr_t src_data = octx->src[1]->data;
const dma_addr_t vsrc = src_data + (size_t) (((in * IC) * IH + ioh * KH) * IW) * sizeof(float);
dma_queue_push(dma_q, dma_make_data(srcb, vsrc),
vtcm_channel_stride, src_channel_stride,
vtcm_channel_stride, IC);
}
static void im2col_patchembed_compute_f16(
const struct htp_im2col_context * ictx,
void * restrict dstb_raw, const float * restrict srcb, uint32_t ioh) {
uint16_t * restrict dstb = (uint16_t *) dstb_raw;
const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW;
const uint32_t KH = ictx->KH, KW = ictx->KW;
const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride;
for (uint32_t iow = 0; iow < OW; iow++) {
uint16_t * dst_patch = dstb + (size_t) iow * patch_stride;
for (uint32_t ikh = 0; ikh < KH; ikh++) {
const int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh;
for (uint32_t iic = 0; iic < IC; iic++) {
uint16_t * out_run = dst_patch + (size_t) (iic * KH + ikh) * KW;
if (iih < 0 || iih >= (int32_t) IH) {
hvx_splat_u16_u(out_run, 0, KW);
continue;
}
const float * src_run = srcb + (size_t) (iic * KH + ikh) * IW + (size_t) iow * KW;
hvx_copy_f16_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW);
}
}
}
}
static void im2col_patchembed_compute_f32(
const struct htp_im2col_context * ictx,
void * restrict dstb_raw, const float * restrict srcb, uint32_t ioh) {
float * restrict dstb = (float *) dstb_raw;
const uint32_t IC = ictx->IC, IH = ictx->IH, IW = ictx->IW;
const uint32_t KH = ictx->KH, KW = ictx->KW;
const uint32_t OW = ictx->OW, patch_stride = ictx->patch_stride;
for (uint32_t iow = 0; iow < OW; iow++) {
float * dst_patch = dstb + (size_t) iow * patch_stride;
for (uint32_t ikh = 0; ikh < KH; ikh++) {
const int32_t iih = (int32_t) ioh * (int32_t) KH + (int32_t) ikh;
for (uint32_t iic = 0; iic < IC; iic++) {
float * out_run = dst_patch + (size_t) (iic * KH + ikh) * KW;
if (iih < 0 || iih >= (int32_t) IH) {
hvx_splat_f32_u(out_run, 0, KW);
continue;
}
const float * src_run = srcb + (size_t) (iic * KH + ikh) * IW + (size_t) iow * KW;
hvx_copy_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW);
}
}
}
}
static void im2col_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
struct htp_ops_context * octx = ictx->octx;
struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith];
const uint32_t OW = ictx->OW;
const uint32_t patch_stride = ictx->patch_stride;
const uint32_t dst_elem = ictx->dst_elem;
im2col_patchembed_compute_fn_t compute = (im2col_patchembed_compute_fn_t) ictx->compute;
const dma_addr_t dst_data = octx->dst->data;
dma_queue * dma_q = octx->ctx->dma[ith];
uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread;
uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread;
float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) };
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows;
const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith;
const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max);
if (row_start >= row_end) {
return;
}
const uint32_t total_rows = row_end - row_start;
if (total_rows == 0) {
return;
}
struct im2col_cursor cur = {
.r = row_start,
.in = ictx->thread_in[ith],
.iod = 0,
.ioh = ictx->thread_ioh[ith],
.c0 = 0,
};
struct im2col_cursor stage = cur;
for (uint32_t i = 0; i < total_rows && i < 2; i++) {
dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0);
im2col_patchembed_dma_stage(ictx, dma_q, srcb2[i], &stage);
im2col_step_row(&stage, ictx);
}
for (uint32_t ri = 0; ri < total_rows; ri++) {
const uint32_t buf = ri & 1u;
float * srcb = srcb2[buf];
void * dstb = dstb2[buf];
dma_queue_pop(dma_q);
dma_queue_pop(dma_q);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r);
compute(ictx, dstb, srcb, cur.ioh);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r);
const dma_addr_t ddr_row = dst_data + (size_t) cur.r * OW * patch_stride * dst_elem;
dma_queue_push(dma_q, dma_make_data(ddr_row, dstb),
OW * patch_stride * dst_elem, OW * patch_stride * dst_elem,
OW * patch_stride * dst_elem, 1);
if (ri + 2 < total_rows) {
im2col_patchembed_dma_stage(ictx, dma_q, srcb, &stage);
im2col_step_row(&stage, ictx);
}
im2col_step_row(&cur, ictx);
}
dma_queue_flush(dma_q);
}
static inline void im2col_3d_step_row(struct im2col_cursor * c, const struct htp_im2col_context * ictx) {
c->r++;
c->ioh++;
if (c->ioh >= ictx->OH) {
c->ioh = 0;
c->iod++;
if (c->iod >= ictx->OD) {
c->iod = 0;
c->in++;
}
}
}
static inline void im2col_3d_step_block(struct im2col_cursor * c, const struct htp_im2col_context * ictx) {
c->c0 += ictx->pe_owb;
if (c->c0 >= ictx->OW) {
c->c0 = 0;
c->r++;
c->ioh++;
if (c->ioh >= ictx->OH) {
c->ioh = 0;
c->iod++;
if (c->iod >= ictx->OD) {
c->iod = 0;
c->in++;
}
}
}
}
static void im2col_3d_blocked_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q,
float * srcb, const struct im2col_cursor * c) {
const uint32_t in = c->in, iod = c->iod, ioh = c->ioh, c0 = c->c0;
const struct htp_ops_context * octx = ictx->octx;
const struct htp_tensor * src1 = octx->src[1];
const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1], s2 = octx->op_params[2];
const int32_t p0 = octx->op_params[3], p1 = octx->op_params[4], p2 = octx->op_params[5];
const int32_t d1 = octx->op_params[7], d2 = octx->op_params[8];
const uint32_t IC = ictx->IC;
const uint32_t ID = ictx->ID, IH = ictx->IH, IW = ictx->IW;
const uint32_t KD = ictx->KD, KH = ictx->KH;
const uint32_t Wb = ictx->pe_wb;
const int32_t win0 = (int32_t) c0 * s0 - p0;
const dma_addr_t src_data = src1->data;
const size_t src_channel_stride = src1->nb[3];
const size_t vtcm_channel_stride = (size_t) KD * KH * Wb * sizeof(float);
const size_t batch_offset = (size_t) (in * IC) * src1->nb[3];
for (uint32_t ikd = 0; ikd < KD; ikd++) {
const int32_t iid = (int32_t) iod * s2 + (int32_t) ikd * d2 - p2;
const bool valid_d = (iid >= 0 && iid < (int32_t) ID);
for (uint32_t ikh = 0; ikh < KH; ikh++) {
const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1;
if (!valid_d || iih < 0 || iih >= (int32_t) IH) {
dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0);
continue;
}
const int32_t lo = win0 < 0 ? -win0 : 0;
int32_t hi = (int32_t) IW - win0;
if (hi > (int32_t) Wb) {
hi = (int32_t) Wb;
}
if (hi <= lo) {
dma_queue_push(dma_q, dma_make_data(srcb, 0), 0, 0, 0, 0);
continue;
}
float * vdst = srcb + (size_t) (ikd * KH + ikh) * Wb + lo;
const dma_addr_t vsrc = src_data + batch_offset +
(size_t) iid * src1->nb[2] +
(size_t) iih * src1->nb[1] +
(size_t) (win0 + lo) * src1->nb[0];
dma_queue_push(dma_q, dma_make_data(vdst, vsrc),
vtcm_channel_stride, src_channel_stride,
(size_t) (hi - lo) * sizeof(float), IC);
}
}
}
#define IM2COL_3D_BLOCKED_COMPUTE_BODY(FNAME, DST_CTYPE, COPY_FN, SPLAT_FN) \
static void FNAME(const struct htp_im2col_context * ictx, \
void * restrict dstb_raw, const float * restrict srcb, \
float * restrict scratch, \
uint32_t nb, uint32_t c0, uint32_t iod, uint32_t ioh) { \
DST_CTYPE * restrict dstb = (DST_CTYPE *) dstb_raw; \
const struct htp_ops_context * octx = ictx->octx; \
const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1], s2 = octx->op_params[2]; \
const int32_t p0 = octx->op_params[3], p1 = octx->op_params[4], p2 = octx->op_params[5]; \
const int32_t d0 = octx->op_params[6], d1 = octx->op_params[7], d2 = octx->op_params[8]; \
const uint32_t IC = ictx->IC; \
const uint32_t ID = ictx->ID, IH = ictx->IH, IW = ictx->IW; \
const uint32_t KD = ictx->KD, KH = ictx->KH, KW = ictx->KW; \
const uint32_t patch_stride = ictx->patch_stride; \
const uint32_t Wb = ictx->pe_wb; \
const int32_t win0 = (int32_t) c0 * s0 - p0; \
for (uint32_t j = 0; j < nb; j++) { \
const uint32_t iow = c0 + j; \
const int32_t iiw0 = (int32_t) iow * s0 - p0; \
DST_CTYPE * dst_patch = dstb + (size_t) j * patch_stride; \
for (uint32_t iic = 0; iic < IC; iic++) { \
for (uint32_t ikd = 0; ikd < KD; ikd++) { \
const int32_t iid = (int32_t) iod * s2 + (int32_t) ikd * d2 - p2; \
for (uint32_t ikh = 0; ikh < KH; ikh++) { \
const int32_t iih = (int32_t) ioh * s1 + (int32_t) ikh * d1 - p1; \
DST_CTYPE * out_run = dst_patch + \
(size_t) (iic * KD * KH + ikd * KH + ikh) * KW; \
if (iid < 0 || iid >= (int32_t) ID || iih < 0 || iih >= (int32_t) IH) { \
SPLAT_FN(out_run, 0, KW); \
continue; \
} \
const float * vrow = srcb + (size_t) (iic * KD * KH + ikd * KH + ikh) * Wb; \
if (d0 == 1) { \
const int32_t lo = iiw0 < 0 ? -iiw0 : 0; \
int32_t hi = (int32_t) IW - iiw0; \
if (hi > (int32_t) KW) { \
hi = (int32_t) KW; \
} \
if (hi <= lo) { \
SPLAT_FN(out_run, 0, KW); \
} else { \
if (lo > 0) { \
SPLAT_FN(out_run, 0, (uint32_t) lo); \
} \
COPY_FN((uint8_t *) (out_run + lo), \
(const uint8_t *) (vrow + (iiw0 + lo - win0)), \
(uint32_t) (hi - lo)); \
if (hi < (int32_t) KW) { \
SPLAT_FN(out_run + hi, 0, (uint32_t) (KW - hi)); \
} \
} \
continue; \
} \
for (uint32_t ikw = 0; ikw < KW; ikw += 64) { \
const uint32_t chunk = MIN(KW - ikw, 64); \
for (uint32_t c = 0; c < chunk; c++) { \
const int32_t iiw = iiw0 + (int32_t) (ikw + c) * d0; \
scratch[c] = (iiw < 0 || iiw >= (int32_t) IW) ? 0.0f : vrow[iiw - win0]; \
} \
COPY_FN((uint8_t *) (out_run + ikw), (const uint8_t *) scratch, chunk); \
} \
} \
} \
} \
} \
}
IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f16, uint16_t, hvx_copy_f16_f32_uu, hvx_splat_u16_u)
IM2COL_3D_BLOCKED_COMPUTE_BODY(im2col_3d_blocked_compute_f32, float, hvx_copy_f32_uu, hvx_splat_f32_u)
static void im2col_3d_blocked_dma_thread(unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
struct htp_ops_context * octx = ictx->octx;
struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith];
const struct htp_tensor * restrict dst = octx->dst;
const uint32_t OW = ictx->OW;
const uint32_t KD = ictx->KD, KH = ictx->KH;
const uint32_t kd_kh = KD * KH;
const uint32_t patch_stride = ictx->patch_stride;
const uint32_t owb = ictx->pe_owb;
const uint32_t nbpr = ictx->pe_nbpr;
const uint32_t dst_elem = ictx->dst_elem;
const dma_addr_t dst_data = dst->data;
dma_queue * dma_q = octx->ctx->dma[ith];
uint8_t * srcb_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread;
uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread;
float * srcb2[2] = { (float *) srcb_base, (float *) (srcb_base + ictx->pe_src_row_bytes) };
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
float * scratch = (float *) (srcb_base + 2 * ictx->pe_src_row_bytes);
const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows;
const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith;
const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max);
if (row_start >= row_end) {
return;
}
const uint32_t nrows_local = row_end - row_start;
const uint32_t total_blocks = nrows_local * nbpr;
if (total_blocks == 0) {
return;
}
struct im2col_cursor cur = {
.r = row_start,
.in = ictx->thread_in[ith],
.iod = ictx->thread_iod[ith],
.ioh = ictx->thread_ioh[ith],
.c0 = 0,
};
struct im2col_cursor stage = cur;
for (uint32_t i = 0; i < total_blocks && i < 2; i++) {
dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0);
im2col_3d_blocked_dma_stage(ictx, dma_q, srcb2[i], &stage);
im2col_3d_step_block(&stage, ictx);
}
im2col_3d_blocked_compute_fn_t compute = (im2col_3d_blocked_compute_fn_t) ictx->compute;
for (uint32_t bi = 0; bi < total_blocks; bi++) {
const uint32_t buf = bi & 1u;
float * srcb = srcb2[buf];
void * dstb = dstb2[buf];
const uint32_t nb = MIN(owb, OW - cur.c0);
dma_queue_pop(dma_q);
for (uint32_t p = 0; p < kd_kh; p++) {
dma_queue_pop(dma_q);
}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r);
compute(ictx, dstb, srcb, scratch, nb, cur.c0, cur.iod, cur.ioh);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r);
const dma_addr_t ddr = dst_data + (size_t) (cur.r * OW + cur.c0) * patch_stride * dst_elem;
dma_queue_push(dma_q, dma_make_data(ddr, dstb),
nb * patch_stride * dst_elem, nb * patch_stride * dst_elem,
nb * patch_stride * dst_elem, 1);
if (bi + 2 < total_blocks) {
im2col_3d_blocked_dma_stage(ictx, dma_q, srcb, &stage);
im2col_3d_step_block(&stage, ictx);
}
im2col_3d_step_block(&cur, ictx);
}
dma_queue_flush(dma_q);
}
static void im2col_3d_patchembed_dma_stage(struct htp_im2col_context * ictx, dma_queue * dma_q,
float * srcb, const struct im2col_cursor * c) {
const struct htp_ops_context * octx = ictx->octx;
const uint32_t IC = ictx->IC;
const uint32_t ID = ictx->ID, IH = ictx->IH, IW = ictx->IW;
const uint32_t KD = ictx->KD, KH = ictx->KH;
const dma_addr_t src_data = octx->src[1]->data;
const uint32_t in = c->in, iod = c->iod, ioh = c->ioh;
const size_t src_channel_stride = (size_t) ID * IH * IW * sizeof(float);
const size_t vtcm_channel_stride = (size_t) KD * KH * IW * sizeof(float);
const size_t row_size = (size_t) KH * IW * sizeof(float);
for (uint32_t ikd = 0; ikd < KD; ikd++) {
const uint32_t iid = iod * KD + ikd;
float * vdst = srcb + (size_t) ikd * KH * IW;
const dma_addr_t vsrc = src_data + (size_t) (((in * IC) * ID + iid) * IH + ioh * KH) * IW * sizeof(float);
dma_queue_push(dma_q, dma_make_data(vdst, vsrc),
vtcm_channel_stride, src_channel_stride, row_size, IC);
}
}
static void im2col_3d_patchembed_compute_f16(
const struct htp_im2col_context * ictx,
void * restrict dstb_raw, const float * restrict srcb) {
uint16_t * restrict dstb = (uint16_t *) dstb_raw;
const uint32_t OW = ictx->OW;
const uint32_t IC = ictx->IC;
const uint32_t KD = ictx->KD;
const uint32_t KH = ictx->KH;
const uint32_t KW = ictx->KW;
const uint32_t IW = ictx->IW;
const uint32_t patch_stride = ictx->patch_stride;
for (uint32_t iow = 0; iow < OW; iow++) {
uint16_t * dst_patch = dstb + (size_t) iow * patch_stride;
for (uint32_t iic = 0; iic < IC; iic++) {
for (uint32_t ikd = 0; ikd < KD; ikd++) {
for (uint32_t ikh = 0; ikh < KH; ikh++) {
uint16_t * out_run = dst_patch + (size_t) (iic * KD * KH + ikd * KH + ikh) * KW;
const float * src_run = srcb + (size_t) (iic * KD * KH + ikd * KH + ikh) * IW + (size_t) iow * KW;
hvx_copy_f16_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW);
}
}
}
}
}
static void im2col_3d_patchembed_compute_f32(
const struct htp_im2col_context * ictx,
void * restrict dstb_raw, const float * restrict srcb) {
float * restrict dstb = (float *) dstb_raw;
const uint32_t OW = ictx->OW;
const uint32_t IC = ictx->IC;
const uint32_t KD = ictx->KD;
const uint32_t KH = ictx->KH;
const uint32_t KW = ictx->KW;
const uint32_t IW = ictx->IW;
const uint32_t patch_stride = ictx->patch_stride;
for (uint32_t iow = 0; iow < OW; iow++) {
float * dst_patch = dstb + (size_t) iow * patch_stride;
for (uint32_t iic = 0; iic < IC; iic++) {
for (uint32_t ikd = 0; ikd < KD; ikd++) {
for (uint32_t ikh = 0; ikh < KH; ikh++) {
float * out_run = dst_patch + (size_t) (iic * KD * KH + ikd * KH + ikh) * KW;
const float * src_run = srcb + (size_t) (iic * KD * KH + ikd * KH + ikh) * IW + (size_t) iow * KW;
hvx_copy_f32_uu((uint8_t *) out_run, (const uint8_t *) src_run, KW);
}
}
}
}
}
static void im2col_3d_patchembed_dma_thread(unsigned int nth, unsigned int ith, void * data) {
(void) nth;
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data;
struct htp_ops_context * octx = ictx->octx;
struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith];
const struct htp_tensor * restrict dst = octx->dst;
const uint32_t KD = ictx->KD;
const uint32_t OW = ictx->OW;
const uint32_t patch_stride = ictx->patch_stride;
const uint32_t dst_elem = ictx->dst_elem;
const dma_addr_t dst_data = dst->data;
dma_queue * dma_q = octx->ctx->dma[ith];
const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows;
const uint32_t row_start = ictx->pe_row_base + ictx->pe_rows_per_thread * ith;
const uint32_t row_end = MIN(row_start + ictx->pe_rows_per_thread, row_end_max);
if (row_start >= row_end) {
return;
}
const uint32_t nrows_local = row_end - row_start;
uint8_t * src_base = ictx->pe_vtcm_src + ith * ictx->pe_src_size_per_thread;
uint8_t * dstb_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread;
float * srcb2[2] = { (float *) src_base, (float *) (src_base + ictx->pe_src_row_bytes) };
void * dstb2[2] = { dstb_base, dstb_base + ictx->pe_dst_row_bytes };
struct im2col_cursor cur = {
.r = row_start,
.in = ictx->thread_in[ith],
.iod = ictx->thread_iod[ith],
.ioh = ictx->thread_ioh[ith],
.c0 = 0,
};
struct im2col_cursor stage = cur;
for (uint32_t i = 0; i < nrows_local && i < 2; i++) {
dma_queue_push(dma_q, dma_make_data(dst_data, dstb2[i]), 0, 0, 0, 0);
im2col_3d_patchembed_dma_stage(ictx, dma_q, srcb2[i], &stage);
im2col_3d_step_row(&stage, ictx);
}
im2col_3d_patchembed_compute_fn_t compute = (im2col_3d_patchembed_compute_fn_t) ictx->compute;
for (uint32_t idx = 0; idx < nrows_local; idx++) {
const uint32_t buf = idx & 1u;
float * srcb = srcb2[buf];
void * dstb = dstb2[buf];
dma_queue_pop(dma_q);
for (uint32_t p = 0; p < KD; p++) {
dma_queue_pop(dma_q);
}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur.r);
compute(ictx, dstb, srcb);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur.r);
const dma_addr_t ddr_row = dst_data + (size_t) cur.r * OW * patch_stride * dst_elem;
dma_queue_push(dma_q, dma_make_data(ddr_row, dstb),
OW * patch_stride * dst_elem, OW * patch_stride * dst_elem,
OW * patch_stride * dst_elem, 1);
if (idx + 2 < nrows_local) {
im2col_3d_patchembed_dma_stage(ictx, dma_q, srcb, &stage);
im2col_3d_step_row(&stage, ictx);
}
im2col_3d_step_row(&cur, ictx);
}
dma_queue_flush(dma_q);
}
int op_im2col(struct htp_ops_context * octx) {
const struct htp_im2col_kernel_params * kparams = (const struct htp_im2col_kernel_params *) octx->kernel_params;
if (kparams->kernel_type == HTP_IM2COL_KERNEL_UNSUPPORTED) {
return HTP_STATUS_NO_SUPPORT;
}
const struct htp_tensor * dst = octx->dst;
uint32_t row_base = 0;
uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range =
htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_base = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
struct htp_im2col_vtcm_layout layout;
htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);
struct htp_im2col_context ictx = { 0 };
uint8_t * const base = octx->ctx->vtcm_base;
const uint32_t pth = octx->n_threads;
ictx.octx = octx;
ictx.pe_row_base = row_base;
ictx.pe_nrows = nrows;
ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);
ictx.pe_src_row_bytes = kparams->src_row_bytes;
ictx.pe_dst_row_bytes = kparams->dst_row_bytes;
ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);
ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst);
ictx.pe_src_size_per_thread = layout.src_bytes_per_thread;
ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread;
ictx.pe_owb = kparams->owb;
ictx.pe_wb = kparams->wb;
ictx.pe_nbpr = kparams->nbpr;
ictx.N = kparams->N;
ictx.IC = kparams->IC;
ictx.ID = kparams->ID;
ictx.IH = kparams->IH;
ictx.IW = kparams->IW;
ictx.KD = kparams->KD;
ictx.KH = kparams->KH;
ictx.KW = kparams->KW;
ictx.OD = kparams->OD;
ictx.OH = kparams->OH;
ictx.OW = kparams->OW;
ictx.patch_stride = kparams->patch_stride;
for (uint32_t ith = 0; ith < pth; ith++) {
const uint32_t r = row_base + ictx.pe_rows_per_thread * ith;
const uint32_t in = fastdiv(r, &kparams->div_OH);
ictx.thread_ioh[ith] = r - in * kparams->OH;
ictx.thread_in[ith] = in;
}
ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float);
if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) {
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_patchembed_compute_f16
: (void *) im2col_patchembed_compute_f32;
work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
} else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) {
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_blocked_compute_f16
: (void *) im2col_blocked_compute_f32;
work_queue_run(octx->ctx->work_queue, im2col_blocked_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
}
return HTP_STATUS_NO_SUPPORT;
}
int op_im2col_3d(struct htp_ops_context * octx) {
const struct htp_im2col_kernel_params * kparams =
(const struct htp_im2col_kernel_params *) octx->kernel_params;
if (kparams->kernel_type == HTP_IM2COL_KERNEL_UNSUPPORTED) {
return HTP_STATUS_NO_SUPPORT;
}
const struct htp_tensor * dst = octx->dst;
uint32_t row_base = 0;
uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range =
htp_tensor_mdev_partition(kparams->total_rows, htp_tensor_mdev_data_aligned(dst) ? kparams->rows_per_chunk : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_base = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
if (!htp_ops_context_set_n_threads(octx, MIN((uint32_t) kparams->n_threads, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
struct htp_im2col_vtcm_layout layout;
htp_im2col_vtcm_layout_build(&layout, kparams->src_row_bytes, kparams->dst_row_bytes, kparams->n_threads);
struct htp_im2col_context ictx = { 0 };
uint8_t * const base = octx->ctx->vtcm_base;
const uint32_t pth = octx->n_threads;
ictx.octx = octx;
ictx.pe_row_base = row_base;
ictx.pe_nrows = nrows;
ictx.pe_rows_per_thread = fastdiv(nrows + pth - 1, &octx->n_threads_div);
ictx.pe_src_row_bytes = kparams->src_row_bytes;
ictx.pe_dst_row_bytes = kparams->dst_row_bytes;
ictx.pe_vtcm_src = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_src);
ictx.pe_vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, layout.off_dst);
ictx.pe_src_size_per_thread = layout.src_bytes_per_thread;
ictx.pe_dst_size_per_thread = layout.dst_bytes_per_thread;
ictx.pe_owb = kparams->owb;
ictx.pe_wb = kparams->wb;
ictx.pe_nbpr = kparams->nbpr;
ictx.N = kparams->N;
ictx.IC = kparams->IC;
ictx.ID = kparams->ID;
ictx.IH = kparams->IH;
ictx.IW = kparams->IW;
ictx.KD = kparams->KD;
ictx.KH = kparams->KH;
ictx.KW = kparams->KW;
ictx.OD = kparams->OD;
ictx.OH = kparams->OH;
ictx.OW = kparams->OW;
ictx.patch_stride = kparams->patch_stride;
for (uint32_t ith = 0; ith < pth; ith++) {
const uint32_t rem = row_base + ictx.pe_rows_per_thread * ith;
const uint32_t q_oh = fastdiv(rem, &kparams->div_OH);
ictx.thread_ioh[ith] = rem - q_oh * kparams->OH;
const uint32_t in = fastdiv(q_oh, &kparams->div_OD);
ictx.thread_iod[ith] = q_oh - in * kparams->OD;
ictx.thread_in[ith] = in;
}
ictx.dst_elem = dst->type == HTP_TYPE_F16 ? sizeof(__fp16) : sizeof(float);
if (kparams->kernel_type == HTP_IM2COL_KERNEL_PATCHEMBED_DMA) {
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_patchembed_compute_f16
: (void *) im2col_3d_patchembed_compute_f32;
work_queue_run(octx->ctx->work_queue, im2col_3d_patchembed_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
} else if (kparams->kernel_type == HTP_IM2COL_KERNEL_BLOCKED_DMA) {
ictx.compute = dst->type == HTP_TYPE_F16 ? (void *) im2col_3d_blocked_compute_f16
: (void *) im2col_3d_blocked_compute_f32;
work_queue_run(octx->ctx->work_queue, im2col_3d_blocked_dma_thread, &ictx, pth);
return HTP_STATUS_OK;
}
return HTP_STATUS_NO_SUPPORT;
}
llama.cpp/ggml/src/ggml-hexagon/htp/im2col-ops.h
+69
View File
@@ -0,0 +1,69 @@
#ifndef HTP_IM2COL_OPS_H
#define HTP_IM2COL_OPS_H
#include "hex-common.h"
#include "hex-fastdiv.h"
#include "htp-vtcm.h"
#include <stdint.h>
enum htp_im2col_kernel_type {
HTP_IM2COL_KERNEL_UNSUPPORTED = 0,
HTP_IM2COL_KERNEL_PATCHEMBED_DMA = 1,
HTP_IM2COL_KERNEL_BLOCKED_DMA = 2,
};
struct htp_im2col_vtcm_layout {
size_t total_bytes;
size_t off_src;
size_t off_dst;
uint32_t src_bytes_per_thread;
uint32_t dst_bytes_per_thread;
};
static inline void htp_im2col_vtcm_layout_build(
struct htp_im2col_vtcm_layout * L,
uint32_t src_row_bytes,
uint32_t dst_row_bytes,
uint32_t n_threads) {
L->src_bytes_per_thread = 2 * src_row_bytes + 256;
L->dst_bytes_per_thread = 2 * dst_row_bytes;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src, (size_t) L->src_bytes_per_thread * n_threads);
VTCM_LAYOUT_ALLOC(off, off_dst, (size_t) L->dst_bytes_per_thread * n_threads);
L->total_bytes = off;
}
struct htp_im2col_kernel_params {
uint8_t kernel_type;
uint8_t n_threads;
uint32_t vtcm_size;
uint32_t src_row_bytes;
uint32_t dst_row_bytes;
uint32_t owb;
uint32_t wb;
uint32_t nbpr;
uint32_t N;
uint32_t IC;
uint32_t ID, IH, IW;
uint32_t KD, KH, KW;
uint32_t OD, OH, OW;
uint32_t patch_stride;
uint32_t total_rows;
uint32_t rows_per_chunk;
struct fastdiv_values div_OH;
struct fastdiv_values div_OD;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_im2col_kernel_params) <= 128, "htp_im2col_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_im2col_kernel_params) <= 128, "htp_im2col_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_IM2COL_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/main.c
+1464
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp/matmul-ops.c
+4594
View File
File diff suppressed because it is too large. Load diff
llama.cpp/ggml/src/ggml-hexagon/htp/matmul-ops.h
+774
View File
@@ -0,0 +1,774 @@
#ifndef HTP_MATMUL_OPS_H
#define HTP_MATMUL_OPS_H
#include <stdint.h>
#include <stddef.h>
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hex-fastdiv.h"
#include "hex-common.h"
#include "htp-vtcm.h"
#ifdef __cplusplus
extern "C" {
#endif
// --- HMX Tile Constraints ---
#define HTP_MM_HMX_TILE_N_COLS 32
#define HTP_MM_HMX_TILE_N_ROWS 32
#define HTP_MM_HMX_TILE_SIZE (32 * 32 * sizeof(__fp16)) // 2048 bytes
#define HTP_MM_HMX_TILE_N_ELMS 1024
#define HTP_MM_HMX_MIN_NROWS 4
#define HTP_MM_WEIGHT_TILE_K_Q2_0 64
#define HTP_MM_WEIGHT_TILE_SIZE_Q2_0 HTP_TENSOR_TILE_SIZE_Q2_0
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0 640
// --- Activation Tiled Block Sizes (including padding) ---
#define HTP_MM_ACT_BLOCK_K 128
#define HTP_MM_ACT_TILE_SIZE_Q8_0 1152
#define HTP_MM_ACT_TILE_SIZE_Q8_1 1280
#define HTP_MM_ACT_TILE_SIZE_F16 2048
#define HTP_MM_MAX_PREFETCH 16
// --- Solver Cost Model Penalty Weights (HMX-specific) ---
#define HTP_MM_HMX_COST_W_DEQUANT 3 // cost penalty for quantized weight loading/dequantization
#define HTP_MM_HMX_COST_A_CONVERT 2 // cost penalty for activation loading/conversion
// --- DMA Activation Transfer Configuration ---
#define HTP_MM_DMA_ACT_ROWS_PER_STEP 2
#define HTP_MM_DMA_ACT_MULTIPLIER (2 * HTP_MM_DMA_ACT_ROWS_PER_STEP)
enum htp_mm_kernel_type {
HTP_MM_KERNEL_UNSUPPORTED = 0,
// HMX paths
HTP_MM_KERNEL_HMX_2D,
HTP_MM_KERNEL_HMX_BATCHED,
// HVX floating-point paths
HTP_MM_KERNEL_HVX_F16_F16_VTCM,
HTP_MM_KERNEL_HVX_F32_F32_VTCM,
HTP_MM_KERNEL_HVX_BF16_F32_VTCM,
// HVX quantized paths
HTP_MM_KERNEL_HVX_QUANT_ROW, // standard row-wise parallel quantization
HTP_MM_KERNEL_HVX_QUANT_BLOCK, // parallel block-wise quantization
};
// Op-specific struct for precomputed matmul params
struct htp_mm_kernel_params {
uint8_t kernel_type; // enum htp_mm_kernel_type
uint8_t pipeline; // 1 = pipelined execution, 0 = standard
uint8_t unused; // reserved
uint8_t n_hmx; // 1 = use HMX, 0 = use HVX
uint8_t n_threads; // Number of threads to spawn
uint8_t n_act_threads; // Number of threads for activation preparation
uint8_t n_prefetch; // Prefetch lookahead buffers/rows in VTCM
uint8_t n_weights; // Number of weights for fused NX
int32_t m_chunk; // Row chunk size (M chunk)
int32_t n_chunk; // Col chunk size (N chunk)
int32_t tile_size; // Weight tile size
int32_t aligned_tile_size; // Aligned weight tile size (padded to 128)
int32_t act_row_size; // Row size for activation scratchpad
int32_t vtcm_size; // Total required scratchpad size in VTCM
int32_t vtcm_bias_size; // bias scratchpad size in VTCM (fused only)
// Precomputed division values
struct fastdiv_values div_ne12_ne1;
struct fastdiv_values div_ne1;
struct fastdiv_values div_r2;
struct fastdiv_values div_r3;
struct fastdiv_values div_ne12;
struct fastdiv_values div_n_act_threads;
struct fastdiv_values div_ne00_padded;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_mm_kernel_params) <= 128, "htp_matmul_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_mm_kernel_params) <= 128, "htp_matmul_kernel_params is too large for kernel_params blob");
#endif
struct mmid_row_mapping {
uint32_t i1;
uint32_t i2;
};
// Search for optimal (mc, nc) chunk sizes within VTCM budget.
static inline int htp_mm_hmx_compute_chunks(size_t vtcm_total,
size_t overhead,
size_t per_n_cost,
size_t per_m_cost,
size_t per_mn_cost,
size_t m,
size_t n,
size_t m_block_cost,
size_t n_block_cost,
size_t * m_chunk_out,
size_t * n_chunk_out,
size_t * total_out) {
if (m == 0 || n == 0) return -1;
if (vtcm_total <= overhead) return -1;
if (per_n_cost == 0 || per_m_cost == 0 || per_mn_cost == 0) return -1;
const size_t usable = vtcm_total - overhead;
size_t best_cost = SIZE_MAX;
size_t best_tail_waste = SIZE_MAX;
size_t best_mn = 0;
size_t best_m = 0, best_n = 0;
const size_t max_nc_budget = (usable / per_n_cost);
const size_t n_max = hex_align_down(hex_smin((size_t)n, max_nc_budget), HTP_MM_HMX_TILE_N_COLS);
for (size_t nc = n_max; nc >= HTP_MM_HMX_TILE_N_COLS; nc -= HTP_MM_HMX_TILE_N_COLS) {
size_t n_fixed = 0, ncmn = 0, mc_denom = 0;
if (hex_mul_overflow(nc, per_n_cost, &n_fixed)) continue;
if (n_fixed >= usable) goto next_nc;
if (hex_mul_overflow(nc, per_mn_cost, &ncmn)) goto next_nc;
if (hex_add_overflow(per_m_cost, ncmn, &mc_denom) || mc_denom == 0) goto next_nc;
{
size_t remain = usable - n_fixed;
size_t mc = remain / mc_denom;
mc = hex_align_down(mc, HTP_MM_HMX_TILE_N_ROWS);
mc = hex_smin(mc, m);
if (mc == 0) {
goto next_nc;
}
size_t mblocks = ((size_t) m + mc - 1) / mc;
size_t nblocks = ((size_t) n + nc - 1) / nc;
size_t cost = mblocks * m_block_cost + nblocks * n_block_cost;
size_t rem = n % nc;
size_t tail_waste = (rem == 0) ? 0 : (nc - rem);
size_t mn = mc * nc;
if (cost < best_cost ||
(cost == best_cost && tail_waste < best_tail_waste) ||
(cost == best_cost && tail_waste == best_tail_waste && mn > best_mn)) {
best_cost = cost;
best_tail_waste = tail_waste;
best_mn = mn;
best_m = mc;
best_n = nc;
}
}
next_nc:
if (nc == HTP_MM_HMX_TILE_N_COLS) break; // avoid size_t underflow
}
if (best_m == 0 || best_n == 0) return -1;
// Compute exact total (with overflow checks)
size_t t0 = 0, t1 = 0, t2 = 0, mn = 0, total = 0;
if (hex_mul_overflow(best_n, per_n_cost, &t0)) return -1;
if (hex_mul_overflow(best_m, per_m_cost, &t1)) return -1;
if (hex_mul_overflow(best_m, best_n, &mn)) return -1;
if (hex_mul_overflow(mn, per_mn_cost, &t2)) return -1;
if (hex_add_overflow(t0, t1, &total)) return -1;
if (hex_add_overflow(total, t2, &total)) return -1;
if (hex_add_overflow(total, overhead, &total)) return -1;
*m_chunk_out = best_m;
*n_chunk_out = best_n;
*total_out = total;
return 0;
}
// weight types whose tiles carry a per-block offset (x = d * q + m): the activations need block sums (q8_1)
// (Q2_K: per-16 k sums, q8_1_s16)
static inline bool htp_mm_weight_has_offset(int weight_type) {
return weight_type == HTP_TYPE_Q4_1 || weight_type == HTP_TYPE_Q4_K || weight_type == HTP_TYPE_Q5_K ||
weight_type == HTP_TYPE_Q2_K || weight_type == HTP_TYPE_Q2_0;
}
// --- Activation/Row Size Helpers ---
static inline size_t htp_mm_q8_0_tiled_row_size(uint32_t ne) {
const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K);
const uint32_t nb_32 = ne_padded / 32;
return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_0;
}
static inline size_t htp_mm_q8_1_tiled_row_size(uint32_t ne) {
const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K);
const uint32_t nb_32 = ne_padded / 32;
return nb_32 * HTP_MM_ACT_TILE_SIZE_Q8_1;
}
static inline size_t htp_mm_f16_tiled_row_size(uint32_t ne) {
const uint32_t ne_padded = hex_round_up(ne, HTP_MM_ACT_BLOCK_K);
const uint32_t nb_32 = ne_padded / 32;
return nb_32 * HTP_MM_ACT_TILE_SIZE_F16;
}
static inline size_t htp_mm_tiled_act_row_size(int weight_type, uint32_t ne) {
if (weight_type == HTP_TYPE_F16) {
return htp_mm_f16_tiled_row_size(ne);
}
return htp_mm_weight_has_offset(weight_type) ? htp_mm_q8_1_tiled_row_size(ne) : htp_mm_q8_0_tiled_row_size(ne);
}
static inline size_t htp_mm_act_raw_row_size(uint32_t ne) {
return hex_round_up(ne, HTP_MM_ACT_BLOCK_K) * sizeof(float);
}
static inline size_t htp_mm_round_up(size_t n, size_t m) {
return ((n + m - 1) / m) * m;
}
static inline bool htp_mm_hmx_pipeline(uint32_t m) {
return m > 32;
}
static inline void htp_mm_hmx_get_2d_chunk_costs(
int wtype, uint32_t k, bool pipeline, uint32_t aligned_tile_size,
size_t * size_per_n_out, size_t * size_per_m_out, size_t * size_per_mn_out
) {
const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16);
const size_t row_stride = htp_tensor_get_row_size(wtype, k);
const size_t vec_dot_size = k * sizeof(uint16_t);
const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype);
const size_t qweight_row_stride = is_quant ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0;
*size_per_n_out = (pipeline ? 2 : 1) * (is_quant ? qweight_row_stride : row_stride) +
(pipeline ? 2 * vec_dot_size : vec_dot_size);
*size_per_m_out = vec_dot_size;
*size_per_mn_out = (pipeline ? 2 : 1) * sizeof(uint16_t);
}
static inline void htp_mm_hmx_get_batched_chunk_costs(
uint32_t k, uint32_t group_size,
size_t * size_per_n_out, size_t * size_per_m_out, size_t * size_per_mn_out
) {
const size_t vec_dot_size = k * sizeof(uint16_t);
*size_per_n_out = 3 * vec_dot_size;
*size_per_m_out = group_size * vec_dot_size;
*size_per_mn_out = sizeof(uint16_t);
}
static inline size_t htp_mm_hmx_get_2d_overhead(bool pipeline, bool is_matmul_id) {
size_t num_regions = pipeline ? 7 : (is_matmul_id ? 4 : 5);
return num_regions * HTP_MM_HMX_TILE_SIZE + 256;
}
static inline size_t htp_mm_hmx_get_batched_overhead(void) {
return 5 * HTP_MM_HMX_TILE_SIZE + 256;
}
struct htp_mm_hmx_vtcm_layout {
// Byte offsets from vtcm_base for each region
size_t off_weight[2]; // [1] is only used when pipelined
size_t off_act;
size_t off_act_f32; // fp32 activation conversion scratch
size_t off_dst[2]; // [1] is only used when pipelined
size_t off_scratch[2]; // dequantization scratch pads
size_t off_scales; // HMX scales (256 bytes)
size_t off_bias; // bias in VTCM
// Cached sizes of regions for HMX kernel use
size_t weight_area_bytes;
size_t act_area_bytes;
size_t act_f32_bytes;
size_t output_area_bytes;
size_t scratch_bytes[2];
size_t act_head_stride;
size_t bias_bytes;
size_t total_bytes;
};
struct htp_mm_hvx_vtcm_layout {
// Byte offsets from vtcm_base for each region
size_t off_act; // vtcm_act (activation)
size_t off_src0; // vtcm_src0 (weight/Wk)
size_t off_bias; // vtcm_bias (bias / fused add only)
size_t off_dst; // vtcm_dst (output scratch)
size_t off_act_raw; // vtcm_act_raw (raw activation DMA staging)
// Cached sizes
size_t src0_bytes;
size_t act_bytes;
size_t bias_bytes;
size_t dst_bytes;
size_t act_raw_bytes;
size_t total_bytes;
};
static inline void htp_mm_hmx_vtcm_layout_build(
struct htp_mm_hmx_vtcm_layout * L,
int kernel_type,
int wtype,
uint32_t k,
size_t mc,
size_t nc,
uint32_t group_size,
bool pipeline,
uint32_t act_threads,
uint32_t aligned_tile_size,
size_t bias_size
) {
size_t off = 0;
if (kernel_type == HTP_MM_KERNEL_HMX_BATCHED) {
const size_t vec_dot_size = k * sizeof(uint16_t);
const size_t act_head_stride = mc * k;
const size_t weight_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE);
const size_t activation_area_size = hex_align_up(group_size * act_head_stride * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE);
const size_t output_area_size = hex_align_up(group_size * mc * nc * sizeof(uint16_t), HTP_MM_HMX_TILE_SIZE);
const size_t scratch_area_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE);
const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128);
// Group A: Permanent activation tiles and scales
size_t off_group_a = 0;
VTCM_LAYOUT_ALLOC(off_group_a, off_act, activation_area_size);
VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_bias, hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE), bias_size > 0);
// Group B: Compute-only buffers (starts at off_group_a)
size_t off_group_b = off_group_a;
VTCM_LAYOUT_ALLOC(off_group_b, off_weight[0], weight_area_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_weight[1], weight_area_size, false);
VTCM_LAYOUT_ALLOC(off_group_b, off_dst[0], output_area_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_dst[1], output_area_size, false);
VTCM_LAYOUT_ALLOC(off_group_b, off_scratch[0], scratch_area_size);
VTCM_LAYOUT_ALLOC(off_group_b, off_scratch[1], scratch_area_size);
const size_t group_b_size = off_group_b - off_group_a;
// Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a)
const size_t max_f32_size = act_threads * 64 * k * sizeof(float);
const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128);
size_t off_group_c = off_group_a;
VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size);
const size_t group_c_size = off_group_c - off_group_a;
L->weight_area_bytes = weight_area_size;
L->act_area_bytes = activation_area_size;
L->act_f32_bytes = act_f32_size;
L->output_area_bytes = output_area_size;
L->scratch_bytes[0] = scratch_area_size;
L->scratch_bytes[1] = scratch_area_size;
L->act_head_stride = act_head_stride;
L->bias_bytes = bias_size;
off = off_group_a + hex_smax(group_b_size, group_c_size);
} else {
// HTP_MM_KERNEL_HMX_2D
const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16);
const size_t row_stride = htp_tensor_get_row_size(wtype, k);
const size_t vec_dot_size = k * sizeof(uint16_t);
const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype);
const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128);
const size_t weight_area_size = is_quant
? hex_align_up((nc / 32) * n_weight_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE)
: hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE);
const size_t act_area_size = hex_align_up(mc * vec_dot_size, HTP_MM_HMX_TILE_SIZE);
const size_t output_area_size = hex_align_up(mc * nc * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE);
const size_t scratch0_size = hex_align_up(nc * vec_dot_size, HTP_MM_HMX_TILE_SIZE);
const size_t scratch1_size = pipeline ? scratch0_size : 0;
// Group A: Scales and activation tiles (must not overlap with Group B or C)
size_t off_group_a = 0;
VTCM_LAYOUT_ALLOC(off_group_a, off_scales, HTP_MM_HMX_TILE_SIZE); // Padded to 2K for alignment and future persistent data
VTCM_LAYOUT_ALLOC(off_group_a, off_act, act_area_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_bias, hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE), bias_size > 0);
// Group B: Compute-only buffers (starts at off_group_a)
size_t off_group_b = off_group_a;
VTCM_LAYOUT_ALLOC(off_group_b, off_weight[0], weight_area_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_weight[1], weight_area_size, pipeline);
VTCM_LAYOUT_ALLOC(off_group_b, off_dst[0], output_area_size);
VTCM_LAYOUT_ALLOC(off_group_b, off_scratch[0], scratch0_size);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_scratch[1], scratch0_size, pipeline);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_dst[1], output_area_size, pipeline);
const size_t group_b_size = off_group_b - off_group_a;
// Group C: Activation prep temporary buffer (overlaps Group B, starting at off_group_a)
const size_t max_f32_size = act_threads * 64 * k * sizeof(float);
const size_t act_f32_size = hex_align_up(hex_smin(max_f32_size, hex_smax(min_f32_size, group_b_size)), 128);
size_t off_group_c = off_group_a;
VTCM_LAYOUT_ALLOC(off_group_c, off_act_f32, act_f32_size);
const size_t group_c_size = off_group_c - off_group_a;
L->weight_area_bytes = weight_area_size;
L->act_area_bytes = act_area_size;
L->act_f32_bytes = act_f32_size;
L->output_area_bytes = output_area_size;
L->scratch_bytes[0] = scratch0_size;
L->scratch_bytes[1] = scratch1_size;
L->act_head_stride = 0;
L->bias_bytes = bias_size;
off = off_group_a + hex_smax(group_b_size, group_c_size);
}
L->total_bytes = off;
}
static inline void htp_mm_hvx_vtcm_layout_build(
struct htp_mm_hvx_vtcm_layout * L,
int kernel_type,
int wtype,
uint32_t ne10, // k
uint32_t act_nrows, // m_total
uint32_t n_threads,
size_t dst_row_size,
size_t src0_row_size,
size_t act_row_size,
size_t bias_row_size,
uint32_t n_prefetch,
bool is_matmul_id,
bool is_fused_nx
) {
(void)act_row_size;
size_t src0_sz = 0;
size_t act_sz = 0;
size_t bias_sz = bias_row_size > 0 ? htp_mm_round_up(bias_row_size, 128) : 0;
size_t dst_sz = 0;
size_t act_raw_sz = 0;
if (is_fused_nx) {
uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
size_t weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128);
size_t tiled_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
size_t q_act_sz = hex_round_up(tiled_act_row_size * act_nrows, 128);
size_t raw_row_size = htp_mm_act_raw_row_size(ne10);
src0_sz = weight_sz_per_thread * n_threads; // shared single-weight prefetch buffer
act_sz = q_act_sz; // quantized activation buffer
bias_sz = 0;
dst_sz = 0;
act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128);
} else if (is_matmul_id) {
const size_t act_row_size_tiled = htp_mm_tiled_act_row_size(wtype, ne10);
const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
const uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
const uint32_t tile_row_size = n_k_tiles * aligned_tile_size;
size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * tile_row_size, 256);
act_sz = htp_mm_round_up(act_row_size_tiled * act_nrows, 256);
size_t raw_row_size = htp_mm_act_raw_row_size(ne10);
src0_sz = src0_sz_per_thread * n_threads;
dst_sz = 0;
bias_sz = 0;
act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128);
} else {
const size_t src0_row_size_padded = htp_mm_round_up(src0_row_size, 128);
const size_t dst_nrows = (act_nrows > 1) ? 0 : 1;
switch (kernel_type) {
case HTP_MM_KERNEL_HVX_F16_F16_VTCM: {
size_t f16_act_row_size = htp_mm_round_up(ne10 * 2, 128);
act_sz = htp_mm_round_up(f16_act_row_size * act_nrows, 256);
src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads;
dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0;
act_raw_sz = hex_round_up(hex_round_up(ne10 * sizeof(float), 128) * act_nrows, 128);
break;
}
case HTP_MM_KERNEL_HVX_F32_F32_VTCM:
case HTP_MM_KERNEL_HVX_BF16_F32_VTCM: {
size_t f32_act_row_size = htp_mm_round_up(ne10 * 4, 128);
act_sz = htp_mm_round_up(f32_act_row_size * act_nrows, 256);
src0_sz = htp_mm_round_up(n_prefetch * src0_row_size_padded, 256) * n_threads;
dst_sz = dst_nrows > 0 ? htp_mm_round_up(dst_row_size, 128) * n_threads : 0;
act_raw_sz = 0;
break;
}
case HTP_MM_KERNEL_HVX_QUANT_BLOCK:
case HTP_MM_KERNEL_HVX_QUANT_ROW: {
size_t q_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10);
uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype);
uint32_t n_weight_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype);
uint32_t tile_row_size = n_weight_tiles * aligned_tile_size;
size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256);
src0_sz = tiled_vtcm_size * n_threads;
act_sz = htp_mm_round_up(q_act_row_size * act_nrows, 256);
size_t dst_slice_per_thread = (dst_nrows > 0 && act_nrows == 1) ? htp_mm_round_up((dst_row_size + n_threads - 1) / n_threads, 128) : 0;
dst_sz = dst_slice_per_thread * n_threads;
size_t raw_row_size = htp_mm_act_raw_row_size(ne10);
act_raw_sz = hex_round_up(raw_row_size * act_nrows, 128);
break;
}
default:
break;
}
}
// Group A: Persistent buffers across chunk compute
size_t off_group_a = 0;
VTCM_LAYOUT_ALLOC(off_group_a, off_act, act_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_a, off_bias, bias_sz, bias_sz > 0);
// Group B: Compute-only buffers (starts at off_group_a)
size_t off_group_b = off_group_a;
VTCM_LAYOUT_ALLOC(off_group_b, off_src0, src0_sz);
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_b, off_dst, dst_sz, dst_sz > 0);
const size_t group_b_size = off_group_b - off_group_a;
// Group C: Raw activation staging buffer (overlaps Group B, starts at off_group_a)
size_t off_group_c = off_group_a;
VTCM_LAYOUT_ALLOC_OPTIONAL(off_group_c, off_act_raw, act_raw_sz, act_raw_sz > 0);
const size_t group_c_size = off_group_c - off_group_a;
L->src0_bytes = src0_sz;
L->act_bytes = act_sz;
L->bias_bytes = bias_sz;
L->dst_bytes = dst_sz;
L->act_raw_bytes = act_raw_sz;
L->total_bytes = off_group_a + hex_smax(group_b_size, group_c_size);
}
static inline bool htp_mm_hvx_solve_vtcm_params(
int kernel_type,
int wtype,
uint32_t ne10,
uint32_t act_nrows,
uint32_t n_threads,
size_t dst_row_size,
size_t src0_row_size,
size_t act_row_size,
size_t bias_row_size,
uint32_t n_prefetch,
size_t vtcm_budget,
struct htp_mm_hvx_vtcm_layout * L_out,
uint32_t * m_chunk_out
) {
struct htp_mm_hvx_vtcm_layout L;
htp_mm_hvx_vtcm_layout_build(
&L, kernel_type, wtype, ne10, act_nrows, n_threads,
dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false
);
if (L.total_bytes <= vtcm_budget) {
*L_out = L;
*m_chunk_out = act_nrows;
return true;
}
const size_t fixed_bytes = L.src0_bytes + L.bias_bytes + L.dst_bytes;
if (vtcm_budget <= fixed_bytes) {
return false;
}
const size_t avail_act = vtcm_budget - fixed_bytes;
size_t row_size = 0;
if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
row_size = htp_mm_tiled_act_row_size(wtype, ne10);
} else if (kernel_type == HTP_MM_KERNEL_HVX_F16_F16_VTCM) {
row_size = hex_round_up(ne10 * 2, 128);
} else {
row_size = hex_round_up(ne10 * 4, 128);
}
if (row_size == 0) {
return false;
}
size_t eff_row_size = row_size;
if (kernel_type == HTP_MM_KERNEL_HVX_QUANT_ROW || kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
eff_row_size += htp_mm_act_raw_row_size(ne10);
}
uint32_t m_chunk = (uint32_t) (avail_act / eff_row_size);
if (m_chunk > 1) {
m_chunk &= ~1U;
}
if (m_chunk > act_nrows) {
m_chunk = act_nrows;
}
if (m_chunk < 1) {
return false;
}
htp_mm_hvx_vtcm_layout_build(
&L, kernel_type, wtype, ne10, m_chunk, n_threads,
dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false
);
while (m_chunk > 2 && L.total_bytes > vtcm_budget) {
m_chunk -= 2;
htp_mm_hvx_vtcm_layout_build(
&L, kernel_type, wtype, ne10, m_chunk, n_threads,
dst_row_size, src0_row_size, act_row_size, bias_row_size, n_prefetch, false, false
);
}
if (L.total_bytes <= vtcm_budget) {
*L_out = L;
*m_chunk_out = m_chunk;
return true;
}
return false;
}
static inline size_t htp_mm_hmx_get_2d_vtcm_size(
int wtype, uint32_t k, size_t mc, size_t nc, bool pipeline, uint32_t act_threads, uint32_t aligned_tile_size, size_t bias_size
) {
struct htp_mm_hmx_vtcm_layout L;
htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, wtype, k, mc, nc, 1, pipeline, act_threads, aligned_tile_size, bias_size);
return L.total_bytes;
}
static inline size_t htp_mm_hmx_get_batched_vtcm_size(
int wtype, uint32_t k, size_t mc, size_t nc, uint32_t group_size, bool pipeline, uint32_t act_threads, size_t bias_size) {
(void)pipeline;
struct htp_mm_hmx_vtcm_layout L;
htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_BATCHED, wtype, k, mc, nc, group_size, false, act_threads, 0, bias_size);
return L.total_bytes;
}
static inline bool htp_mm_hmx_solve_batched_params(
int wtype,
uint32_t k,
uint32_t ne01_padded,
uint32_t ne11,
uint32_t group_size,
int n_threads,
bool pipeline,
size_t bias_size,
size_t vtcm_budget,
size_t * m_chunk_out,
size_t * n_chunk_out,
int * act_threads_out,
size_t * vtcm_size_out
) {
size_t best_mblocks = SIZE_MAX;
int best_act_threads = 0;
size_t best_m_chunk = 0;
size_t best_n_chunk = 0;
size_t best_vtcm_size = 0;
int act_threads = n_threads;
while (act_threads >= 1) {
size_t group_overhead = htp_mm_hmx_get_batched_overhead() + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0);
size_t group_size_per_n, group_size_per_m, group_size_per_mn;
htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn);
size_t m_chunk_candidate = 0;
size_t n_chunk_candidate = 0;
size_t vtcm_size_candidate = 0;
if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded,
(size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT,
&m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {
size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, pipeline, act_threads, bias_size);
if (exact_size <= vtcm_budget) {
size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate;
if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {
best_mblocks = mblocks;
best_act_threads = act_threads;
best_m_chunk = m_chunk_candidate;
best_n_chunk = n_chunk_candidate;
best_vtcm_size = exact_size;
}
}
}
if (act_threads == 1) {
act_threads = 0;
} else {
act_threads /= 2;
}
}
if (best_act_threads > 0) {
*m_chunk_out = best_m_chunk;
*n_chunk_out = best_n_chunk;
*vtcm_size_out = best_vtcm_size;
*act_threads_out = best_act_threads;
return true;
}
return false;
}
static inline bool htp_mm_hmx_solve_2d_params(
int wtype,
uint32_t k,
uint32_t m_id_rows,
uint32_t ne01_padded,
uint32_t ne11_padded,
uint32_t m_for_cost,
int n_threads,
bool pipeline,
bool is_matmul_id,
uint32_t aligned_tile_size,
size_t bias_size,
size_t vtcm_budget,
size_t * m_chunk_out,
size_t * n_chunk_out,
int * act_threads_out,
size_t * vtcm_size_out
) {
size_t best_mblocks = SIZE_MAX;
int best_act_threads = 0;
size_t best_m_chunk = 0;
size_t best_n_chunk = 0;
size_t best_vtcm_size = 0;
const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded;
int act_threads = n_threads;
while (act_threads >= 1) {
size_t simple_2d_overhead = htp_mm_hmx_get_2d_overhead(pipeline, is_matmul_id) + (bias_size > 0 ? hex_align_up(bias_size, HTP_MM_HMX_TILE_SIZE) : 0);
size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn;
htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn);
size_t m_chunk_candidate = 0;
size_t n_chunk_candidate = 0;
size_t vtcm_size_candidate = 0;
if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded,
(size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT,
&m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {
size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size, bias_size);
if (exact_size <= vtcm_budget) {
size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate;
if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {
best_mblocks = mblocks;
best_act_threads = act_threads;
best_m_chunk = m_chunk_candidate;
best_n_chunk = n_chunk_candidate;
best_vtcm_size = exact_size;
}
}
}
if (act_threads == 1) {
act_threads = 0;
} else {
act_threads /= 2;
}
}
if (best_act_threads > 0) {
*m_chunk_out = best_m_chunk;
*n_chunk_out = best_n_chunk;
*vtcm_size_out = best_vtcm_size;
*act_threads_out = best_act_threads;
return true;
}
return false;
}
#ifdef __cplusplus
}
#endif
#endif // HTP_MATMUL_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/pad-ops.c
+588
View File
@@ -0,0 +1,588 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic ignored "-Wunused-function"
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <string.h>
#include "dma-queue.h"
#include "hvx-utils.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
#include "pad-ops.h"
/* Circular wrap: maps any integer x into [0, n) */
static inline uint32_t wrap_around(int32_t x, uint32_t n, const struct fastdiv_values * div_n) {
if (x >= 0) {
const uint32_t ux = (uint32_t) x;
return ux < n ? ux : fastmodulo(ux, n, div_n);
}
const uint32_t ux = (uint32_t) (-x);
const uint32_t rem = ux < n ? ux : fastmodulo(ux, n, div_n);
return rem ? (n - rem) : 0;
}
/* Decompose a flat dst row index into (i1, i2, i3) */
static inline void pad_decompose_row(uint32_t ir, uint32_t ne1, uint32_t ne2,
const struct htp_pad_kernel_params * kparams,
uint32_t *i1, uint32_t *i2, uint32_t *i3) {
*i3 = fastdiv(ir, &kparams->div_ne1_ne2);
const uint32_t rem = ir - (*i3) * (ne1 * ne2);
*i2 = fastdiv(rem, &kparams->div_ne1);
*i1 = rem - (*i2) * ne1;
}
/* Return non-zero if row (i1,i2,i3) falls in the non-padded interior */
static inline int pad_is_interior(uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t rp1, uint32_t ne1,
int32_t lp2, int32_t rp2, uint32_t ne2,
int32_t lp3, int32_t rp3, uint32_t ne3) {
return ((int32_t)i1 >= lp1 && (int32_t)i1 < (int32_t)ne1 - rp1) &&
((int32_t)i2 >= lp2 && (int32_t)i2 < (int32_t)ne2 - rp2) &&
((int32_t)i3 >= lp3 && (int32_t)i3 < (int32_t)ne3 - rp3);
}
/* Compute the DDR src row pointer for a zero-pad interior row */
static inline const uint8_t * pad_src_row_ptr(const struct htp_tensor * src,
uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t lp2, int32_t lp3) {
return (const uint8_t *) src->data
+ (i1 - (uint32_t)lp1) * src->nb[1]
+ (i2 - (uint32_t)lp2) * src->nb[2]
+ (i3 - (uint32_t)lp3) * src->nb[3];
}
static inline dma_addr_t pad_src_row_data(const struct htp_tensor * src,
uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t lp2, int32_t lp3) {
return src->data
+ (i1 - (uint32_t)lp1) * src->nb[1]
+ (i2 - (uint32_t)lp2) * src->nb[2]
+ (i3 - (uint32_t)lp3) * src->nb[3];
}
/* Compute the DDR src row pointer for a circular row (wrap-around indexing) */
static inline const uint8_t * pad_circ_src_row_ptr(const struct htp_tensor * src,
const struct htp_pad_kernel_params * kparams,
uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t lp2, int32_t lp3) {
return (const uint8_t *) src->data
+ wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1]
+ wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2]
+ wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3];
}
static inline dma_addr_t pad_circ_src_row_data(const struct htp_tensor * src,
const struct htp_pad_kernel_params * kparams,
uint32_t i1, uint32_t i2, uint32_t i3,
int32_t lp1, int32_t lp2, int32_t lp3) {
return src->data
+ wrap_around((int32_t)i1 - lp1, src->ne[1], &kparams->div_ne01) * src->nb[1]
+ wrap_around((int32_t)i2 - lp2, src->ne[2], &kparams->div_ne02) * src->nb[2]
+ wrap_around((int32_t)i3 - lp3, src->ne[3], &kparams->div_ne03) * src->nb[3];
}
struct htp_pad_context {
struct htp_ops_context * octx;
const struct htp_pad_kernel_params * kparams;
int32_t lp0, rp0;
int32_t lp1, rp1;
int32_t lp2, rp2;
int32_t lp3, rp3;
uint32_t nrows_per_thread;
uint32_t total_dst_rows;
uint32_t row_start;
size_t type_size;
// Row sizes for DMA kernel (populated when VTCM is available)
size_t src_row_size;
size_t src_row_size_aligned;
size_t dst_row_size;
size_t dst_row_size_aligned;
};
#define htp_pad_preamble \
const struct htp_tensor * src = octx->src[0]; \
const struct htp_tensor * dst = octx->dst; \
\
const uint32_t ne00 = src->ne[0]; \
const uint32_t nb00 = src->nb[0]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3]; \
\
const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \
const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \
const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \
const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \
\
const size_t type_size = pctx->type_size; \
\
const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \
const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows);
#define htp_pad_dma_preamble \
const size_t src_row_size = pctx->src_row_size; \
const size_t src_row_size_aligned = pctx->src_row_size_aligned; \
const size_t dst_row_size = pctx->dst_row_size; \
const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \
\
uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \
uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \
\
dma_queue * dma_q = octx->ctx->dma[ith];
// ---------------------------------------------------------------------------
// HVX vectorized PAD kernel
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
uint32_t i1, i2, i3;
pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
const int interior = pad_is_interior(i1, i2, i3,
lp1, rp1, ne1,
lp2, rp2, ne2,
lp3, rp3, ne3);
if (!interior) {
hvx_splat_f32_u(dst_ptr, 0.0f, ne0);
} else {
const uint8_t * src_ptr = pad_src_row_ptr(src, i1, i2, i3, lp1, lp2, lp3);
if (lp0 > 0) {
hvx_splat_f32_u(dst_ptr, 0.0f, (uint32_t)lp0);
}
uint8_t * dst_row_start = dst_ptr + (size_t)lp0 * type_size;
if (nb00 == type_size) {
hvx_copy_f32_uu(dst_row_start, src_ptr, ne00);
} else {
for (uint32_t i = 0; i < ne00; i++) {
memcpy(dst_row_start + i * type_size,
src_ptr + (size_t)i * nb00,
type_size);
}
}
if (rp0 > 0) {
hvx_splat_f32_u(dst_ptr + ((size_t)lp0 + ne00) * type_size, 0.0f, (uint32_t)rp0);
}
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end);
}
// ---------------------------------------------------------------------------
// HVX + DMA PAD kernel - aligned, double-buffered
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
htp_pad_dma_preamble;
// -----------------------------------------------------------------------
// Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the
// double-buffer pipeline before the main loop begins.
// -----------------------------------------------------------------------
for (uint32_t ir = row_start, spad_idx = 0; ir < row_end && spad_idx < 2; ir++, spad_idx++) {
uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned;
uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned;
dma_queue_push(dma_q,
dma_make_data(dst->data, dst_spad_cur),
dst_row_size, dst_row_size_aligned, dst_row_size, 0);
uint32_t i1, i2, i3;
pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
const int interior = pad_is_interior(i1, i2, i3,
lp1, rp1, ne1,
lp2, rp2, ne2,
lp3, rp3, ne3);
const dma_addr_t src_data = interior
? pad_src_row_data(src, i1, i2, i3, lp1, lp2, lp3) : src->data;
// Interior row: real DMA (1 row) from DDR to VTCM.
// Border row: null DMA (nrows=0)
dma_queue_push(dma_q,
dma_make_data(src_spad_cur, src_data),
src_row_size_aligned, src_row_size, src_row_size, interior ? 1 : 0);
}
// -----------------------------------------------------------------------
// Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops,
// push dst DMA and prefetch src for the next+1 row.
// -----------------------------------------------------------------------
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = row_start; ir < row_end; ir++) {
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src;
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst;
uint32_t i1, i2, i3;
pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
const int interior = pad_is_interior(i1, i2, i3,
lp1, rp1, ne1,
lp2, rp2, ne2,
lp3, rp3, ne3);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
if (!interior) {
hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0);
} else {
hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0);
uint8_t * dst_interior = dst_spad_cur + (size_t)lp0 * type_size;
if ((uintptr_t)dst_interior % VLEN == 0) {
hvx_copy_f32_aa(dst_interior, src_spad_cur, ne00);
} else {
hvx_copy_f32_ua(dst_interior, src_spad_cur, ne00);
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
dma_queue_push(dma_q,
dma_make_data(dst_data, dst_spad_cur),
dst_row_size, dst_row_size_aligned, dst_row_size, 1);
const uint32_t next_row = ir + 2;
if (next_row < row_end) {
uint32_t ni1, ni2, ni3;
pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &ni1, &ni2, &ni3);
const int next_interior = pad_is_interior(ni1, ni2, ni3,
lp1, rp1, ne1,
lp2, rp2, ne2,
lp3, rp3, ne3);
const dma_addr_t next_src_data = next_interior
? pad_src_row_data(src, ni1, ni2, ni3, lp1, lp2, lp3) : src->data;
dma_queue_push(dma_q,
dma_make_data(src_spad_cur, next_src_data),
src_row_size_aligned, src_row_size, src_row_size, next_interior ? 1 : 0);
}
}
dma_queue_flush(dma_q);
FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end);
}
// ---------------------------------------------------------------------------
// HVX circular PAD kernel
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
uint32_t i1, i2, i3;
pad_decompose_row(dst_row, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
const uint8_t * src_row = pad_circ_src_row_ptr(src, pctx->kparams, i1, i2, i3, lp1, lp2, lp3);
if (nb00 == type_size) {
if (lp0 > 0) {
if ((uint32_t)lp0 < 32) {
memcpy(dst_ptr,
src_row + (size_t)(ne00 - (uint32_t)lp0) * type_size,
(size_t)lp0 * type_size);
} else {
hvx_copy_f32_uu(dst_ptr,
src_row + (size_t)(ne00 - (uint32_t)lp0) * type_size,
(uint32_t)lp0);
}
}
hvx_copy_f32_uu(dst_ptr + (size_t)lp0 * type_size, src_row, ne00);
if (rp0 > 0) {
if ((uint32_t)rp0 < 32) {
memcpy(dst_ptr + ((size_t)lp0 + ne00) * type_size,
src_row,
(size_t)rp0 * type_size);
} else {
hvx_copy_f32_uu(dst_ptr + ((size_t)lp0 + ne00) * type_size,
src_row,
(uint32_t)rp0);
}
}
} else {
for (uint32_t i = 0; i < (uint32_t)lp0; i++) {
*(float *)(dst_ptr + i * type_size) =
*(const float *)(src_row + (size_t)(ne00 - (uint32_t)lp0 + i) * nb00);
}
for (uint32_t i = 0; i < ne00; i++) {
*(float *)(dst_ptr + ((size_t)lp0 + i) * type_size) =
*(const float *)(src_row + (size_t)i * nb00);
}
for (uint32_t i = 0; i < (uint32_t)rp0; i++) {
*(float *)(dst_ptr + ((size_t)lp0 + ne00 + i) * type_size) =
*(const float *)(src_row + (size_t)i * nb00);
}
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end);
}
// ---------------------------------------------------------------------------
// HVX + DMA circular PAD kernel - aligned, double-buffered
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) {
const struct htp_pad_context * pctx = (const struct htp_pad_context *) data;
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
htp_pad_dma_preamble;
// -----------------------------------------------------------------------
// Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the
// double-buffer pipeline. Every row is a real src DMA (no null DMAs).
// -----------------------------------------------------------------------
for (uint32_t ir = row_start, spad_idx = 0; ir < row_end && spad_idx < 2; ir++, spad_idx++) {
uint8_t * src_spad_cur = src_spad_base + spad_idx * src_row_size_aligned;
uint8_t * dst_spad_cur = dst_spad_base + spad_idx * dst_row_size_aligned;
dma_queue_push(dma_q,
dma_make_data(dst->data, dst_spad_cur),
dst_row_size, dst_row_size_aligned, dst_row_size, 0);
uint32_t pi1, pi2, pi3;
pad_decompose_row(ir, ne1, ne2, pctx->kparams, &pi1, &pi2, &pi3);
const dma_addr_t src_data = pad_circ_src_row_data(src, pctx->kparams, pi1, pi2, pi3, lp1, lp2, lp3);
dma_queue_push(dma_q,
dma_make_data(src_spad_cur, src_data),
src_row_size_aligned, src_row_size, src_row_size, 1);
}
// -----------------------------------------------------------------------
// Main loop: pop completed DMAs, assemble circular row in VTCM with
// aligned HVX ops, push dst DMA and prefetch src for the next+1 row.
// -----------------------------------------------------------------------
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = row_start; ir < row_end; ir++) {
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma_q).src;
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma_q).dst;
uint32_t i1, i2, i3;
pad_decompose_row(ir, ne1, ne2, pctx->kparams, &i1, &i2, &i3);
const dma_addr_t dst_data = dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
if (lp0 > 0) {
uint8_t * dst_left = dst_spad_cur;
const uint8_t * src_left = src_spad_cur + (size_t)(ne00 - (uint32_t)lp0) * type_size;
if ((uint32_t)lp0 < 32) {
memcpy(dst_left, src_left, (size_t)lp0 * type_size);
} else {
hvx_copy_f32_uu(dst_left, src_left, (uint32_t)lp0);
}
}
{
uint8_t * dst_mid = dst_spad_cur + (size_t)lp0 * type_size;
if ((uintptr_t)dst_mid % VLEN == 0) {
hvx_copy_f32_aa(dst_mid, src_spad_cur, ne00);
} else {
hvx_copy_f32_ua(dst_mid, src_spad_cur, ne00);
}
}
if (rp0 > 0) {
uint8_t * dst_right = dst_spad_cur + ((size_t)lp0 + ne00) * type_size;
if ((uint32_t)rp0 < 32) {
memcpy(dst_right, src_spad_cur, (size_t)rp0 * type_size);
} else {
if ((uintptr_t)dst_right % VLEN == 0) {
hvx_copy_f32_aa(dst_right, src_spad_cur, (uint32_t)rp0);
} else {
hvx_copy_f32_ua(dst_right, src_spad_cur, (uint32_t)rp0);
}
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
dma_queue_push(dma_q,
dma_make_data(dst_data, dst_spad_cur),
dst_row_size, dst_row_size_aligned, dst_row_size, 1);
const uint32_t next_row = ir + 2;
if (next_row < row_end) {
uint32_t nri1, nri2, nri3;
pad_decompose_row(next_row, ne1, ne2, pctx->kparams, &nri1, &nri2, &nri3);
const dma_addr_t next_src_data = pad_circ_src_row_data(src, pctx->kparams, nri1, nri2, nri3, lp1, lp2, lp3);
dma_queue_push(dma_q,
dma_make_data(src_spad_cur, next_src_data),
src_row_size_aligned, src_row_size, src_row_size, 1);
}
}
dma_queue_flush(dma_q);
FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end);
}
int op_pad(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
const struct htp_pad_kernel_params * kparams =
(const struct htp_pad_kernel_params *) octx->kernel_params;
const int32_t lp0 = octx->op_params[0];
const int32_t rp0 = octx->op_params[1];
const int32_t lp1 = octx->op_params[2];
const int32_t rp1 = octx->op_params[3];
const int32_t lp2 = octx->op_params[4];
const int32_t rp2 = octx->op_params[5];
const int32_t lp3 = octx->op_params[6];
const int32_t rp3 = octx->op_params[7];
const int32_t circular = octx->op_params[8];
const uint32_t total_dst_rows = kparams->total_rows;
const size_t type_size = kparams->elem_size;
const size_t src_row_size = src0->ne[0] * type_size;
const size_t dst_row_size = dst->ne[0] * type_size;
uint32_t row_start = 0;
uint32_t nrows = total_dst_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
if (!htp_ops_context_set_n_threads(octx, MIN(kparams->n_threads, nrows))) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t n_threads = octx->n_threads;
if (!kparams->use_dma && (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
return HTP_STATUS_NO_SUPPORT;
}
struct htp_pad_vtcm_layout layout = { 0 };
if (kparams->use_dma) {
htp_pad_vtcm_layout_build(&layout, src_row_size, dst_row_size, n_threads);
octx->src0_spad.size_per_thread = layout.src0_bytes_per_thread;
octx->dst_spad.size_per_thread = layout.dst_bytes_per_thread;
octx->src0_spad.size = n_threads * layout.src0_bytes_per_thread;
octx->dst_spad.size = n_threads * layout.dst_bytes_per_thread;
octx->src0_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_src0);
octx->dst_spad.data = VTCM_LAYOUT_PTR(uint8_t, octx->ctx->vtcm_base, layout.off_dst);
octx->src0_spad.src = NULL;
octx->dst_spad.src = NULL;
}
struct htp_pad_context pctx = {
.octx = octx,
.kparams = kparams,
.lp0 = lp0, .rp0 = rp0,
.lp1 = lp1, .rp1 = rp1,
.lp2 = lp2, .rp2 = rp2,
.lp3 = lp3, .rp3 = rp3,
.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_dst_rows = nrows,
.row_start = row_start,
.type_size = type_size,
.src_row_size = src_row_size,
.src_row_size_aligned = layout.src_row_size_aligned,
.dst_row_size = dst_row_size,
.dst_row_size_aligned = layout.dst_row_size_aligned,
};
FARF(HIGH, "pad-hvx%s%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) pads=(%d,%d,%d,%d,%d,%d,%d,%d)\n",
circular ? "-circ" : "",
kparams->use_dma ? "-dma" : "",
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3);
switch (kparams->kernel_type) {
case HTP_PAD_KERNEL_HVX_CIRCULAR_DMA:
work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads);
break;
case HTP_PAD_KERNEL_HVX_CIRCULAR:
work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads);
break;
case HTP_PAD_KERNEL_HVX_DMA:
work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads);
break;
default:
work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads);
break;
}
return HTP_STATUS_OK;
}
llama.cpp/ggml/src/ggml-hexagon/htp/pad-ops.h
+65
View File
@@ -0,0 +1,65 @@
#ifndef HTP_PAD_OPS_H
#define HTP_PAD_OPS_H
#include "hex-common.h"
#include "hex-fastdiv.h"
#include "htp-vtcm.h"
#include <stdint.h>
enum htp_pad_kernel_type {
HTP_PAD_KERNEL_HVX = 0,
HTP_PAD_KERNEL_HVX_DMA = 1,
HTP_PAD_KERNEL_HVX_CIRCULAR = 2,
HTP_PAD_KERNEL_HVX_CIRCULAR_DMA = 3,
};
struct htp_pad_vtcm_layout {
size_t total_bytes;
size_t off_src0;
size_t off_dst;
size_t src_row_size_aligned;
size_t dst_row_size_aligned;
size_t src0_bytes_per_thread;
size_t dst_bytes_per_thread;
};
static inline void htp_pad_vtcm_layout_build(
struct htp_pad_vtcm_layout * L,
size_t src_row_size,
size_t dst_row_size,
uint32_t n_threads
) {
L->src_row_size_aligned = hex_round_up(src_row_size, 128);
L->dst_row_size_aligned = hex_round_up(dst_row_size, 128);
L->src0_bytes_per_thread = 2 * L->src_row_size_aligned;
L->dst_bytes_per_thread = 2 * L->dst_row_size_aligned;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src0, L->src0_bytes_per_thread * n_threads);
VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
L->total_bytes = off;
}
struct htp_pad_kernel_params {
uint8_t kernel_type;
uint32_t n_threads;
uint32_t total_rows;
uint32_t use_dma;
uint32_t elem_size;
uint32_t vtcm_size;
struct fastdiv_values div_ne1;
struct fastdiv_values div_ne1_ne2;
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02;
struct fastdiv_values div_ne03;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_pad_kernel_params) <= 128, "htp_pad_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_PAD_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/pool-ops.c
+390
View File
@@ -0,0 +1,390 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#include <float.h>
#include <HAP_farf.h>
#include "hex-common.h"
#include "dma-queue.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-inverse.h"
#include "hvx-types.h"
#include "hvx-utils.h"
#include "pool-ops.h"
#define HTP_POOL_MAX 0
#define HTP_POOL_AVG 1
// Fast path: exact non-overlapping tiling (stride == kernel, no padding), kernel_x in {1,2}.
// Every window is guaranteed fully in-bounds, so this never needs boundary clamping.
static void pool_plane_hvx(
const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) {
const bool is_max = (p->pool_op == HTP_POOL_MAX);
const HVX_Vector scale = hvx_vec_splat_f32(p->inv_kernel_area);
const HVX_Vector seed = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0);
for (uint32_t oy = 0; oy < p->dst_y; ++oy) {
const uint32_t sy = oy * p->kernel_y;
for (uint32_t ox = 0; ox < p->dst_x; ox += VLEN_FP32) {
const uint32_t rem = p->dst_x - ox;
const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
HVX_Vector acc = seed;
for (uint32_t ky = 0; ky < p->kernel_y; ++ky) {
const float * row = src + (sy + ky) * p->src_x;
if (p->kernel_x == 2) {
const HVX_Vector v0 = *(const HVX_UVector *) (row + ox * 2);
const HVX_Vector v1 = *(const HVX_UVector *) (row + ox * 2 + VLEN_FP32);
const HVX_VectorPair deinterleaved = Q6_W_vdeal_VVR(v1, v0, -4);
const HVX_Vector lo = Q6_V_lo_W(deinterleaved);
const HVX_Vector hi = Q6_V_hi_W(deinterleaved);
acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, Q6_Vsf_vmax_VsfVsf(lo, hi))
: hvx_vec_add_f32_f32(acc, hvx_vec_add_f32_f32(lo, hi));
} else if (p->kernel_x == 1) {
const HVX_Vector v = *(const HVX_UVector *) (row + ox);
acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) : hvx_vec_add_f32_f32(acc, v);
}
}
hvx_vec_store_u(dst + oy * p->dst_x + ox, nbytes, is_max ? acc : hvx_vec_mul_f32_f32(acc, scale));
}
}
}
// Narrow exact-tiling path. The input is staged in VTCM with one vector of
// guard space, so full-width loads are safe even when src_x is below 32.
static void pool_plane_hvx_narrow(
const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) {
const bool is_max = (p->pool_op == HTP_POOL_MAX);
const HVX_Vector scale = hvx_vec_splat_f32(p->inv_kernel_area);
const HVX_Vector seed = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0);
for (uint32_t oy = 0; oy < p->dst_y; ++oy) {
const uint32_t sy = oy * p->kernel_y;
HVX_Vector acc = seed;
for (uint32_t ky = 0; ky < p->kernel_y; ++ky) {
const float * row = src + (sy + ky) * p->src_x;
const HVX_Vector v0 = *(const HVX_UVector *) row;
HVX_Vector v = v0;
if (p->kernel_x == 2) {
// The second vector is zero because a narrow row has fewer
// than 32 input elements. The low lanes still contain the
// complete even/odd pairs needed by the output.
const HVX_Vector zero = Q6_V_vsplat_R(0);
const HVX_VectorPair deinterleaved = Q6_W_vdeal_VVR(zero, v0, -4);
const HVX_Vector even = Q6_V_lo_W(deinterleaved);
const HVX_Vector odd = Q6_V_hi_W(deinterleaved);
v = is_max ? Q6_Vsf_vmax_VsfVsf(even, odd)
: hvx_vec_add_f32_f32(even, odd);
}
acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v)
: hvx_vec_add_f32_f32(acc, v);
}
hvx_vec_store_u(dst + oy * p->dst_x, p->dst_x * sizeof(float),
is_max ? acc : hvx_vec_mul_f32_f32(acc, scale));
}
}
static void pool_plane_global(
const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) {
const uint32_t n = p->src_x * p->src_y;
float val;
if (p->pool_op == HTP_POOL_MAX) {
val = hvx_reduce_max_f32((const uint8_t *) src, n);
} else {
val = hvx_reduce_sum_f32((const uint8_t *) src, n) * p->inv_kernel_area;
}
hvx_vec_store_u(dst, sizeof(float), hvx_vec_splat_f32(val));
}
static void pool_plane_block(
const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) {
for (uint32_t oy = 0; oy < p->dst_y; ++oy) {
const float * row = src + oy * p->src_x;
float * dst_row = dst + oy * p->dst_x;
for (uint32_t ox = 0; ox < p->dst_x; ++ox) {
const uint8_t * block = (const uint8_t *) (row + ox * p->kernel_x);
float val;
if (p->pool_op == HTP_POOL_MAX) {
val = hvx_reduce_max_f32(block, p->kernel_x);
} else {
val = hvx_reduce_sum_f32(block, p->kernel_x) * p->inv_kernel_area;
}
hvx_vec_store_u(dst_row + ox, sizeof(float), hvx_vec_splat_f32(val));
}
}
}
// General path: arbitrary kernel/stride/padding
// Vertical padding is uniform across a whole output row (iy0 depends only on oy, not ox),
// so it collapses to one valid-ky range per row instead of a per-element check.
static inline void pool_row_bounds_y(
const struct htp_pool_2d_kernel_params * p, uint32_t oy, int32_t * iy0, uint32_t * ky_lo, uint32_t * ky_hi) {
*iy0 = (int32_t) (oy * p->stride_y) - p->pad_y;
const int32_t lo = -(*iy0);
const int32_t hi = (int32_t) p->src_y - *iy0;
*ky_lo = (uint32_t) MAX(0, lo);
*ky_hi = (uint32_t) MAX(0, MIN((int32_t) p->kernel_y, hi));
}
static inline void pool_pixel_boundary_vec(
const float * src, float * dst_row, const struct htp_pool_2d_kernel_params * p,
uint32_t ox, int32_t iy0, uint32_t ky_lo, uint32_t ky_hi, bool is_max) {
const int32_t ix0 = (int32_t) (ox * p->stride_x) - p->pad_x;
const int32_t kx_lo = MAX(0, -ix0);
const int32_t kx_hi = MIN((int32_t) p->kernel_x, (int32_t) p->src_x - ix0);
if (kx_lo >= kx_hi || ky_lo >= ky_hi) {
HVX_Vector empty_val = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0);
hvx_vec_store_u(dst_row + ox, sizeof(float), empty_val);
return;
}
const uint32_t valid_kx = (uint32_t) (kx_hi - kx_lo);
const HVX_Vector mask_identity = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0);
HVX_Vector acc = mask_identity;
for (uint32_t ky = ky_lo; ky < ky_hi; ++ky) {
const float * row = src + (uint32_t) (iy0 + (int32_t) ky) * p->src_x;
for (uint32_t k = 0; k < valid_kx; k += VLEN_FP32) {
const uint32_t k_rem = valid_kx - k;
const uint32_t n = (k_rem < VLEN_FP32) ? k_rem : VLEN_FP32;
const HVX_VectorPred q = Q6_Q_vsetq_R(n * sizeof(float));
const HVX_Vector raw = *(const HVX_UVector *) (row + ix0 + kx_lo + k);
const HVX_Vector v = Q6_V_vmux_QVV(q, raw, mask_identity);
acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) : hvx_vec_add_f32_f32(acc, v);
}
}
HVX_Vector reduced = is_max ? hvx_vec_reduce_max_f32(acc) : hvx_vec_reduce_sum_f32(acc);
if (!is_max) {
HVX_Vector scale_vec;
if (p->avg_divide_count) {
const uint32_t count = (ky_hi - ky_lo) * valid_kx;
scale_vec = hvx_vec_inverse_f32(hvx_vec_splat_f32((float) count));
} else {
scale_vec = hvx_vec_splat_f32(p->inv_kernel_area);
}
reduced = hvx_vec_mul_f32_f32(reduced, scale_vec);
}
hvx_vec_store_u(dst_row + ox, sizeof(float), reduced);
}
static inline void pool_row_general_boundary_vec(
const float * src, float * dst_row, const struct htp_pool_2d_kernel_params * p,
uint32_t ox_start, uint32_t ox_end, int32_t iy0, uint32_t ky_lo, uint32_t ky_hi, bool is_max) {
for (uint32_t ox = ox_start; ox < ox_end; ++ox) {
pool_pixel_boundary_vec(src, dst_row, p, ox, iy0, ky_lo, ky_hi, is_max);
}
}
// Vectorized interior loop.
static inline void pool_row_general_vec(
const float * src, float * dst_row, const struct htp_pool_2d_kernel_params * p,
uint32_t ox_start, uint32_t ox_end, int32_t iy0, uint32_t ky_lo, uint32_t ky_hi, bool is_max) {
if (ox_start >= ox_end) {
return;
}
if (p->stride_x != 1 && p->stride_x != 2) {
pool_row_general_boundary_vec(src, dst_row, p, ox_start, ox_end, iy0, ky_lo, ky_hi, is_max);
return;
}
const HVX_Vector scale = hvx_vec_splat_f32(p->inv_kernel_area);
const HVX_Vector seed = is_max ? hvx_vec_splat_f32(-FLT_MAX) : Q6_V_vsplat_R(0);
for (uint32_t ox = ox_start; ox < ox_end; ox += VLEN_FP32) {
const uint32_t rem = ox_end - ox;
const uint32_t nbytes = (rem < VLEN_FP32) ? (rem * sizeof(float)) : VLEN;
const int32_t ix0 = (int32_t) (ox * p->stride_x) - p->pad_x;
HVX_Vector acc = seed;
for (uint32_t ky = ky_lo; ky < ky_hi; ++ky) {
const float * row = src + (uint32_t) (iy0 + (int32_t) ky) * p->src_x;
if (p->stride_x == 1) {
for (uint32_t kx = 0; kx < p->kernel_x; ++kx) {
const HVX_Vector v = *(const HVX_UVector *) (row + ix0 + (int32_t) kx);
acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, v) : hvx_vec_add_f32_f32(acc, v);
}
} else {
for (uint32_t kx = 0; kx < p->kernel_x; kx += 2) {
const HVX_Vector v0 = *(const HVX_UVector *) (row + ix0 + (int32_t) kx);
const HVX_Vector v1 = *(const HVX_UVector *) (row + ix0 + (int32_t) kx + VLEN_FP32);
const HVX_VectorPair deinterleaved = Q6_W_vdeal_VVR(v1, v0, -4);
const HVX_Vector lo = Q6_V_lo_W(deinterleaved);
acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, lo) : hvx_vec_add_f32_f32(acc, lo);
if (kx + 1 < p->kernel_x) {
const HVX_Vector hi = Q6_V_hi_W(deinterleaved);
acc = is_max ? Q6_Vsf_vmax_VsfVsf(acc, hi) : hvx_vec_add_f32_f32(acc, hi);
}
}
}
}
hvx_vec_store_u(dst_row + ox, nbytes, is_max ? acc : hvx_vec_mul_f32_f32(acc, scale));
}
}
static void pool_plane_general(
const float * src, float * dst, const struct htp_pool_2d_kernel_params * p) {
const bool is_max = (p->pool_op == HTP_POOL_MAX);
for (uint32_t oy = 0; oy < p->dst_y; ++oy) {
int32_t iy0;
uint32_t ky_lo, ky_hi;
pool_row_bounds_y(p, oy, &iy0, &ky_lo, &ky_hi);
float * dst_row = dst + oy * p->dst_x;
pool_row_general_boundary_vec(src, dst_row, p, 0, p->ox_lo, iy0, ky_lo, ky_hi, is_max);
pool_row_general_vec(src, dst_row, p, p->ox_lo, p->ox_hi, iy0, ky_lo, ky_hi, is_max);
pool_row_general_boundary_vec(src, dst_row, p, p->ox_hi, p->dst_x, iy0, ky_lo, ky_hi, is_max);
}
}
typedef void (*pool_plane_fn_t)(const float * src, float * dst, const struct htp_pool_2d_kernel_params * p);
struct pool_2d_context {
struct htp_ops_context * octx;
const struct htp_pool_2d_kernel_params * kparams;
struct htp_pool_vtcm_layout layout;
pool_plane_fn_t pool_plane;
uint32_t n_threads;
uint32_t plane_start;
uint32_t plane_count;
uint32_t planes_per_thread;
};
static void pool_2d_thread(unsigned int nth, unsigned int ith, void * data) {
struct pool_2d_context * ctx = (struct pool_2d_context *) data;
const struct htp_pool_2d_kernel_params * p = ctx->kparams;
const struct htp_pool_vtcm_layout * L = &ctx->layout;
const struct htp_tensor * src0 = ctx->octx->src[0];
const struct htp_tensor * dst = ctx->octx->dst;
pool_plane_fn_t pool_plane = ctx->pool_plane;
const uint32_t planes_per_thread = ctx->planes_per_thread;
const uint32_t first = ctx->plane_start + ith * planes_per_thread;
const uint32_t last = MIN(first + planes_per_thread, ctx->plane_start + ctx->plane_count);
if (first >= last) {
return;
}
struct htp_thread_trace * tr = &ctx->octx->ctx->trace[ith];
dma_queue * dma_queue = ctx->octx->ctx->dma[ith];
const uint32_t src_plane_bytes = L->src_plane_bytes;
const uint32_t dst_plane_bytes = L->dst_plane_bytes;
const uint32_t src_spad_half = (uint32_t) L->src_spad_half_size;
const uint32_t dst_spad_half = (uint32_t) L->dst_spad_half_size;
uint8_t * vtcm_base = (uint8_t *) ctx->octx->ctx->vtcm_base;
uint8_t * src_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_src) + ith * L->src_bytes_per_thread;
uint8_t * dst_spad = VTCM_LAYOUT_PTR(uint8_t, vtcm_base, L->off_dst) + ith * L->dst_bytes_per_thread;
float * srcb2[2] = { (float *) src_spad, (float *) (src_spad + src_spad_half) };
float * dstb2[2] = { (float *) dst_spad, (float *) (dst_spad + dst_spad_half) };
const uint32_t total = last - first;
// Warm up the pipeline: push up to 2 initial (dummy dst, src) transfer pairs.
for (uint32_t i = 0; i < total && i < 2; ++i) {
dma_queue_push(dma_queue,
dma_make_data(dst->data, dstb2[i]),
dst_plane_bytes, dst_spad_half,
dst_plane_bytes, 0);
const dma_addr_t src_addr = src0->data + (first + i) * src_plane_bytes;
dma_queue_push(dma_queue,
dma_make_data(srcb2[i], src_addr),
src_spad_half, src_plane_bytes,
src_plane_bytes, 1);
}
for (uint32_t i = 0; i < total; ++i) {
const uint32_t plane = first + i;
const uint32_t buf = i & 1u;
float * srcb = srcb2[buf];
float * dstb = dstb2[buf];
dma_queue_pop(dma_queue); // dst writeback from plane i - 2 (or dummy on iter 0, 1)
dma_queue_pop(dma_queue); // input for plane i
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) plane);
pool_plane(srcb, dstb, p);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) plane);
const dma_addr_t dst_addr = dst->data + plane * dst_plane_bytes;
dma_queue_push(dma_queue,
dma_make_data(dst_addr, dstb),
dst_plane_bytes, dst_spad_half,
dst_plane_bytes, 1);
if (i + 2 < total) {
const dma_addr_t next_src_addr = src0->data + (plane + 2) * src_plane_bytes;
dma_queue_push(dma_queue,
dma_make_data(srcb, next_src_addr),
src_spad_half, src_plane_bytes,
src_plane_bytes, 1);
}
}
dma_queue_flush(dma_queue);
FARF(HIGH, "pool2d-f32-dma %d/%d: %ux%ux%ux%u -> %ux%ux%ux%u (%u:%u)\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], first, last);
(void) nth;
}
int op_pool_2d(struct htp_ops_context * octx) {
const struct htp_tensor * dst = octx->dst;
const struct htp_pool_2d_kernel_params * p =
(const struct htp_pool_2d_kernel_params *) octx->kernel_params;
uint32_t plane_start = 0;
uint32_t plane_count = p->planes;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
plane_count, htp_tensor_mdev_data_aligned(dst) ? p->planes_per_chunk : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
plane_start = range.start;
plane_count = range.count;
}
if (plane_count == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = MIN(p->n_threads, plane_count);
if (!htp_ops_context_set_n_threads(octx, n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t planes_per_thread = fastdiv(plane_count + n_threads - 1, &octx->n_threads_div);
pool_plane_fn_t pool_plane;
switch (p->kernel_type) {
case HTP_POOL_KERNEL_GLOBAL: pool_plane = pool_plane_global; break;
case HTP_POOL_KERNEL_BLOCK: pool_plane = pool_plane_block; break;
case HTP_POOL_KERNEL_HVX_NARROW: pool_plane = pool_plane_hvx_narrow; break;
case HTP_POOL_KERNEL_HVX: pool_plane = pool_plane_hvx; break;
default: pool_plane = pool_plane_general; break;
}
struct pool_2d_context ctx = {
.octx = octx,
.kparams = p,
.pool_plane = pool_plane,
.n_threads = n_threads,
.plane_start = plane_start,
.plane_count = plane_count,
.planes_per_thread = planes_per_thread,
};
htp_pool_vtcm_layout_build(&ctx.layout, p->src_x, p->src_y, p->dst_x, p->dst_y, n_threads);
work_queue_run(octx->ctx->work_queue, pool_2d_thread, &ctx, n_threads);
return HTP_STATUS_OK;
}
int op_pool_1d(struct htp_ops_context * octx) {
return op_pool_2d(octx);
}
llama.cpp/ggml/src/ggml-hexagon/htp/pool-ops.h
+95
View File
@@ -0,0 +1,95 @@
#ifndef HTP_POOL_OPS_H
#define HTP_POOL_OPS_H
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include "hex-common.h"
#include "htp-vtcm.h"
enum htp_pool_kernel_type {
HTP_POOL_KERNEL_GENERAL = 0,
HTP_POOL_KERNEL_HVX,
HTP_POOL_KERNEL_HVX_NARROW,
HTP_POOL_KERNEL_GLOBAL,
HTP_POOL_KERNEL_BLOCK,
};
struct htp_pool_2d_kernel_params {
uint8_t kernel_type;
uint32_t src_x;
uint32_t src_y;
uint32_t dst_x;
uint32_t dst_y;
uint32_t kernel_x;
uint32_t kernel_y;
uint32_t stride_x;
uint32_t stride_y;
int32_t pad_x;
int32_t pad_y;
uint32_t n_threads;
uint32_t planes;
uint32_t planes_per_chunk;
uint32_t pool_op;
uint32_t avg_divide_count;
uint32_t ox_lo;
uint32_t ox_hi;
float inv_kernel_area;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_pool_2d_kernel_params) <= 128, "htp_pool_2d_kernel_params is too large");
#else
_Static_assert(sizeof(struct htp_pool_2d_kernel_params) <= 128, "htp_pool_2d_kernel_params is too large");
#endif
struct htp_pool_vtcm_layout {
size_t total_bytes;
size_t off_src;
size_t off_dst;
uint32_t src_plane_bytes;
uint32_t dst_plane_bytes;
size_t src_bytes_per_thread;
size_t dst_bytes_per_thread;
size_t src_spad_half_size;
size_t dst_spad_half_size;
};
static inline void htp_pool_vtcm_layout_build(
struct htp_pool_vtcm_layout * L,
uint32_t src_x,
uint32_t src_y,
uint32_t dst_x,
uint32_t dst_y,
uint32_t n_threads
) {
// Full-plane double buffering (256 bytes guard space for vector loads)
L->src_plane_bytes = src_x * src_y * sizeof(float);
L->dst_plane_bytes = dst_x * dst_y * sizeof(float);
L->src_spad_half_size = hex_round_up(L->src_plane_bytes + 256, 128);
L->dst_spad_half_size = hex_round_up(L->dst_plane_bytes, 128);
L->src_bytes_per_thread = 2 * L->src_spad_half_size;
L->dst_bytes_per_thread = 2 * L->dst_spad_half_size;
size_t off = 0;
VTCM_LAYOUT_ALLOC(off, off_src, L->src_bytes_per_thread * n_threads);
VTCM_LAYOUT_ALLOC(off, off_dst, L->dst_bytes_per_thread * n_threads);
L->total_bytes = off;
}
static inline bool htp_pool_solve_layout(
struct htp_pool_vtcm_layout * layout,
uint32_t src_x,
uint32_t src_y,
uint32_t dst_x,
uint32_t dst_y,
uint32_t n_threads,
size_t vtcm_budget
) {
htp_pool_vtcm_layout_build(layout, src_x, src_y, dst_x, dst_y, n_threads);
return layout->total_bytes <= vtcm_budget;
}
#endif // HTP_POOL_OPS_H
llama.cpp/ggml/src/ggml-hexagon/htp/repeat-ops.c
+163
View File
@@ -0,0 +1,163 @@
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic ignored "-Wunused-function"
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <string.h>
#include "hvx-utils.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "htp-ctx.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "repeat-ops.h"
struct htp_repeat_context {
struct htp_ops_context * octx;
const struct htp_repeat_kernel_params * kparams;
uint32_t nrows_per_thread;
uint32_t total_dst_rows; // ne1 * ne2 * ne3
uint32_t row_start;
size_t type_size;
};
static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * data) {
const struct htp_repeat_context * rctx = (const struct htp_repeat_context *) data;
struct htp_ops_context * octx = rctx->octx;
const struct htp_repeat_kernel_params * kparams = rctx->kparams;
const struct htp_tensor * src = octx->src[0];
const struct htp_tensor * dst = octx->dst;
const uint32_t ne00 = src->ne[0];
const uint32_t ne01 = src->ne[1];
const uint32_t ne02 = src->ne[2];
const uint32_t ne03 = src->ne[3];
const uint32_t nb00 = src->nb[0];
const uint32_t nb01 = src->nb[1];
const uint32_t nb02 = src->nb[2];
const uint32_t nb03 = src->nb[3];
const uint32_t ne0 = dst->ne[0];
const uint32_t ne1 = dst->ne[1];
const uint32_t ne2 = dst->ne[2];
const uint32_t ne3 = dst->ne[3];
const uint32_t nb0 = dst->nb[0];
const uint32_t nb1 = dst->nb[1];
const uint32_t nb2 = dst->nb[2];
const uint32_t nb3 = dst->nb[3];
const uint32_t nr0 = kparams->nr0;
const size_t row_bytes = ne00 * rctx->type_size;
const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith;
const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows);
if (row_start >= row_end) {
return;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
// Decompose flat dst row index into (i1, i2, i3)
const uint32_t dst_row_div_ne1 = fastdiv(dst_row, &kparams->div_ne1);
const uint32_t i1 = dst_row - dst_row_div_ne1 * ne1;
const uint32_t i3 = fastdiv(dst_row, &kparams->div_ne12);
const uint32_t i2 = dst_row_div_ne1 - i3 * ne2;
// Map to source indices (tiling)
const uint32_t k1 = fastmodulo(i1, ne01, &kparams->div_ne01);
const uint32_t k2 = fastmodulo(i2, ne02, &kparams->div_ne02);
const uint32_t k3 = fastmodulo(i3, ne03, &kparams->div_ne03);
const uint8_t * src_row = (const uint8_t *) src->data + k1 * nb01 + k2 * nb02 + k3 * nb03;
uint8_t * dst_base = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
// Tile along dimension 0
for (uint32_t i0 = 0; i0 < nr0; i0++) {
uint8_t * dst_ptr = dst_base + i0 * ne00 * nb0;
memcpy(dst_ptr, src_row, row_bytes);
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end);
}
int op_repeat(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
size_t type_size;
switch (src0->type) {
case HTP_TYPE_F32: type_size = 4; break;
case HTP_TYPE_F16: type_size = 2; break;
default:
FARF(ERROR, "repeat: unsupported type %u\n", src0->type);
return HTP_STATUS_NO_SUPPORT;
}
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
const struct htp_repeat_kernel_params * kparams =
(const struct htp_repeat_kernel_params *) octx->kernel_params;
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
const size_t dst_row_size = dst->ne[0] * type_size;
uint32_t row_start = 0;
uint32_t nrows = total_dst_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
struct htp_repeat_context rctx = {
.octx = octx,
.kparams = kparams,
.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_dst_rows = nrows,
.row_start = row_start,
.type_size = type_size,
};
FARF(HIGH, "repeat: (%ux%ux%ux%u) -> (%ux%ux%ux%u) nr=(%u,%u,%u,%u)\n",
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
kparams->nr0, kparams->nr1, kparams->nr2, kparams->nr3);
work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads);
return HTP_STATUS_OK;
}
llama.cpp/ggml/src/ggml-hexagon/htp/repeat-ops.h
+27
View File
@@ -0,0 +1,27 @@
#ifndef HTP_REPEAT_OPS_H
#define HTP_REPEAT_OPS_H
#include "hex-fastdiv.h"
#include <stdint.h>
struct htp_repeat_kernel_params {
uint32_t n_threads;
uint32_t nr0;
uint32_t nr1;
uint32_t nr2;
uint32_t nr3;
struct fastdiv_values div_ne1;
struct fastdiv_values div_ne12;
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02;
struct fastdiv_values div_ne03;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_repeat_kernel_params) <= 128, "htp_repeat_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_REPEAT_OPS_H
Loaded 100 of 3918 files, more files were not shown because too many files have changed in this diff. Show more