ebd048fc5e
* opencl: rework FA kernel for f16 and f32
* opencl: flash-attention prefill prepass kernels
- flash_attn_kv_pad_f16 pads the tail KV tile to a BLOCK_N multiple
- flash_attn_mask_pad_f16 pads the matching mask tile
- flash_attn_blk_f16 classifies each KV tile per query block as
fully masked / mixed / fully unmasked, so
the main kernel can skip fully-masked tiles
and the mask lookup for fully-unmasked ones
* opencl: FA kernels for q4_0 and q8_0
* opencl: `set_rows` for f32 to q8_0/q4_0
* opencl: dequant kernels for q4_0 and q8_0
* opencl: add FA tile tuning table with override
* opencl: wire host side for FA
* opencl: q4_0 MoE tensors are also SOA'ed
* opencl: cosmetic fix
* opencl: refactor, also clarify some code paths in comments
* opencl: fix inifity for `-cl-finite-math-only`
---------
Co-authored-by: Li He <lih@qti.qualcomm.com>
210 lines
5.0 KiB
CMake
210 lines
5.0 KiB
CMake
find_package(OpenCL REQUIRED)
|
|
find_package(Python3 REQUIRED)
|
|
|
|
set(TARGET_NAME ggml-opencl)
|
|
|
|
ggml_add_backend_library(${TARGET_NAME}
|
|
ggml-opencl.cpp
|
|
../../include/ggml-opencl.h)
|
|
target_link_libraries(${TARGET_NAME} PRIVATE ${OpenCL_LIBRARIES})
|
|
target_include_directories(${TARGET_NAME} PRIVATE ${OpenCL_INCLUDE_DIRS})
|
|
|
|
if (GGML_OPENCL_PROFILING)
|
|
message(STATUS "OpenCL profiling enabled (increases CPU overhead)")
|
|
add_compile_definitions(GGML_OPENCL_PROFILING)
|
|
endif ()
|
|
|
|
add_compile_definitions(GGML_OPENCL_SOA_Q)
|
|
add_compile_definitions(GGML_OPENCL_TARGET_VERSION=${GGML_OPENCL_TARGET_VERSION})
|
|
|
|
if (GGML_OPENCL_USE_ADRENO_KERNELS)
|
|
message(STATUS "OpenCL will use matmul kernels optimized for Adreno")
|
|
add_compile_definitions(GGML_OPENCL_USE_ADRENO_KERNELS)
|
|
endif ()
|
|
|
|
if (GGML_OPENCL_EMBED_KERNELS)
|
|
add_compile_definitions(GGML_OPENCL_EMBED_KERNELS)
|
|
|
|
set(EMBED_KERNEL_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/kernels/embed_kernel.py")
|
|
file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/autogenerated")
|
|
|
|
target_include_directories(${TARGET_NAME} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/autogenerated")
|
|
endif ()
|
|
|
|
function(ggml_opencl_add_kernel KNAME)
|
|
set(KERN_HDR ${CMAKE_CURRENT_BINARY_DIR}/autogenerated/${KNAME}.cl.h)
|
|
set(KERN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/kernels/${KNAME}.cl)
|
|
|
|
if (GGML_OPENCL_EMBED_KERNELS)
|
|
message(STATUS "opencl: embedding kernel ${KNAME}")
|
|
|
|
# Python must be accessible from command line
|
|
add_custom_command(
|
|
OUTPUT ${KERN_HDR}
|
|
COMMAND ${Python3_EXECUTABLE} ${EMBED_KERNEL_SCRIPT} ${KERN_SRC} ${KERN_HDR}
|
|
DEPENDS ${KERN_SRC} ${EMBED_KERNEL_SCRIPT}
|
|
COMMENT "Generate ${KERN_HDR}"
|
|
)
|
|
|
|
target_sources(${TARGET_NAME} PRIVATE ${KERN_HDR})
|
|
else ()
|
|
message(STATUS "opencl: adding kernel ${KNAME}")
|
|
configure_file(${KERN_SRC} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${KNAME}.cl COPYONLY)
|
|
endif ()
|
|
endfunction()
|
|
|
|
set(GGML_OPENCL_KERNELS
|
|
add
|
|
add_id
|
|
argsort
|
|
tri
|
|
fill
|
|
clamp
|
|
cpy
|
|
cvt
|
|
diag_mask_inf
|
|
diag
|
|
div
|
|
gelu
|
|
get_rows
|
|
glu
|
|
group_norm
|
|
solve_tri
|
|
im2col_f32
|
|
im2col_f16
|
|
mean
|
|
mul_mv_f16_f16
|
|
mul_mv_f16_f32_1row
|
|
mul_mv_f16_f32_l4
|
|
mul_mv_f16_f32
|
|
mul_mv_f32_f32
|
|
mul_mv_q4_0_f32
|
|
mul_mv_q4_0_f32_v
|
|
mul_mv_q4_0_f32_8x_flat
|
|
mul_mv_q4_0_f32_1d_8x_flat
|
|
mul_mv_q4_0_f32_1d_16x_flat
|
|
mul_mv_q4_1_f32
|
|
mul_mv_q4_1_f32_flat
|
|
mul_mv_q4_k_f32
|
|
mul_mv_q4_k_f32_flat
|
|
mul_mv_q5_0_f32
|
|
mul_mv_q5_0_f32_flat
|
|
mul_mv_q5_1_f32
|
|
mul_mv_q5_1_f32_flat
|
|
mul_mv_q5_k_f32
|
|
mul_mv_q5_k_f32_flat
|
|
mul_mv_q6_k_f32
|
|
mul_mv_q6_k_f32_flat
|
|
mul_mv_q8_0_f32
|
|
mul_mv_q8_0_f32_flat
|
|
mul_mv_iq4_nl_f32
|
|
mul_mv_iq4_nl_f32_flat
|
|
mul_mv_mxfp4_f32
|
|
mul_mv_mxfp4_f32_flat
|
|
mul_mv_id_q4_0_f32_8x_flat
|
|
mul_mv_id_q8_0_f32
|
|
mul_mv_id_q8_0_f32_flat
|
|
mul_mv_id_mxfp4_f32
|
|
mul_mv_id_mxfp4_f32_flat
|
|
gemm_moe_q4_0_f32_ns
|
|
gemv_moe_q4_0_f32_ns
|
|
gemm_moe_q4_1_f32_ns
|
|
gemv_moe_q4_1_f32_ns
|
|
gemm_moe_q5_0_f32_ns
|
|
gemv_moe_q5_0_f32_ns
|
|
gemm_moe_q5_1_f32_ns
|
|
gemv_moe_q5_1_f32_ns
|
|
gemm_moe_q4_k_f32_ns
|
|
gemv_moe_q4_k_f32_ns
|
|
gemm_moe_q5_k_f32_ns
|
|
gemv_moe_q5_k_f32_ns
|
|
gemm_moe_q6_k_f32_ns
|
|
gemv_moe_q6_k_f32_ns
|
|
gemm_moe_mxfp4_f32
|
|
gemv_moe_mxfp4_f32
|
|
gemm_moe_mxfp4_f32_ns
|
|
gemv_moe_mxfp4_f32_ns
|
|
moe_reorder_b
|
|
moe_sort_by_expert
|
|
mul_mm_f32_f32_l4_lm
|
|
mul_mm_f16_f32_l4_lm
|
|
mul_mm_q4_0_f32_l4_lm
|
|
mul_mm_q4_1_f32_l4_lm
|
|
mul_mm_q5_0_f32_l4_lm
|
|
mul_mm_q5_1_f32_l4_lm
|
|
mul_mm_q8_0_f32_l4_lm
|
|
mul_mm_iq4_nl_f32_l4_lm
|
|
mul_mm_q4_k_f32_l4_lm
|
|
mul_mm_q5_k_f32_l4_lm
|
|
mul_mm_q6_k_f32_l4_lm
|
|
gemv_noshuffle_q4_0_f32
|
|
gemv_noshuffle_q4_0_f32_spec
|
|
gemm_noshuffle_q4_0_f32
|
|
gemv_noshuffle_q4_1_f32
|
|
gemm_noshuffle_q4_1_f32
|
|
gemv_noshuffle_q5_0_f32
|
|
gemm_noshuffle_q5_0_f32
|
|
gemv_noshuffle_q5_1_f32
|
|
gemm_noshuffle_q5_1_f32
|
|
gemv_noshuffle_iq4_nl_f32
|
|
gemm_noshuffle_iq4_nl_f32
|
|
gemv_noshuffle_q8_0_f32
|
|
gemm_noshuffle_q8_0_f32
|
|
gemv_noshuffle_q4_k_f32
|
|
gemm_noshuffle_q4_k_f32
|
|
gemv_noshuffle_q6_k_f32
|
|
gemm_noshuffle_q6_k_f32
|
|
gemv_noshuffle_q5_k_f32
|
|
gemm_noshuffle_q5_k_f32
|
|
mul
|
|
neg
|
|
norm
|
|
relu
|
|
l2_norm
|
|
rms_norm
|
|
rope
|
|
scale
|
|
set_rows
|
|
sigmoid
|
|
silu
|
|
softmax_4_f32
|
|
softmax_4_f16
|
|
softmax_f32
|
|
softmax_f16
|
|
sqr
|
|
sqrt
|
|
ssm_conv
|
|
gated_delta_net
|
|
sub
|
|
sum_rows
|
|
cumsum
|
|
transpose
|
|
concat
|
|
tsembd
|
|
upscale
|
|
tanh
|
|
exp
|
|
expm1
|
|
softplus
|
|
pad
|
|
repeat
|
|
mul_mat_f16_f32
|
|
mul_mm_f16_f32_kq_kqv
|
|
conv2d
|
|
conv2d_f16_f32
|
|
flash_attn_pre_f16
|
|
flash_attn_f32_f16
|
|
flash_attn_f32_q8_0
|
|
flash_attn_f32_q4_0
|
|
flash_attn_f16
|
|
flash_attn_f32
|
|
)
|
|
|
|
if (GGML_OPENCL_USE_ADRENO_KERNELS)
|
|
list(APPEND GGML_OPENCL_KERNELS gemm_xmem_f16_f32_os8)
|
|
endif ()
|
|
|
|
foreach (K ${GGML_OPENCL_KERNELS})
|
|
ggml_opencl_add_kernel(${K})
|
|
endforeach()
|