7296c961d9
* Allow use of OpenCL GPU-based BLAS using ClBlast instead of OpenBLAS for context processing * Improve ClBlast implementation, avoid recreating buffers, remove redundant transfers * Finish merge of ClBlast support * Move CLBlast implementation to separate file Add buffer reuse code (adapted from slaren's cuda implementation) * Add q4_2 and q4_3 CLBlast support, improve code * Double CLBlast speed by disabling OpenBLAS thread workaround Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com> Co-authored-by: slaren <2141330+slaren@users.noreply.github.com> * Fix device selection env variable names * Fix cast in opencl kernels * Add CLBlast to CMakeLists.txt * Replace buffer pool with static buffers a, b, qb, c Fix compile warnings * Fix typos, use GGML_TYPE defines, improve code * Improve btype dequant kernel selection code, add error if type is unsupported * Improve code quality * Move internal stuff out of header * Use internal enums instead of CLBlast enums * Remove leftover C++ includes and defines * Make event use easier to read Co-authored-by: Henri Vasserman <henv@hot.ee> * Use c compiler for opencl files * Simplify code, fix include * First check error, then release event * Make globals static, fix indentation * Rename dequant kernels file to conform with other file names * Fix import cl file name --------- Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com> Co-authored-by: slaren <2141330+slaren@users.noreply.github.com> Co-authored-by: Henri Vasserman <henv@hot.ee> Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
85 lines
2 KiB
Common Lisp
85 lines
2 KiB
Common Lisp
#define MULTILINE_QUOTE(...) #__VA_ARGS__
|
|
const char * clblast_dequant = MULTILINE_QUOTE(
|
|
|
|
struct block_q4_0
|
|
{
|
|
float d;
|
|
uchar qs[16];
|
|
};
|
|
|
|
__kernel void dequantize_row_q4_0(__global struct block_q4_0* blocks, __global float* result) {
|
|
const uint i = get_global_id(0) / 32;
|
|
const uint l = get_local_id(0);
|
|
|
|
const float d = blocks[i].d;
|
|
|
|
const uchar vi = blocks[i].qs[l];
|
|
|
|
const uint index = i*32 + l*2;
|
|
result[index + 0] = ((vi & 0xf) - 8)*d;
|
|
result[index + 1] = ((vi >> 4) - 8)*d;
|
|
}
|
|
|
|
struct block_q4_1
|
|
{
|
|
float d;
|
|
float m;
|
|
uchar qs[16];
|
|
};
|
|
|
|
__kernel void dequantize_row_q4_1(__global struct block_q4_1* blocks, __global float* result) {
|
|
const uint i = get_global_id(0) / 32;
|
|
const uint l = get_local_id(0);
|
|
|
|
const float d = blocks[i].d;
|
|
const float m = blocks[i].m;
|
|
|
|
const uchar vi = blocks[i].qs[l];
|
|
|
|
const uint index = i*32 + l*2;
|
|
result[index + 0] = (vi & 0xf) * d + m;
|
|
result[index + 1] = (vi >> 4) * d + m;
|
|
}
|
|
|
|
struct block_q4_2
|
|
{
|
|
ushort d;
|
|
uchar qs[8];
|
|
};
|
|
|
|
__kernel void dequantize_row_q4_2(__global struct block_q4_2* blocks, __global float* result) {
|
|
const uint i = get_global_id(0) / 16;
|
|
const uint l = get_local_id(0);
|
|
|
|
const float d = vload_half(0, (__global half*) &blocks[i].d);;
|
|
|
|
const uchar vi = blocks[i].qs[l];
|
|
|
|
const uint index = i*16 + l*2;
|
|
result[index + 0] = ((vi & 0xf) - 8)*d;
|
|
result[index + 1] = ((vi >> 4) - 8)*d;
|
|
}
|
|
|
|
struct block_q4_3
|
|
{
|
|
ushort d;
|
|
ushort m;
|
|
uchar qs[8];
|
|
};
|
|
|
|
__kernel void dequantize_row_q4_3(__global struct block_q4_3* blocks, __global float* result) {
|
|
const uint i = get_global_id(0) / 16;
|
|
const uint l = get_local_id(0);
|
|
|
|
const float d = vload_half(0, (__global half*) &(blocks[i].d));
|
|
const float m = vload_half(0, (__global half*) &(blocks[i].m));
|
|
|
|
const uchar vi = blocks[i].qs[l];
|
|
|
|
const uint index = i*16 + l*2;
|
|
result[index + 0] = (vi & 0xf) * d + m;
|
|
result[index + 1] = (vi >> 4) * d + m;
|
|
}
|
|
|
|
);
|