* llama : ggml-backend integration * ggml-backend : add names to buffers * fix unmap after loading * batched-bench : add tensor_split param * llama : check for null tensor_split * ggml-backend : increase GGML_MAX_BACKENDS * improve graph splitting, partial fix for --no-kv-offload * cuda : add ggml-backend split buffer support * cuda : do not create buffer types for devices that don't exist (fixes usage without CUDA devices available) * ggml : fix null backend dereference (#4807) * ggml : fix null backend dereference * ggml : also check ggml_backend_is_cpu * test-backend-ops : check buffer allocation failures * llama : add cparam (split_mode) and command line argument (--split-mode, -sm) to configure the split mode (none, layer or row) * ggml : fix mul_mat_id work size * llama : rewrite session kv load/set without graphs * minor * llama : only initialize used backends, free backends on context free * llama : abort ctx if cuda backend init fails * llama : rewrite lora with ggml-backend and compute on CPU ggml-ci * llama : only map to a backend buffer the region of the file mapping containing the tensors used in the buffer * opencl : add ggml-backend buffer type * cuda : only use batched_cublas with batched mat muls (fixes fp16 tg perf) * llama : on Metal, by default offload the full model ggml-ci * metal : page align the data ptr (#4854) * Apply suggestions from code review Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * cuda : fix split buffer free * address review comments * llama-bench : add split-mode parameter * fix whitespace * opencl : fix double initialization * server : add --split-mode parameter * use async copy and compute to improve multi-gpu performance ggml-ci * use async memcpys to copy the graph outputs to the CPU * fix opencl * use a host buffer for the cpu compute buffer for faster copies to the gpu --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
95 lines
4 KiB
C
95 lines
4 KiB
C
#pragma once
|
|
|
|
#include "ggml.h"
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
struct ggml_backend;
|
|
struct ggml_backend_buffer;
|
|
struct ggml_backend_buffer_type;
|
|
|
|
//
|
|
// Legacy API
|
|
//
|
|
|
|
typedef struct ggml_allocr * ggml_allocr_t;
|
|
|
|
// initialize allocator for use with CPU backend only
|
|
GGML_API ggml_allocr_t ggml_allocr_new(void * data, size_t size, size_t alignment);
|
|
GGML_API ggml_allocr_t ggml_allocr_new_measure(size_t alignment);
|
|
|
|
// initialize allocator for use with ggml-backend
|
|
GGML_API ggml_allocr_t ggml_allocr_new_from_buffer(struct ggml_backend_buffer * buffer);
|
|
GGML_API ggml_allocr_t ggml_allocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer
|
|
GGML_API ggml_allocr_t ggml_allocr_new_measure_from_backend(struct ggml_backend * backend);
|
|
|
|
GGML_API struct ggml_backend_buffer * ggml_allocr_get_buffer(ggml_allocr_t alloc);
|
|
|
|
// tell the allocator to parse nodes following the order described in the list
|
|
// you should call this if your graph are optimized to execute out-of-order
|
|
GGML_API void ggml_allocr_set_parse_seq(ggml_allocr_t alloc, const int * list, int n);
|
|
|
|
GGML_API void ggml_allocr_free (ggml_allocr_t alloc);
|
|
GGML_API bool ggml_allocr_is_measure (ggml_allocr_t alloc);
|
|
GGML_API void ggml_allocr_reset (ggml_allocr_t alloc);
|
|
GGML_API void ggml_allocr_alloc (ggml_allocr_t alloc, struct ggml_tensor * tensor);
|
|
GGML_API size_t ggml_allocr_max_size (ggml_allocr_t alloc);
|
|
|
|
GGML_API size_t ggml_allocr_alloc_graph(ggml_allocr_t alloc, struct ggml_cgraph * graph);
|
|
|
|
//
|
|
// ggml-backend v2 API
|
|
//
|
|
|
|
// Separate tensor and graph allocator objects
|
|
// This is necessary for multi-backend allocation because the graph allocator needs to use multiple tensor allocators
|
|
// The original API is kept as a wrapper around the new API
|
|
|
|
// Tensor allocator
|
|
typedef struct ggml_tallocr * ggml_tallocr_t;
|
|
|
|
GGML_API ggml_tallocr_t ggml_tallocr_new(void * data, size_t size, size_t alignment);
|
|
GGML_API ggml_tallocr_t ggml_tallocr_new_measure(size_t alignment);
|
|
GGML_API ggml_tallocr_t ggml_tallocr_new_from_buft(struct ggml_backend_buffer_type * buft, size_t size);
|
|
GGML_API ggml_tallocr_t ggml_tallocr_new_from_backend(struct ggml_backend * backend, size_t size); // allocates an owned buffer
|
|
GGML_API ggml_tallocr_t ggml_tallocr_new_from_buffer(struct ggml_backend_buffer * buffer);
|
|
GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_buft(struct ggml_backend_buffer_type * buft);
|
|
GGML_API ggml_tallocr_t ggml_tallocr_new_measure_from_backend(struct ggml_backend * backend);
|
|
|
|
GGML_API struct ggml_backend_buffer * ggml_tallocr_get_buffer(ggml_tallocr_t talloc);
|
|
|
|
GGML_API void ggml_tallocr_free (ggml_tallocr_t talloc);
|
|
GGML_API bool ggml_tallocr_is_measure (ggml_tallocr_t talloc);
|
|
GGML_API void ggml_tallocr_reset (ggml_tallocr_t talloc);
|
|
GGML_API void ggml_tallocr_alloc (ggml_tallocr_t talloc, struct ggml_tensor * tensor);
|
|
GGML_API size_t ggml_tallocr_max_size (ggml_tallocr_t talloc);
|
|
|
|
|
|
// Graph allocator
|
|
typedef struct ggml_gallocr * ggml_gallocr_t;
|
|
|
|
GGML_API ggml_gallocr_t ggml_gallocr_new(void);
|
|
GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);
|
|
|
|
GGML_API void ggml_gallocr_set_parse_seq(ggml_gallocr_t galloc, const int * list, int n);
|
|
GGML_API size_t ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, ggml_tallocr_t talloc, struct ggml_cgraph * graph);
|
|
|
|
// Allocate tensors from the allocators given by the hash table
|
|
GGML_API void ggml_gallocr_alloc_graph_n(
|
|
ggml_gallocr_t galloc,
|
|
struct ggml_cgraph * graph,
|
|
struct ggml_hash_set hash_set,
|
|
ggml_tallocr_t * hash_node_talloc);
|
|
|
|
|
|
// Utils
|
|
// Create a buffer and allocate all the tensors in a ggml_context
|
|
GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, struct ggml_backend_buffer_type * buft);
|
|
GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, struct ggml_backend * backend);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|