2023-04-10 21:59:13 +02:00
|
|
|
// Internal header to be included only by llama.cpp.
|
|
|
|
// Contains wrappers around OS interfaces.
|
|
|
|
|
|
|
|
#ifndef LLAMA_UTIL_H
|
|
|
|
#define LLAMA_UTIL_H
|
|
|
|
|
|
|
|
#include <cstdio>
|
|
|
|
#include <cstdint>
|
|
|
|
#include <cerrno>
|
|
|
|
#include <cstring>
|
|
|
|
#include <cstdarg>
|
|
|
|
#include <cstdlib>
|
|
|
|
#include <climits>
|
|
|
|
|
|
|
|
#include <string>
|
|
|
|
#include <vector>
|
2023-05-23 13:04:39 +02:00
|
|
|
#include <stdexcept>
|
2023-04-10 21:59:13 +02:00
|
|
|
|
|
|
|
#ifdef __has_include
|
|
|
|
#if __has_include(<unistd.h>)
|
|
|
|
#include <unistd.h>
|
|
|
|
#if defined(_POSIX_MAPPED_FILES)
|
|
|
|
#include <sys/mman.h>
|
|
|
|
#endif
|
2023-04-30 17:51:57 +02:00
|
|
|
#if defined(_POSIX_MEMLOCK_RANGE)
|
|
|
|
#include <sys/resource.h>
|
|
|
|
#endif
|
2023-04-10 21:59:13 +02:00
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if defined(_WIN32)
|
|
|
|
#define WIN32_LEAN_AND_MEAN
|
2023-04-30 17:51:57 +02:00
|
|
|
#ifndef NOMINMAX
|
|
|
|
#define NOMINMAX
|
|
|
|
#endif
|
2023-04-10 21:59:13 +02:00
|
|
|
#include <windows.h>
|
|
|
|
#include <io.h>
|
|
|
|
#include <stdio.h> // for _fseeki64
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define LLAMA_ASSERT(x) \
|
|
|
|
do { \
|
|
|
|
if (!(x)) { \
|
|
|
|
fprintf(stderr, "LLAMA_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
|
|
|
|
abort(); \
|
|
|
|
} \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
#ifdef __GNUC__
|
2023-04-30 17:51:57 +02:00
|
|
|
#ifdef __MINGW32__
|
|
|
|
__attribute__((format(gnu_printf, 1, 2)))
|
|
|
|
#else
|
2023-04-10 21:59:13 +02:00
|
|
|
__attribute__((format(printf, 1, 2)))
|
|
|
|
#endif
|
2023-04-30 17:51:57 +02:00
|
|
|
#endif
|
2023-04-10 21:59:13 +02:00
|
|
|
static std::string format(const char * fmt, ...) {
|
|
|
|
va_list ap, ap2;
|
|
|
|
va_start(ap, fmt);
|
|
|
|
va_copy(ap2, ap);
|
|
|
|
int size = vsnprintf(NULL, 0, fmt, ap);
|
|
|
|
LLAMA_ASSERT(size >= 0 && size < INT_MAX);
|
|
|
|
std::vector<char> buf(size + 1);
|
|
|
|
int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
|
|
|
|
LLAMA_ASSERT(size2 == size);
|
|
|
|
va_end(ap2);
|
|
|
|
va_end(ap);
|
|
|
|
return std::string(buf.data(), size);
|
2023-04-30 17:51:57 +02:00
|
|
|
}
|
2023-04-10 21:59:13 +02:00
|
|
|
|
|
|
|
struct llama_file {
|
|
|
|
// use FILE * so we don't have to re-open the file to mmap
|
|
|
|
FILE * fp;
|
|
|
|
size_t size;
|
|
|
|
|
|
|
|
llama_file(const char * fname, const char * mode) {
|
|
|
|
fp = std::fopen(fname, mode);
|
|
|
|
if (fp == NULL) {
|
2023-05-23 13:04:39 +02:00
|
|
|
throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
seek(0, SEEK_END);
|
|
|
|
size = tell();
|
|
|
|
seek(0, SEEK_SET);
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t tell() const {
|
|
|
|
#ifdef _WIN32
|
|
|
|
__int64 ret = _ftelli64(fp);
|
|
|
|
#else
|
|
|
|
long ret = std::ftell(fp);
|
|
|
|
#endif
|
|
|
|
LLAMA_ASSERT(ret != -1); // this really shouldn't fail
|
|
|
|
return (size_t) ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
void seek(size_t offset, int whence) {
|
|
|
|
#ifdef _WIN32
|
|
|
|
int ret = _fseeki64(fp, (__int64) offset, whence);
|
|
|
|
#else
|
|
|
|
int ret = std::fseek(fp, (long) offset, whence);
|
|
|
|
#endif
|
|
|
|
LLAMA_ASSERT(ret == 0); // same
|
|
|
|
}
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
void read_raw(void * ptr, size_t len) const {
|
|
|
|
if (len == 0) {
|
2023-04-10 21:59:13 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
errno = 0;
|
2023-05-23 13:04:39 +02:00
|
|
|
std::size_t ret = std::fread(ptr, len, 1, fp);
|
2023-04-10 21:59:13 +02:00
|
|
|
if (ferror(fp)) {
|
2023-05-23 13:04:39 +02:00
|
|
|
throw std::runtime_error(format("read error: %s", strerror(errno)));
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
if (ret != 1) {
|
2023-05-23 13:04:39 +02:00
|
|
|
throw std::runtime_error(std::string("unexpectedly reached end of file"));
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::uint32_t read_u32() {
|
|
|
|
std::uint32_t ret;
|
|
|
|
read_raw(&ret, sizeof(ret));
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string read_string(std::uint32_t len) {
|
|
|
|
std::vector<char> chars(len);
|
|
|
|
read_raw(chars.data(), len);
|
|
|
|
return std::string(chars.data(), len);
|
|
|
|
}
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
void write_raw(const void * ptr, size_t len) const {
|
|
|
|
if (len == 0) {
|
2023-04-10 21:59:13 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
errno = 0;
|
2023-05-23 13:04:39 +02:00
|
|
|
size_t ret = std::fwrite(ptr, len, 1, fp);
|
2023-04-10 21:59:13 +02:00
|
|
|
if (ret != 1) {
|
2023-05-23 13:04:39 +02:00
|
|
|
throw std::runtime_error(format("write error: %s", strerror(errno)));
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void write_u32(std::uint32_t val) {
|
|
|
|
write_raw(&val, sizeof(val));
|
|
|
|
}
|
|
|
|
|
|
|
|
~llama_file() {
|
|
|
|
if (fp) {
|
|
|
|
std::fclose(fp);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
#if defined(_WIN32)
|
|
|
|
static std::string llama_format_win_err(DWORD err) {
|
|
|
|
LPSTR buf;
|
|
|
|
size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
|
|
|
|
NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);
|
|
|
|
if (!size) {
|
|
|
|
return "FormatMessageA failed";
|
|
|
|
}
|
|
|
|
std::string ret(buf, size);
|
|
|
|
LocalFree(buf);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
struct llama_mmap {
|
|
|
|
void * addr;
|
|
|
|
size_t size;
|
|
|
|
|
|
|
|
llama_mmap(const llama_mmap &) = delete;
|
|
|
|
|
|
|
|
#ifdef _POSIX_MAPPED_FILES
|
|
|
|
static constexpr bool SUPPORTED = true;
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */) {
|
2023-04-10 21:59:13 +02:00
|
|
|
size = file->size;
|
|
|
|
int fd = fileno(file->fp);
|
|
|
|
int flags = MAP_SHARED;
|
|
|
|
#ifdef __linux__
|
|
|
|
flags |= MAP_POPULATE;
|
|
|
|
#endif
|
|
|
|
addr = mmap(NULL, file->size, PROT_READ, flags, fd, 0);
|
|
|
|
if (addr == MAP_FAILED) {
|
2023-05-23 13:04:39 +02:00
|
|
|
throw std::runtime_error(format("mmap failed: %s", strerror(errno)));
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
if (prefetch > 0) {
|
2023-04-30 17:51:57 +02:00
|
|
|
// Advise the kernel to preload the mapped memory
|
2023-05-23 13:04:39 +02:00
|
|
|
if (madvise(addr, std::min(file->size, prefetch), MADV_WILLNEED)) {
|
2023-04-30 17:51:57 +02:00
|
|
|
fprintf(stderr, "warning: madvise(.., MADV_WILLNEED) failed: %s\n",
|
|
|
|
strerror(errno));
|
|
|
|
}
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
~llama_mmap() {
|
|
|
|
munmap(addr, size);
|
|
|
|
}
|
|
|
|
#elif defined(_WIN32)
|
|
|
|
static constexpr bool SUPPORTED = true;
|
|
|
|
|
2023-04-30 17:51:57 +02:00
|
|
|
llama_mmap(struct llama_file * file, bool prefetch = true) {
|
2023-04-10 21:59:13 +02:00
|
|
|
size = file->size;
|
|
|
|
|
|
|
|
HANDLE hFile = (HANDLE) _get_osfhandle(_fileno(file->fp));
|
|
|
|
|
|
|
|
HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
|
|
|
|
DWORD error = GetLastError();
|
|
|
|
|
|
|
|
if (hMapping == NULL) {
|
2023-05-23 13:04:39 +02:00
|
|
|
throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);
|
|
|
|
error = GetLastError();
|
|
|
|
CloseHandle(hMapping);
|
|
|
|
|
|
|
|
if (addr == NULL) {
|
2023-05-23 13:04:39 +02:00
|
|
|
throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
|
2023-04-30 17:51:57 +02:00
|
|
|
#if _WIN32_WINNT >= _WIN32_WINNT_WIN8
|
|
|
|
if (prefetch) {
|
|
|
|
// Advise the kernel to preload the mapped memory
|
|
|
|
WIN32_MEMORY_RANGE_ENTRY range;
|
|
|
|
range.VirtualAddress = addr;
|
|
|
|
range.NumberOfBytes = (SIZE_T)size;
|
|
|
|
if (!PrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {
|
|
|
|
fprintf(stderr, "warning: PrefetchVirtualMemory failed: %s\n",
|
|
|
|
llama_format_win_err(GetLastError()).c_str());
|
|
|
|
}
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
2023-04-30 17:51:57 +02:00
|
|
|
#else
|
|
|
|
#pragma message("warning: You are building for pre-Windows 8; prefetch not supported")
|
|
|
|
#endif // _WIN32_WINNT >= _WIN32_WINNT_WIN8
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
~llama_mmap() {
|
|
|
|
if (!UnmapViewOfFile(addr)) {
|
|
|
|
fprintf(stderr, "warning: UnmapViewOfFile failed: %s\n",
|
|
|
|
llama_format_win_err(GetLastError()).c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static constexpr bool SUPPORTED = false;
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
llama_mmap(struct llama_file *, bool prefetch = true) {
|
|
|
|
(void)prefetch;
|
|
|
|
throw std::runtime_error(std::string("mmap not supported"));
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
|
|
|
// Represents some region of memory being locked using mlock or VirtualLock;
|
|
|
|
// will automatically unlock on destruction.
|
|
|
|
struct llama_mlock {
|
|
|
|
void * addr = NULL;
|
|
|
|
size_t size = 0;
|
|
|
|
bool failed_already = false;
|
|
|
|
|
|
|
|
llama_mlock() {}
|
|
|
|
llama_mlock(const llama_mlock &) = delete;
|
|
|
|
|
|
|
|
~llama_mlock() {
|
|
|
|
if (size) {
|
|
|
|
raw_unlock(addr, size);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
void init(void * ptr) {
|
|
|
|
LLAMA_ASSERT(addr == NULL && size == 0);
|
|
|
|
addr = ptr;
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void grow_to(size_t target_size) {
|
|
|
|
LLAMA_ASSERT(addr);
|
|
|
|
if (failed_already) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
size_t granularity = lock_granularity();
|
|
|
|
target_size = (target_size + granularity - 1) & ~(granularity - 1);
|
|
|
|
if (target_size > size) {
|
|
|
|
if (raw_lock((uint8_t *) addr + size, target_size - size)) {
|
|
|
|
size = target_size;
|
|
|
|
} else {
|
|
|
|
failed_already = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef _POSIX_MEMLOCK_RANGE
|
|
|
|
static constexpr bool SUPPORTED = true;
|
|
|
|
|
|
|
|
size_t lock_granularity() {
|
|
|
|
return (size_t) sysconf(_SC_PAGESIZE);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef __APPLE__
|
|
|
|
#define MLOCK_SUGGESTION \
|
|
|
|
"Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \
|
|
|
|
"decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MLOCK (ulimit -l).\n"
|
|
|
|
#else
|
|
|
|
#define MLOCK_SUGGESTION \
|
|
|
|
"Try increasing RLIMIT_MLOCK ('ulimit -l' as root).\n"
|
|
|
|
#endif
|
|
|
|
|
|
|
|
bool raw_lock(const void * addr, size_t size) {
|
|
|
|
if (!mlock(addr, size)) {
|
|
|
|
return true;
|
|
|
|
} else {
|
2023-04-30 17:51:57 +02:00
|
|
|
char* errmsg = std::strerror(errno);
|
|
|
|
bool suggest = (errno == ENOMEM);
|
|
|
|
|
|
|
|
// Check if the resource limit is fine after all
|
|
|
|
struct rlimit lock_limit;
|
|
|
|
if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit))
|
|
|
|
suggest = false;
|
|
|
|
if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size))
|
|
|
|
suggest = false;
|
|
|
|
|
|
|
|
fprintf(stderr, "warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",
|
|
|
|
size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");
|
2023-04-10 21:59:13 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#undef MLOCK_SUGGESTION
|
|
|
|
|
|
|
|
void raw_unlock(void * addr, size_t size) {
|
|
|
|
if (munlock(addr, size)) {
|
|
|
|
fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#elif defined(_WIN32)
|
|
|
|
static constexpr bool SUPPORTED = true;
|
|
|
|
|
|
|
|
size_t lock_granularity() {
|
|
|
|
SYSTEM_INFO si;
|
|
|
|
GetSystemInfo(&si);
|
|
|
|
return (size_t) si.dwPageSize;
|
|
|
|
}
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
bool raw_lock(void * ptr, size_t len) {
|
2023-04-10 21:59:13 +02:00
|
|
|
for (int tries = 1; ; tries++) {
|
2023-05-23 13:04:39 +02:00
|
|
|
if (VirtualLock(ptr, len)) {
|
2023-04-10 21:59:13 +02:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
if (tries == 2) {
|
|
|
|
fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",
|
2023-05-23 13:04:39 +02:00
|
|
|
len, size, llama_format_win_err(GetLastError()).c_str());
|
2023-04-10 21:59:13 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// It failed but this was only the first try; increase the working
|
|
|
|
// set size and try again.
|
|
|
|
SIZE_T min_ws_size, max_ws_size;
|
|
|
|
if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {
|
|
|
|
fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n",
|
|
|
|
llama_format_win_err(GetLastError()).c_str());
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
// Per MSDN: "The maximum number of pages that a process can lock
|
|
|
|
// is equal to the number of pages in its minimum working set minus
|
|
|
|
// a small overhead."
|
|
|
|
// Hopefully a megabyte is enough overhead:
|
2023-05-23 13:04:39 +02:00
|
|
|
size_t increment = len + 1048576;
|
2023-04-10 21:59:13 +02:00
|
|
|
// The minimum must be <= the maximum, so we need to increase both:
|
2023-04-30 17:51:57 +02:00
|
|
|
min_ws_size += increment;
|
|
|
|
max_ws_size += increment;
|
2023-04-10 21:59:13 +02:00
|
|
|
if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {
|
|
|
|
fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n",
|
|
|
|
llama_format_win_err(GetLastError()).c_str());
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
void raw_unlock(void * ptr, size_t len) {
|
|
|
|
if (!VirtualUnlock(ptr, len)) {
|
2023-04-10 21:59:13 +02:00
|
|
|
fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n",
|
|
|
|
llama_format_win_err(GetLastError()).c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static constexpr bool SUPPORTED = false;
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
size_t lock_granularity() {
|
|
|
|
return (size_t) 65536;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool raw_lock(const void * addr, size_t len) {
|
2023-04-10 21:59:13 +02:00
|
|
|
fprintf(stderr, "warning: mlock not supported on this system\n");
|
2023-05-23 13:04:39 +02:00
|
|
|
return false;
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
void raw_unlock(const void * addr, size_t len) {}
|
2023-04-10 21:59:13 +02:00
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
|
|
|
// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.
|
|
|
|
struct llama_buffer {
|
|
|
|
uint8_t * addr = NULL;
|
|
|
|
size_t size = 0;
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
llama_buffer() = default;
|
|
|
|
|
|
|
|
void resize(size_t len) {
|
2023-04-10 21:59:13 +02:00
|
|
|
delete[] addr;
|
2023-05-23 13:04:39 +02:00
|
|
|
addr = new uint8_t[len];
|
|
|
|
size = len;
|
2023-04-10 21:59:13 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
~llama_buffer() {
|
|
|
|
delete[] addr;
|
|
|
|
}
|
2023-05-23 13:04:39 +02:00
|
|
|
|
|
|
|
// disable copy and move
|
|
|
|
llama_buffer(const llama_buffer&) = delete;
|
|
|
|
llama_buffer(llama_buffer&&) = delete;
|
|
|
|
llama_buffer& operator=(const llama_buffer&) = delete;
|
|
|
|
llama_buffer& operator=(llama_buffer&&) = delete;
|
2023-04-10 21:59:13 +02:00
|
|
|
};
|
2023-04-30 17:51:57 +02:00
|
|
|
|
|
|
|
#ifdef GGML_USE_CUBLAS
|
|
|
|
#include "ggml-cuda.h"
|
|
|
|
struct llama_ctx_buffer {
|
|
|
|
uint8_t * addr = NULL;
|
2023-05-23 13:04:39 +02:00
|
|
|
bool is_cuda;
|
2023-04-30 17:51:57 +02:00
|
|
|
size_t size = 0;
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
llama_ctx_buffer() = default;
|
|
|
|
|
2023-04-30 17:51:57 +02:00
|
|
|
void resize(size_t size) {
|
2023-05-23 13:04:39 +02:00
|
|
|
free();
|
|
|
|
|
|
|
|
addr = (uint8_t *) ggml_cuda_host_malloc(size);
|
2023-04-30 17:51:57 +02:00
|
|
|
if (addr) {
|
2023-05-23 13:04:39 +02:00
|
|
|
is_cuda = true;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
// fall back to pageable memory
|
|
|
|
addr = new uint8_t[size];
|
|
|
|
is_cuda = false;
|
2023-04-30 17:51:57 +02:00
|
|
|
}
|
|
|
|
this->size = size;
|
|
|
|
}
|
|
|
|
|
2023-05-23 13:04:39 +02:00
|
|
|
void free() {
|
2023-04-30 17:51:57 +02:00
|
|
|
if (addr) {
|
2023-05-23 13:04:39 +02:00
|
|
|
if (is_cuda) {
|
|
|
|
ggml_cuda_host_free(addr);
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
delete[] addr;
|
|
|
|
}
|
2023-04-30 17:51:57 +02:00
|
|
|
}
|
2023-05-23 13:04:39 +02:00
|
|
|
addr = NULL;
|
2023-04-30 17:51:57 +02:00
|
|
|
}
|
2023-05-23 13:04:39 +02:00
|
|
|
|
|
|
|
~llama_ctx_buffer() {
|
|
|
|
free();
|
|
|
|
}
|
|
|
|
|
|
|
|
// disable copy and move
|
|
|
|
llama_ctx_buffer(const llama_ctx_buffer&) = delete;
|
|
|
|
llama_ctx_buffer(llama_ctx_buffer&&) = delete;
|
|
|
|
llama_ctx_buffer& operator=(const llama_ctx_buffer&) = delete;
|
|
|
|
llama_ctx_buffer& operator=(llama_ctx_buffer&&) = delete;
|
2023-04-30 17:51:57 +02:00
|
|
|
};
|
|
|
|
#else
|
|
|
|
typedef llama_buffer llama_ctx_buffer;
|
|
|
|
#endif
|
|
|
|
|
2023-04-10 21:59:13 +02:00
|
|
|
#endif
|