f305bad11e
Problem ------- `nix build` fails with missing `Accelerate.h`. Changes ------- - Fix build of the llama.cpp with nix for Intel: add the same SDK frameworks as for ARM - Add `quantize` app to the output of nix flake - Extend nix devShell with llama-python so we can use convertScript Testing ------- Testing the steps with nix: 1. `nix build` Get the model and then 2. `nix develop` and then `python convert.py models/llama-2-7b.ggmlv3.q4_0.bin` 3. `nix run llama.cpp#quantize -- open_llama_7b/ggml-model-f16.gguf ./models/ggml-model-q4_0.bin 2` 4. `nix run llama.cpp#llama -- -m models/ggml-model-q4_0.bin -p "What is nix?" -n 400 --temp 0.8 -e -t 8` Co-authored-by: Volodymyr Vitvitskyi <volodymyrvitvitskyi@SamsungPro.local>
100 lines
3.4 KiB
Nix
100 lines
3.4 KiB
Nix
{
|
|
inputs = {
|
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
|
flake-utils.url = "github:numtide/flake-utils";
|
|
};
|
|
outputs = { self, nixpkgs, flake-utils }:
|
|
flake-utils.lib.eachDefaultSystem (system:
|
|
let
|
|
inherit (pkgs.stdenv) isAarch32 isAarch64 isDarwin;
|
|
buildInputs = with pkgs; [ openmpi ];
|
|
osSpecific = with pkgs; buildInputs ++
|
|
(
|
|
if isAarch64 && isDarwin then
|
|
with pkgs.darwin.apple_sdk_11_0.frameworks; [
|
|
Accelerate
|
|
MetalKit
|
|
]
|
|
else if isAarch32 && isDarwin then
|
|
with pkgs.darwin.apple_sdk.frameworks; [
|
|
Accelerate
|
|
CoreGraphics
|
|
CoreVideo
|
|
]
|
|
else if isDarwin then
|
|
with pkgs.darwin.apple_sdk.frameworks; [
|
|
Accelerate
|
|
CoreGraphics
|
|
CoreVideo
|
|
]
|
|
else
|
|
with pkgs; [ openblas ]
|
|
);
|
|
pkgs = import nixpkgs { inherit system; };
|
|
nativeBuildInputs = with pkgs; [ cmake pkgconfig ];
|
|
llama-python =
|
|
pkgs.python3.withPackages (ps: with ps; [ numpy sentencepiece ]);
|
|
postPatch = ''
|
|
substituteInPlace ./ggml-metal.m \
|
|
--replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";"
|
|
substituteInPlace ./*.py --replace '/usr/bin/env python' '${llama-python}/bin/python'
|
|
'';
|
|
postInstall = ''
|
|
mv $out/bin/main $out/bin/llama
|
|
mv $out/bin/server $out/bin/llama-server
|
|
'';
|
|
cmakeFlags = [ "-DLLAMA_BUILD_SERVER=ON" "-DLLAMA_MPI=ON" "-DBUILD_SHARED_LIBS=ON" "-DCMAKE_SKIP_BUILD_RPATH=ON" ];
|
|
in {
|
|
packages.default = pkgs.stdenv.mkDerivation {
|
|
name = "llama.cpp";
|
|
src = ./.;
|
|
postPatch = postPatch;
|
|
nativeBuildInputs = nativeBuildInputs;
|
|
buildInputs = osSpecific;
|
|
cmakeFlags = cmakeFlags
|
|
++ (if isAarch64 && isDarwin then [
|
|
"-DCMAKE_C_FLAGS=-D__ARM_FEATURE_DOTPROD=1"
|
|
"-DLLAMA_METAL=ON"
|
|
] else [
|
|
"-DLLAMA_BLAS=ON"
|
|
"-DLLAMA_BLAS_VENDOR=OpenBLAS"
|
|
]);
|
|
postInstall = postInstall;
|
|
meta.mainProgram = "llama";
|
|
};
|
|
packages.opencl = pkgs.stdenv.mkDerivation {
|
|
name = "llama.cpp";
|
|
src = ./.;
|
|
postPatch = postPatch;
|
|
nativeBuildInputs = nativeBuildInputs;
|
|
buildInputs = with pkgs; buildInputs ++ [ clblast ];
|
|
cmakeFlags = cmakeFlags ++ [
|
|
"-DLLAMA_CLBLAST=ON"
|
|
];
|
|
postInstall = postInstall;
|
|
meta.mainProgram = "llama";
|
|
};
|
|
apps.llama-server = {
|
|
type = "app";
|
|
program = "${self.packages.${system}.default}/bin/llama-server";
|
|
};
|
|
apps.llama-embedding = {
|
|
type = "app";
|
|
program = "${self.packages.${system}.default}/bin/embedding";
|
|
};
|
|
apps.llama = {
|
|
type = "app";
|
|
program = "${self.packages.${system}.default}/bin/llama";
|
|
};
|
|
apps.quantize = {
|
|
type = "app";
|
|
program = "${self.packages.${system}.default}/bin/quantize";
|
|
};
|
|
apps.default = self.apps.${system}.llama;
|
|
devShells.default = pkgs.mkShell {
|
|
buildInputs = [ llama-python ];
|
|
packages = nativeBuildInputs ++ osSpecific;
|
|
};
|
|
});
|
|
}
|