dc07dc492e
* convert: Fix permute calls and method/func definitions * Cleanups for gguf-py * Minor types cleanups. * Initial implementation of handling merges and special tokens * convert: Handle special tokens and merges in vocab only mode convert: Vocab only mode no longer requires loading model tensors * gguf: Refactor tensor name mapping * convert: Fix type hint for special_token_types in SpecialVocab * Use common special vocab handling in various conversion scripts * First pass at implementing suggested changes * Second pass * gguf: SpecialVocab: Fix issue with special token content not in a dict gguf: SpecialVocab: Allow skipping handling of merges * convert-falcon-hf-to-gguf: Support --vocab-only option, bail out if no tokenizer.json * convert-gptneox-hf-to-gguf and convert: Only handle merges for BPE tokenizer * gguf: SpecialVocab: Actually set load_merges in object * Uniform args parsing and vocab only mode for convert examples * convert.py: Set gpt2 as tokenizer model when using BPE * Squish last type warning in gguf.py - yay!
30 lines
697 B
TOML
30 lines
697 B
TOML
[tool.poetry]
|
|
name = "gguf"
|
|
version = "0.2.1"
|
|
description = "Write ML models in GGUF for GGML"
|
|
authors = ["GGML <ggml@ggml.ai>"]
|
|
packages = [
|
|
{include = "gguf"},
|
|
{include = "gguf/py.typed"},
|
|
]
|
|
readme = "README.md"
|
|
homepage = "https://ggml.ai"
|
|
repository = "https://github.com/ggerganov/llama.cpp"
|
|
keywords = ["ggml", "gguf", "llama.cpp"]
|
|
classifiers = [
|
|
"Programming Language :: Python :: 3",
|
|
"License :: OSI Approved :: MIT License",
|
|
"Operating System :: OS Independent",
|
|
]
|
|
|
|
[tool.poetry.dependencies]
|
|
python = ">=3.8"
|
|
numpy = ">=1.17"
|
|
|
|
[tool.poetry.dev-dependencies]
|
|
pytest = "^5.2"
|
|
|
|
[build-system]
|
|
requires = ["poetry-core>=1.0.0"]
|
|
build-backend = "poetry.core.masonry.api"
|