haraldwolff/llama.cpp
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Fix grammar-based sampling issue in server (#2566)

Browse source
This commit is contained in:
Martin Krasser 2023-08-10 12:16:38 +02:00 committed by GitHub
parent 916a9acdd0
commit 1638757767
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
examples/server/server.cpp
View file

@ -196,6 +196,7 @@ struct llama_server_context
llama_context *ctx = nullptr;
gpt_params params;
grammar_parser::parse_state parsed_grammar;
llama_grammar *grammar = nullptr;
bool truncated = false;
@ -241,10 +242,13 @@ struct llama_server_context
stopped_limit = false;
stopping_word = "";
multibyte_pending = 0;
grammar = nullptr;
n_remain = 0;
n_past = 0;
if (grammar != nullptr) {
llama_grammar_free(grammar);
grammar = nullptr;
}
}
bool loadModel(const gpt_params &params_)
@ -265,8 +269,6 @@ struct llama_server_context
bool loadGrammar()
{
if (!params.grammar.empty()) {
grammar_parser::parse_state parsed_grammar;
parsed_grammar = grammar_parser::parse(params.grammar.c_str());
// will be empty (default) if there are parse errors
if (parsed_grammar.rules.empty()) {