diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index c49ef263c19..a82bc925bc8 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -219,15 +219,15 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup // Handle if we should return "name" instead of "functions" if config.FunctionsConfig.FunctionName { jsStruct := funcs.ToJSONNameStructure() - config.Grammar = jsStruct.Grammar("", config.FunctionsConfig.ParallelCalls) + config.Grammar = jsStruct.Grammar(config.FunctionsConfig.GrammarPrefix, "", config.FunctionsConfig.ParallelCalls, config.FunctionsConfig.GrammarMessage) } else { jsStruct := funcs.ToJSONFunctionStructure() - config.Grammar = jsStruct.Grammar("", config.FunctionsConfig.ParallelCalls) + config.Grammar = jsStruct.Grammar(config.FunctionsConfig.GrammarPrefix, "", config.FunctionsConfig.ParallelCalls, config.FunctionsConfig.GrammarMessage) } case input.JSONFunctionGrammarObject != nil: - config.Grammar = input.JSONFunctionGrammarObject.Grammar("", config.FunctionsConfig.ParallelCalls) + config.Grammar = input.JSONFunctionGrammarObject.Grammar(config.FunctionsConfig.GrammarPrefix, "", config.FunctionsConfig.ParallelCalls, config.FunctionsConfig.GrammarMessage) case input.JSONFunctionGrammarObjectName != nil: - config.Grammar = input.JSONFunctionGrammarObjectName.Grammar("", config.FunctionsConfig.ParallelCalls) + config.Grammar = input.JSONFunctionGrammarObjectName.Grammar(config.FunctionsConfig.GrammarPrefix, "", config.FunctionsConfig.ParallelCalls, config.FunctionsConfig.GrammarMessage) default: // Force picking one of the functions by the request if config.FunctionToCall() != "" { diff --git a/pkg/functions/grammar_json_schema.go b/pkg/functions/grammar_json_schema.go index ede52fab486..6f056b53bac 100644 --- a/pkg/functions/grammar_json_schema.go +++ b/pkg/functions/grammar_json_schema.go @@ -8,6 +8,8 @@ import ( "regexp" "sort" "strings" + + "github.com/go-skynet/LocalAI/pkg/utils" ) const ( @@ -48,6 +50,10 @@ var ( [^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) )* "\"" space`, + "freestring": `( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* space`, "null": `"null" space`, } @@ -111,22 +117,54 @@ const array = `arr ::= (",\n" realvalue)* )? "]"` -func (sc *JSONSchemaConverter) finalizeGrammar(maybeArray bool) string { +func (sc *JSONSchemaConverter) finalizeGrammar(suffix string, maybeArray, maybeString bool) string { var lines []string + + swapRoot := maybeArray || maybeString || suffix != "" + // write down the computed rules. // if maybeArray is true, we need to add the array rule and slightly tweak the root rule for name, rule := range sc.rules { - if maybeArray && name == "root" { + if swapRoot && name == "root" { name = "realvalue" } lines = append(lines, fmt.Sprintf("%s ::= %s", name, rule)) } + if !swapRoot { + return strings.Join(lines, "\n") + } + + newRoot := "realvalue" if maybeArray { - lines = append(lines, fmt.Sprintf("%s ::= %s", "root", "arr | realvalue")) - lines = append(lines, array) + newRoot = "arr | realvalue" + } + + if suffix != "" { + // quote newlines in suffix + suffix = utils.EscapeNewLines(suffix) + + if maybeArray && maybeString { + newRoot = "(" + newRoot + ")" + } + + if maybeString { + //newRoot = "( (\"" + suffix + "\" " + newRoot + ") | freestring ) " + newRoot = "( \"" + suffix + "\" " + newRoot + " | freestring ) " + } else { + newRoot = "\"" + suffix + "\" " + "" + newRoot + "" + } + } else if maybeString { + if maybeArray { + // newRoot = "(" + newRoot + ")" + } + + newRoot = "freestring | " + newRoot } + lines = append(lines, fmt.Sprintf("%s ::= %s", "root", newRoot)) + lines = append(lines, array) + return strings.Join(lines, "\n") } @@ -251,15 +289,16 @@ func (sc *JSONSchemaConverter) resolveReference(ref string, rootSchema map[strin return def } -func (sc *JSONSchemaConverter) Grammar(schema map[string]interface{}, maybeArray bool) string { +func (sc *JSONSchemaConverter) Grammar(suffix string, schema map[string]interface{}, maybeArray, maybeString bool) string { + sc.addRule("freestring", PRIMITIVE_RULES["freestring"]) sc.visit(schema, "", schema) - return sc.finalizeGrammar(maybeArray) + return sc.finalizeGrammar(suffix, maybeArray, maybeString) } -func (sc *JSONSchemaConverter) GrammarFromBytes(b []byte, maybeArray bool) string { +func (sc *JSONSchemaConverter) GrammarFromBytes(suffix string, b []byte, maybeArray, maybeString bool) string { var schema map[string]interface{} _ = json.Unmarshal(b, &schema) - return sc.Grammar(schema, maybeArray) + return sc.Grammar(suffix, schema, maybeArray, maybeString) } func jsonString(v interface{}) string { @@ -302,9 +341,9 @@ type JSONFunctionStructureName struct { Defs map[string]interface{} `json:"$defs,omitempty"` } -func (j JSONFunctionStructureName) Grammar(propOrder string, maybeArray bool) string { +func (j JSONFunctionStructureName) Grammar(suffix string, propOrder string, maybeArray, maybeString bool) string { dat, _ := json.Marshal(j) - return NewJSONSchemaConverter(propOrder).GrammarFromBytes(dat, maybeArray) + return NewJSONSchemaConverter(propOrder).GrammarFromBytes(suffix, dat, maybeArray, maybeString) } type JSONFunctionStructureFunction struct { @@ -313,7 +352,7 @@ type JSONFunctionStructureFunction struct { Defs map[string]interface{} `json:"$defs,omitempty"` } -func (j JSONFunctionStructureFunction) Grammar(propOrder string, maybeArray bool) string { +func (j JSONFunctionStructureFunction) Grammar(suffix string, propOrder string, maybeArray, maybeString bool) string { dat, _ := json.Marshal(j) - return NewJSONSchemaConverter(propOrder).GrammarFromBytes(dat, maybeArray) + return NewJSONSchemaConverter(propOrder).GrammarFromBytes(suffix, dat, maybeArray, maybeString) } diff --git a/pkg/functions/grammar_json_schema_test.go b/pkg/functions/grammar_json_schema_test.go index 83fae3723e4..1a578cc4388 100644 --- a/pkg/functions/grammar_json_schema_test.go +++ b/pkg/functions/grammar_json_schema_test.go @@ -8,6 +8,97 @@ import ( . "github.com/onsi/gomega" ) +var testFunctions = []ItemFunction{ + { + Type: "object", + Properties: FunctionProperties{ + Function: FunctionName{ + Const: "create_event", + }, + Arguments: Argument{ // this is OpenAI's parameter + Type: "object", + Properties: map[string]interface{}{ + "title": map[string]string{"type": "string"}, + "date": map[string]string{"type": "string"}, + "time": map[string]string{"type": "string"}, + }, + }, + }, + }, + { + Type: "object", + Properties: FunctionProperties{ + Function: FunctionName{ + Const: "search", + }, + Arguments: Argument{ + Type: "object", + Properties: map[string]interface{}{ + "query": map[string]string{"type": "string"}, + }, + }, + }, + }, +} + +var testFunctionsName = []ItemName{ + { + Type: "object", + Properties: NameProperties{ + Function: FunctionName{ + Const: "create_event", + }, + Arguments: Argument{ // this is OpenAI's parameter + Type: "object", + Properties: map[string]interface{}{ + "title": map[string]string{"type": "string"}, + "date": map[string]string{"type": "string"}, + "time": map[string]string{"type": "string"}, + }, + }, + }, + }, + { + Type: "object", + Properties: NameProperties{ + Function: FunctionName{ + Const: "search", + }, + Arguments: Argument{ + Type: "object", + Properties: map[string]interface{}{ + "query": map[string]string{"type": "string"}, + }, + }, + }, + }, +} + +func rootResult(s string) string { + return `root-0-name ::= "\"create_event\"" +freestring ::= ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* space +root-0 ::= "{" space "\"arguments\"" space ":" space root-0-arguments "," space "\"name\"" space ":" space root-0-name "}" space +root-1-arguments ::= "{" space "\"query\"" space ":" space string "}" space +realvalue ::= root-0 | root-1 +root ::= ` + s + ` +space ::= " "? +root-0-arguments ::= "{" space "\"date\"" space ":" space string "," space "\"time\"" space ":" space string "," space "\"title\"" space ":" space string "}" space +root-1 ::= "{" space "\"arguments\"" space ":" space root-1-arguments "," space "\"name\"" space ":" space root-1-name "}" space +string ::= "\"" ( +[^"\\] | +"\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) +)* "\"" space +arr ::= +"[\n" ( + realvalue +(",\n" realvalue)* +)? "]" +root-1-name ::= "\"search\""` +} + const ( testInput1 = ` { @@ -42,6 +133,10 @@ const ( }` inputResult1 = `root-0-function ::= "\"create_event\"" +freestring ::= ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* space root-0 ::= "{" space "\"arguments\"" space ":" space root-0-arguments "," space "\"function\"" space ":" space root-0-function "}" space root-1-arguments ::= "{" space "\"query\"" space ":" space string "}" space root ::= root-0 | root-1 @@ -55,6 +150,10 @@ string ::= "\"" ( root-1-function ::= "\"search\""` inputResult2 = `root-0-function ::= "\"create_event\"" +freestring ::= ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* space root-0 ::= "{" space "\"arguments\"" space ":" space root-0-arguments "," space "\"function\"" space ":" space root-0-function "}" space root-1-arguments ::= "{" space "\"query\"" space ":" space string "}" space realvalue ::= root-0 | root-1 @@ -106,6 +205,10 @@ root-1-function ::= "\"search\""` }` inputResult3 = `root-0-name ::= "\"create_event\"" +freestring ::= ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* space root-0 ::= "{" space "\"arguments\"" space ":" space root-0-arguments "," space "\"name\"" space ":" space root-0-name "}" space root-1-arguments ::= "{" space "\"query\"" space ":" space string "}" space root ::= root-0 | root-1 @@ -119,6 +222,10 @@ string ::= "\"" ( root-1-name ::= "\"search\""` inputResult4 = `root-0-name ::= "\"create_event\"" +freestring ::= ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* space root-0 ::= "{" space "\"arguments\"" space ":" space root-0-arguments "," space "\"name\"" space ":" space root-0-name "}" space root-1-arguments ::= "{" space "\"query\"" space ":" space string "}" space realvalue ::= root-0 | root-1 @@ -141,7 +248,7 @@ root-1-name ::= "\"search\""` var _ = Describe("JSON schema grammar tests", func() { Context("JSON", func() { It("generates a valid grammar from JSON schema", func() { - grammar := NewJSONSchemaConverter("").GrammarFromBytes([]byte(testInput1), false) + grammar := NewJSONSchemaConverter("").GrammarFromBytes("", []byte(testInput1), false, false) results := strings.Split(inputResult1, "\n") for _, r := range results { if r != "" { @@ -151,7 +258,7 @@ var _ = Describe("JSON schema grammar tests", func() { Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n")))) }) It("generates a valid grammar from JSON schema", func() { - grammar := NewJSONSchemaConverter("").GrammarFromBytes([]byte(testInput2), false) + grammar := NewJSONSchemaConverter("").GrammarFromBytes("", []byte(testInput2), false, false) results := strings.Split(inputResult3, "\n") for _, r := range results { if r != "" { @@ -163,40 +270,9 @@ var _ = Describe("JSON schema grammar tests", func() { It("generates a valid grammar from JSON Objects", func() { structuredGrammar := JSONFunctionStructureFunction{ - OneOf: []ItemFunction{ - { - Type: "object", - Properties: FunctionProperties{ - Function: FunctionName{ - Const: "create_event", - }, - Arguments: Argument{ // this is OpenAI's parameter - Type: "object", - Properties: map[string]interface{}{ - "title": map[string]string{"type": "string"}, - "date": map[string]string{"type": "string"}, - "time": map[string]string{"type": "string"}, - }, - }, - }, - }, - { - Type: "object", - Properties: FunctionProperties{ - Function: FunctionName{ - Const: "search", - }, - Arguments: Argument{ - Type: "object", - Properties: map[string]interface{}{ - "query": map[string]string{"type": "string"}, - }, - }, - }, - }, - }} + OneOf: testFunctions} - grammar := structuredGrammar.Grammar("", false) + grammar := structuredGrammar.Grammar("", "", false, false) results := strings.Split(inputResult1, "\n") for _, r := range results { if r != "" { @@ -208,40 +284,9 @@ var _ = Describe("JSON schema grammar tests", func() { It("generates a valid grammar from JSON Objects for multiple function return", func() { structuredGrammar := JSONFunctionStructureFunction{ - OneOf: []ItemFunction{ - { - Type: "object", - Properties: FunctionProperties{ - Function: FunctionName{ - Const: "create_event", - }, - Arguments: Argument{ // this is OpenAI's parameter - Type: "object", - Properties: map[string]interface{}{ - "title": map[string]string{"type": "string"}, - "date": map[string]string{"type": "string"}, - "time": map[string]string{"type": "string"}, - }, - }, - }, - }, - { - Type: "object", - Properties: FunctionProperties{ - Function: FunctionName{ - Const: "search", - }, - Arguments: Argument{ - Type: "object", - Properties: map[string]interface{}{ - "query": map[string]string{"type": "string"}, - }, - }, - }, - }, - }} + OneOf: testFunctions} - grammar := structuredGrammar.Grammar("", true) + grammar := structuredGrammar.Grammar("", "", true, false) results := strings.Split(inputResult2, "\n") for _, r := range results { if r != "" { @@ -253,40 +298,9 @@ var _ = Describe("JSON schema grammar tests", func() { It("generates a valid grammar from JSON Objects for multiple function return", func() { structuredGrammar := JSONFunctionStructureName{ - OneOf: []ItemName{ - { - Type: "object", - Properties: NameProperties{ - Function: FunctionName{ - Const: "create_event", - }, - Arguments: Argument{ // this is OpenAI's parameter - Type: "object", - Properties: map[string]interface{}{ - "title": map[string]string{"type": "string"}, - "date": map[string]string{"type": "string"}, - "time": map[string]string{"type": "string"}, - }, - }, - }, - }, - { - Type: "object", - Properties: NameProperties{ - Function: FunctionName{ - Const: "search", - }, - Arguments: Argument{ - Type: "object", - Properties: map[string]interface{}{ - "query": map[string]string{"type": "string"}, - }, - }, - }, - }, - }} + OneOf: testFunctionsName} - grammar := structuredGrammar.Grammar("", true) + grammar := structuredGrammar.Grammar("", "", true, false) results := strings.Split(inputResult4, "\n") for _, r := range results { if r != "" { @@ -295,5 +309,72 @@ var _ = Describe("JSON schema grammar tests", func() { } Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar) }) + + It("generates a valid grammar from JSON Objects for multiple function return with a suffix and array", func() { + structuredGrammar := JSONFunctionStructureName{ + OneOf: testFunctionsName} + + grammar := structuredGrammar.Grammar("suffix", "", true, false) + results := strings.Split(rootResult(`"suffix" arr | realvalue`), "\n") + for _, r := range results { + if r != "" { + Expect(grammar).To(ContainSubstring(r)) + } + } + Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar) + }) + It("generates a valid grammar from JSON Objects with a suffix", func() { + structuredGrammar := JSONFunctionStructureName{ + OneOf: testFunctionsName} + + grammar := structuredGrammar.Grammar("suffix", "", false, false) + results := strings.Split(rootResult(`"suffix" realvalue`), "\n") + for _, r := range results { + if r != "" { + Expect(grammar).To(ContainSubstring(r)) + } + } + Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar) + }) + It("generates a valid grammar from JSON Objects with a suffix and could return string", func() { + structuredGrammar := JSONFunctionStructureName{ + OneOf: testFunctionsName} + + grammar := structuredGrammar.Grammar("suffix", "", false, true) + results := strings.Split(rootResult(`( "suffix" realvalue | freestring )`), "\n") + for _, r := range results { + if r != "" { + Expect(grammar).To(ContainSubstring(r)) + } + } + Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar) + }) + It("generates a valid grammar from JSON Objects with a suffix that could return text or an array of tools", func() { + structuredGrammar := JSONFunctionStructureName{ + OneOf: testFunctionsName} + + grammar := structuredGrammar.Grammar("suffix", "", true, true) + results := strings.Split(rootResult(`( "suffix" (arr | realvalue) | freestring )`), "\n") + for _, r := range results { + if r != "" { + Expect(grammar).To(ContainSubstring(r)) + } + } + Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar) + }) + + It("generates a valid grammar from JSON Objects without a suffix that could return text or an array of tools or just string", func() { + structuredGrammar := JSONFunctionStructureName{ + OneOf: testFunctionsName} + + grammar := structuredGrammar.Grammar("", "", true, true) + results := strings.Split(rootResult(`freestring | arr | realvalue`), "\n") + for _, r := range results { + if r != "" { + Expect(grammar).To(ContainSubstring(r)) + } + } + Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar) + }) }) }) diff --git a/pkg/functions/parse.go b/pkg/functions/parse.go index c6941ff690b..0246d70e7b0 100644 --- a/pkg/functions/parse.go +++ b/pkg/functions/parse.go @@ -4,21 +4,49 @@ import ( "encoding/json" "fmt" "regexp" + "strings" "github.com/go-skynet/LocalAI/pkg/utils" "github.com/rs/zerolog/log" ) +// FunctionsConfig is the configuration for the tool/function call. +// It includes setting to map the function name and arguments from the response +// and, for instance, also if processing the requests with BNF grammars. type FunctionsConfig struct { - DisableNoAction bool `yaml:"disable_no_action"` - NoActionFunctionName string `yaml:"no_action_function_name"` + // DisableNoAction disables the "no action" tool + // By default we inject a tool that does nothing and is used to return an answer from the LLM + DisableNoAction bool `yaml:"disable_no_action"` + + // NoActionFunctionName is the name of the function that does nothing. It defaults to "answer" + NoActionFunctionName string `yaml:"no_action_function_name"` + + // NoActionDescriptionName is the name of the function that returns the description of the no action function NoActionDescriptionName string `yaml:"no_action_description_name"` - ParallelCalls bool `yaml:"parallel_calls"` - NoGrammar bool `yaml:"no_grammar"` - ResponseRegex string `yaml:"response_regex"` + // ParallelCalls enables the LLM to return multiple function calls in the same response + ParallelCalls bool `yaml:"parallel_calls"` + + // GrammarMessage enables the LLM to return strings and not only JSON objects + // This is useful for models to not constraing returning only JSON and also messages back to the user + GrammarMessage bool `yaml:"grammar_message"` + + // NoGrammar disables the grammar parsing and parses the responses directly from the LLM + NoGrammar bool `yaml:"no_grammar"` + + // ResponseRegex is a named regex to extract the function name and arguments from the response + ResponseRegex string `yaml:"response_regex"` + + // JSONRegexMatch is a regex to extract the JSON object from the response JSONRegexMatch string `yaml:"json_regex_match"` + // GrammarPrefix is the suffix to append to the grammar when being generated + // This is useful when models prepend a tag before returning JSON + GrammarPrefix string `yaml:"grammar_prefix"` + + // ReplaceResults allow to replace strings in the results before parsing them + ReplaceResults map[string]string `yaml:"replace_results"` + // FunctionName enable the LLM to return { "name": "function_name", "arguments": { "arg1": "value1", "arg2": "value2" } } // instead of { "function": "function_name", "arguments": { "arg1": "value1", "arg2": "value2" } }. // This might be useful for certain models trained with the function name as the first token. @@ -31,6 +59,15 @@ type FuncCallResults struct { } func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncCallResults { + log.Debug().Msgf("LLM result: %s", llmresult) + + for k, v := range functionConfig.ReplaceResults { + log.Debug().Msgf("Replacing %s with %s", k, v) + llmresult = strings.ReplaceAll(llmresult, k, v) + } + + log.Debug().Msgf("LLM result(processed): %s", llmresult) + multipleResults := functionConfig.ParallelCalls useGrammars := !functionConfig.NoGrammar @@ -48,7 +85,7 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC s = utils.EscapeNewLines(s) err := json.Unmarshal([]byte(s), &ss) if err != nil { - log.Error().Err(err).Str("escapedLLMResult", s).Msg("unable to unmarshal llm result") + log.Warn().Err(err).Str("escapedLLMResult", s).Msg("unable to unmarshal llm result") } log.Debug().Msgf("Function return: %s %+v", s, ss) @@ -132,7 +169,7 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC s := utils.EscapeNewLines(llmresult) err := json.Unmarshal([]byte(s), &ss) if err != nil { - log.Error().Err(err).Str("escapedLLMResult", s).Msg("multiple results: unable to unmarshal llm result") + log.Warn().Err(err).Str("escapedLLMResult", s).Msg("multiple results: unable to unmarshal llm result") } log.Debug().Msgf("Function return: %s %+v", s, ss)