diff --git a/pkg/llm-d-inference-sim/simulator.go b/pkg/llm-d-inference-sim/simulator.go index 9656a5b..e3168a3 100644 --- a/pkg/llm-d-inference-sim/simulator.go +++ b/pkg/llm-d-inference-sim/simulator.go @@ -400,7 +400,7 @@ func (s *VllmSimulator) reqProcessingWorker(ctx context.Context, id int) { if reqCtx.isChatCompletion && req.getToolChoice() != toolChoiceNone && req.getTools() != nil { toolCalls, finishReason, completionTokens, err = createToolCalls(req.getTools(), req.getToolChoice()) } - if toolCalls == nil { + if toolCalls == nil && err == nil { // Either no tool calls were defined, or we randomly chose not to create tool calls, // so we generate a response text. responseTokens, finishReason, completionTokens, err = req.createResponseText(s.mode) diff --git a/pkg/llm-d-inference-sim/tools_test.go b/pkg/llm-d-inference-sim/tools_test.go index 52ca9c1..a84f380 100644 --- a/pkg/llm-d-inference-sim/tools_test.go +++ b/pkg/llm-d-inference-sim/tools_test.go @@ -160,12 +160,16 @@ var toolWith3DArray = []openai.ChatCompletionToolParam{ "type": "object", "properties": map[string]interface{}{ "tensor": map[string]interface{}{ - "type": "array", + "type": "array", + "minItems": 2, "items": map[string]any{ - "type": "array", + "type": "array", + "minItems": 0, + "maxItems": 1, "items": map[string]any{ - "type": "array", - "items": map[string]string{"type": "string"}, + "type": "array", + "items": map[string]string{"type": "string"}, + "maxItems": 3, }, }, "description": "List of strings", @@ -177,6 +181,28 @@ var toolWith3DArray = []openai.ChatCompletionToolParam{ }, } +var toolWithWrongMinMax = []openai.ChatCompletionToolParam{ + { + Function: openai.FunctionDefinitionParam{ + Name: "multiply_numbers", + Description: openai.String("Multiply an array of numbers"), + Parameters: openai.FunctionParameters{ + "type": "object", + "properties": map[string]interface{}{ + "numbers": map[string]interface{}{ + "type": "array", + "items": map[string]string{"type": "number"}, + "description": "List of numbers to multiply", + "minItems": 3, + "maxItems": 1, + }, + }, + "required": []string{"numbers"}, + }, + }, + }, +} + var toolWithObjects = []openai.ChatCompletionToolParam{ { Function: openai.FunctionDefinitionParam{ @@ -525,6 +551,14 @@ var _ = Describe("Simulator for request with tools", func() { err = json.Unmarshal([]byte(tc.Function.Arguments), &args) Expect(err).NotTo(HaveOccurred()) Expect(args["tensor"]).ToNot(BeEmpty()) + tensor := args["tensor"] + Expect(len(tensor)).To(BeNumerically(">=", 2)) + for _, elem := range tensor { + Expect(len(elem)).To(Or(Equal(0), Equal(1))) + for _, inner := range elem { + Expect(len(inner)).To(Or(Equal(1), Equal(2), Equal(3))) + } + } }, func(mode string) string { return "mode: " + mode @@ -536,6 +570,32 @@ var _ = Describe("Simulator for request with tools", func() { Entry(nil, modeRandom), ) + DescribeTable("array parameter with wrong min and max items, no streaming", + func(mode string) { + ctx := context.TODO() + client, err := startServer(ctx, mode) + Expect(err).NotTo(HaveOccurred()) + + openaiclient := openai.NewClient( + option.WithBaseURL(baseURL), + option.WithHTTPClient(client)) + + params := openai.ChatCompletionNewParams{ + Messages: []openai.ChatCompletionMessageParamUnion{openai.UserMessage(userMessage)}, + Model: model, + ToolChoice: openai.ChatCompletionToolChoiceOptionUnionParam{OfAuto: param.NewOpt("required")}, + Tools: toolWithWrongMinMax, + } + + _, err = openaiclient.Chat.Completions.New(ctx, params) + Expect(err).To(HaveOccurred()) + }, + func(mode string) string { + return "mode: " + mode + }, + Entry(nil, modeRandom), + ) + DescribeTable("objects, no streaming", func(mode string) { ctx := context.TODO() diff --git a/pkg/llm-d-inference-sim/tools_utils.go b/pkg/llm-d-inference-sim/tools_utils.go index c6c3c5f..273dd08 100644 --- a/pkg/llm-d-inference-sim/tools_utils.go +++ b/pkg/llm-d-inference-sim/tools_utils.go @@ -53,7 +53,11 @@ func createToolCalls(tools []tool, toolChoice string) ([]toolCall, string, int, // In case of 'required' at least one tool call has to be created, and we randomly choose // the number of calls starting from one. Otherwise, we start from 0, and in case we randomly // choose the number of calls to be 0, response text will be generated instead of a tool call. - numberOfCalls := randomInt(len(tools), toolChoice == toolChoiceRequired) + min := 0 + if toolChoice == toolChoiceRequired { + min = 1 + } + numberOfCalls := randomInt(min, len(tools)) if numberOfCalls == 0 { return nil, "", 0, nil } @@ -61,7 +65,7 @@ func createToolCalls(tools []tool, toolChoice string) ([]toolCall, string, int, calls := make([]toolCall, 0) for i := range numberOfCalls { // Randomly choose which tools to call. We may call the same tool more than once. - index := randomInt(len(tools)-1, false) + index := randomInt(0, len(tools)-1) args, err := generateToolArguments(tools[index]) if err != nil { return nil, "", 0, err @@ -130,7 +134,7 @@ func createArgument(property any) (any, error) { if ok { enumArray, ok := enum.([]any) if ok && len(enumArray) > 0 { - index := randomInt(len(enumArray)-1, false) + index := randomInt(0, len(enumArray)-1) return enumArray[index], nil } } @@ -139,13 +143,24 @@ func createArgument(property any) (any, error) { case "string": return getStringArgument(), nil case "number": - return randomInt(100, false), nil + return randomInt(0, 100), nil case "boolean": return flipCoin(), nil case "array": items := propertyMap["items"] itemsMap := items.(map[string]any) - numberOfElements := randomInt(5, true) + minItems := 1 + maxItems := 5 + if value, ok := propertyMap["minItems"]; ok { + minItems = int(value.(float64)) + } + if value, ok := propertyMap["maxItems"]; ok { + maxItems = int(value.(float64)) + } + if minItems > maxItems { + return nil, fmt.Errorf("minItems (%d) is greater than maxItems(%d)", minItems, maxItems) + } + numberOfElements := randomInt(minItems, maxItems) array := make([]any, numberOfElements) for i := range numberOfElements { elem, err := createArgument(itemsMap) @@ -177,7 +192,7 @@ func createArgument(property any) (any, error) { } func getStringArgument() string { - index := randomInt(len(fakeStringArguments)-1, false) + index := randomInt(0, len(fakeStringArguments)-1) return fakeStringArguments[index] } @@ -336,6 +351,14 @@ const schema = `{ "items": { "type": "string" } + }, + "minItems": { + "type": "integer", + "minimum": 0 + }, + "maxItems": { + "type": "integer", + "minimum": 0 } }, "required": [ diff --git a/pkg/llm-d-inference-sim/utils.go b/pkg/llm-d-inference-sim/utils.go index e76e0d2..9701b04 100644 --- a/pkg/llm-d-inference-sim/utils.go +++ b/pkg/llm-d-inference-sim/utils.go @@ -62,7 +62,7 @@ func getMaxTokens(maxCompletionTokens *int64, maxTokens *int64) (*int64, error) // getRandomResponseText returns random response text from the pre-defined list of responses // considering max completion tokens if it is not nil, and a finish reason (stop or length) func getRandomResponseText(maxCompletionTokens *int64) (string, string) { - index := randomInt(len(chatCompletionFakeResponses)-1, false) + index := randomInt(0, len(chatCompletionFakeResponses)-1) text := chatCompletionFakeResponses[index] return getResponseText(maxCompletionTokens, text) @@ -105,26 +105,22 @@ func randomNumericString(length int) string { digits := "0123456789" result := make([]byte, length) for i := 0; i < length; i++ { - num := randomInt(9, false) + num := randomInt(0, 9) result[i] = digits[num] } return string(result) } -// Returns an integer between 0 and max (included), unless startFromeOne is true, -// in which case returns an integer between 1 and max (included) -func randomInt(max int, startFromOne bool) int { +// Returns an integer between min and max (included) +func randomInt(min int, max int) int { src := rand.NewSource(time.Now().UnixNano()) r := rand.New(src) - if startFromOne { - return r.Intn(max) + 1 // [1, max] - } - return r.Intn(max + 1) // [0, max] + return r.Intn(max-min+1) + min } // Returns true or false randomly func flipCoin() bool { - return randomInt(1, false) != 0 + return randomInt(0, 1) != 0 } // Regular expression for the response tokenization