From f1be7d0ce0ee945e4b895788ecf8751d5369bb83 Mon Sep 17 00:00:00 2001 From: Sean Goedecke Date: Mon, 21 Jul 2025 00:07:09 +0000 Subject: [PATCH 1/4] Use string format for jsonSchema --- cmd/eval/eval_test.go | 16 +---- cmd/run/run_test.go | 17 +----- examples/json_schema_prompt.yml | 98 ++++++++++++++---------------- pkg/prompt/prompt.go | 67 ++++++++++++++++----- pkg/prompt/prompt_test.go | 102 +++++++++++++++++++++----------- 5 files changed, 162 insertions(+), 138 deletions(-) diff --git a/cmd/eval/eval_test.go b/cmd/eval/eval_test.go index 2909959f..90228766 100644 --- a/cmd/eval/eval_test.go +++ b/cmd/eval/eval_test.go @@ -569,21 +569,7 @@ name: JSON Schema Evaluation description: Testing responseFormat and jsonSchema in eval model: openai/gpt-4o responseFormat: json_schema -jsonSchema: - name: response_schema - strict: true - schema: - type: object - properties: - message: - type: string - description: The response message - confidence: - type: number - description: Confidence score - required: - - message - additionalProperties: false +jsonSchema: '{"name": "response_schema", "strict": true, "schema": {"type": "object", "properties": {"message": {"type": "string", "description": "The response message"}, "confidence": {"type": "number", "description": "Confidence score"}}, "required": ["message"], "additionalProperties": false}}' testData: - input: "hello" expected: "hello world" diff --git a/cmd/run/run_test.go b/cmd/run/run_test.go index 072a212e..94db2b63 100644 --- a/cmd/run/run_test.go +++ b/cmd/run/run_test.go @@ -341,22 +341,7 @@ name: JSON Schema Test description: Test responseFormat and jsonSchema model: openai/test-model responseFormat: json_schema -jsonSchema: - name: person_schema - strict: true - schema: - type: object - properties: - name: - type: string - description: The name - age: - type: integer - description: The age - required: - - name - - age - additionalProperties: false +jsonSchema: '{"name": "person_schema", "strict": true, "schema": {"type": "object", "properties": {"name": {"type": "string", "description": "The name"}, "age": {"type": "integer", "description": "The age"}}, "required": ["name", "age"], "additionalProperties": false}}' messages: - role: system content: You are a helpful assistant. diff --git a/examples/json_schema_prompt.yml b/examples/json_schema_prompt.yml index a10484dc..3f340647 100644 --- a/examples/json_schema_prompt.yml +++ b/examples/json_schema_prompt.yml @@ -1,64 +1,52 @@ -name: JSON Schema Response Example -description: Example prompt demonstrating responseFormat and jsonSchema usage -model: openai/gpt-4o +name: JSON Schema String Format Example +description: Example using JSON string format for jsonSchema +model: openai/gpt-4o-mini responseFormat: json_schema -jsonSchema: - name: Person Information Schema - strict: true - schema: - type: object - description: A structured response containing person information - properties: - name: - type: string - description: The full name of the person - age: - type: integer - description: The age of the person in years - minimum: 0 - maximum: 150 - email: - type: string - description: The email address of the person - format: email - skills: - type: array - description: A list of skills the person has - items: - type: string - address: - type: object - description: The person's address - properties: - street: - type: string - description: Street address - city: - type: string - description: City name - country: - type: string - description: Country name - required: - - city - - country - required: - - name - - age +jsonSchema: |- + { + "name": "animal_description", + "strict": true, + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the animal" + }, + "habitat": { + "type": "string", + "description": "The habitat where the animal lives" + }, + "diet": { + "type": "string", + "description": "What the animal eats", + "enum": ["carnivore", "herbivore", "omnivore"] + }, + "characteristics": { + "type": "array", + "description": "Key characteristics of the animal", + "items": { + "type": "string" + } + } + }, + "required": ["name", "habitat", "diet"], + "additionalProperties": false + } + } messages: - role: system - content: You are a helpful assistant that provides structured information about people. + content: You are a helpful assistant that provides detailed information about animals. - role: user - content: "Generate information for a person named {{name}} who is {{age}} years old." + content: "Describe a {{animal}} in detail." testData: - - name: "Alice Johnson" - age: "30" - - name: "Bob Smith" - age: "25" + - animal: "dog" + - animal: "cat" + - animal: "elephant" evaluators: - - name: has-required-fields + - name: has-name string: contains: "name" - - name: valid-json-structure + - name: has-habitat string: - contains: "age" + contains: "habitat" diff --git a/pkg/prompt/prompt.go b/pkg/prompt/prompt.go index de60c4c3..a224b33c 100644 --- a/pkg/prompt/prompt.go +++ b/pkg/prompt/prompt.go @@ -2,6 +2,7 @@ package prompt import ( + "encoding/json" "fmt" "os" "strings" @@ -67,11 +68,30 @@ type Choice struct { Score float64 `yaml:"score"` } -// JsonSchema represents a JSON schema for structured responses -type JsonSchema struct { - Name string `yaml:"name" json:"name"` - Strict *bool `yaml:"strict,omitempty" json:"strict,omitempty"` - Schema map[string]interface{} `yaml:"schema" json:"schema"` +// JsonSchema represents a JSON schema for structured responses as a JSON string +type JsonSchema string + +// UnmarshalYAML implements custom YAML unmarshaling for JsonSchema +// Only supports JSON string format +func (js *JsonSchema) UnmarshalYAML(node *yaml.Node) error { + // Only support string nodes (JSON format) + if node.Kind != yaml.ScalarNode { + return fmt.Errorf("jsonSchema must be a JSON string") + } + + var jsonStr string + if err := node.Decode(&jsonStr); err != nil { + return err + } + + // Validate that it's valid JSON + var temp interface{} + if err := json.Unmarshal([]byte(jsonStr), &temp); err != nil { + return fmt.Errorf("invalid JSON in jsonSchema: %w", err) + } + + *js = JsonSchema(jsonStr) + return nil } // LoadFromFile loads and parses a prompt file from the given path @@ -105,16 +125,24 @@ func (f *File) validateResponseFormat() error { return fmt.Errorf("invalid responseFormat: %s. Must be 'text', 'json_object', or 'json_schema'", *f.ResponseFormat) } - // If responseFormat is "json_schema", jsonSchema must be provided with required fields + // If responseFormat is "json_schema", jsonSchema must be provided if *f.ResponseFormat == "json_schema" { if f.JsonSchema == nil { return fmt.Errorf("jsonSchema is required when responseFormat is 'json_schema'") } - if f.JsonSchema.Name == "" { - return fmt.Errorf("jsonSchema.name is required when responseFormat is 'json_schema'") + + // Parse and validate the JSON schema + var schema map[string]interface{} + if err := json.Unmarshal([]byte(*f.JsonSchema), &schema); err != nil { + return fmt.Errorf("invalid JSON in jsonSchema: %w", err) + } + + // Check for required fields + if _, ok := schema["name"]; !ok { + return fmt.Errorf("jsonSchema must contain 'name' field") } - if f.JsonSchema.Schema == nil { - return fmt.Errorf("jsonSchema.schema is required when responseFormat is 'json_schema'") + if _, ok := schema["schema"]; !ok { + return fmt.Errorf("jsonSchema must contain 'schema' field") } } @@ -193,13 +221,20 @@ func (f *File) BuildChatCompletionOptions(messages []azuremodels.ChatMessage) az Type: *f.ResponseFormat, } if f.JsonSchema != nil { - // Convert JsonSchema to map[string]interface{} - schemaMap := make(map[string]interface{}) - schemaMap["name"] = f.JsonSchema.Name - if f.JsonSchema.Strict != nil { - schemaMap["strict"] = *f.JsonSchema.Strict + // Parse the JSON schema string into a map + var schemaMap map[string]interface{} + if err := json.Unmarshal([]byte(*f.JsonSchema), &schemaMap); err != nil { + // This should not happen as we validate during unmarshaling + // but we'll handle it gracefully + schemaMap = map[string]interface{}{ + "name": "default_schema", + "strict": true, + "schema": map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{}, + }, + } } - schemaMap["schema"] = f.JsonSchema.Schema responseFormat.JsonSchema = &schemaMap } req.ResponseFormat = responseFormat diff --git a/pkg/prompt/prompt_test.go b/pkg/prompt/prompt_test.go index 31066b3b..51275725 100644 --- a/pkg/prompt/prompt_test.go +++ b/pkg/prompt/prompt_test.go @@ -1,6 +1,7 @@ package prompt import ( + "encoding/json" "os" "path/filepath" "testing" @@ -139,27 +140,35 @@ messages: require.Nil(t, promptFile.JsonSchema) }) - t.Run("loads prompt file with responseFormat json_schema and jsonSchema", func(t *testing.T) { + t.Run("loads prompt file with responseFormat json_schema and jsonSchema as JSON string", func(t *testing.T) { const yamlBody = ` -name: JSON Schema Response Format Test -description: Test with JSON schema response format +name: JSON Schema String Format Test +description: Test with JSON schema as JSON string model: openai/gpt-4o responseFormat: json_schema -jsonSchema: - name: person_info - strict: true - schema: - type: object - properties: - name: - type: string - description: The name of the person - age: - type: integer - description: The age of the person - required: - - name - additionalProperties: false +jsonSchema: |- + { + "name": "describe_animal", + "strict": true, + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name of the animal" + }, + "habitat": { + "type": "string", + "description": "The habitat the animal lives in" + } + }, + "additionalProperties": false, + "required": [ + "name", + "habitat" + ] + } + } messages: - role: user content: "Hello" @@ -175,10 +184,29 @@ messages: require.NotNil(t, promptFile.ResponseFormat) require.Equal(t, "json_schema", *promptFile.ResponseFormat) require.NotNil(t, promptFile.JsonSchema) - require.Equal(t, "person_info", promptFile.JsonSchema.Name) - require.True(t, *promptFile.JsonSchema.Strict) - require.Contains(t, promptFile.JsonSchema.Schema, "type") - require.Contains(t, promptFile.JsonSchema.Schema, "properties") + + // Parse the JSON schema string to verify its contents + var schema map[string]interface{} + err = json.Unmarshal([]byte(*promptFile.JsonSchema), &schema) + require.NoError(t, err) + + require.Equal(t, "describe_animal", schema["name"]) + require.Equal(t, true, schema["strict"]) + require.Contains(t, schema, "schema") + + // Verify the nested schema structure + nestedSchema := schema["schema"].(map[string]interface{}) + require.Equal(t, "object", nestedSchema["type"]) + require.Contains(t, nestedSchema, "properties") + require.Contains(t, nestedSchema, "required") + + properties := nestedSchema["properties"].(map[string]interface{}) + require.Contains(t, properties, "name") + require.Contains(t, properties, "habitat") + + required := nestedSchema["required"].([]interface{}) + require.Contains(t, required, "name") + require.Contains(t, required, "habitat") }) t.Run("validates invalid responseFormat", func(t *testing.T) { @@ -224,23 +252,25 @@ messages: }) t.Run("BuildChatCompletionOptions includes responseFormat and jsonSchema", func(t *testing.T) { + jsonSchemaStr := `{ + "name": "test_schema", + "strict": true, + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The name" + } + }, + "required": ["name"] + } + }` + promptFile := &File{ Model: "openai/gpt-4o", ResponseFormat: func() *string { s := "json_schema"; return &s }(), - JsonSchema: &JsonSchema{ - Name: "test_schema", - Strict: func() *bool { b := true; return &b }(), - Schema: map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "name": map[string]interface{}{ - "type": "string", - "description": "The name", - }, - }, - "required": []string{"name"}, - }, - }, + JsonSchema: func() *JsonSchema { js := JsonSchema(jsonSchemaStr); return &js }(), } messages := []azuremodels.ChatMessage{ From 54155631c81c0b4ab6a80c6cc218563c3feb99ff Mon Sep 17 00:00:00 2001 From: Sean Goedecke Date: Mon, 21 Jul 2025 00:09:54 +0000 Subject: [PATCH 2/4] Update example prompt --- examples/json_schema_prompt.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/json_schema_prompt.yml b/examples/json_schema_prompt.yml index 3f340647..ffb34b1b 100644 --- a/examples/json_schema_prompt.yml +++ b/examples/json_schema_prompt.yml @@ -1,5 +1,5 @@ -name: JSON Schema String Format Example -description: Example using JSON string format for jsonSchema +name: JSON Schema Response Example +description: Example prompt demonstrating responseFormat and jsonSchema usage model: openai/gpt-4o-mini responseFormat: json_schema jsonSchema: |- From dae657699f222ee3debaed0d1d9dcd0ca4d25a96 Mon Sep 17 00:00:00 2001 From: Sean Goedecke Date: Mon, 21 Jul 2025 03:55:29 +0000 Subject: [PATCH 3/4] Refactor away the double parsing --- pkg/prompt/prompt.go | 46 ++++++++++++--------------------------- pkg/prompt/prompt_test.go | 13 ++++++----- 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/pkg/prompt/prompt.go b/pkg/prompt/prompt.go index a224b33c..05911cb7 100644 --- a/pkg/prompt/prompt.go +++ b/pkg/prompt/prompt.go @@ -68,8 +68,11 @@ type Choice struct { Score float64 `yaml:"score"` } -// JsonSchema represents a JSON schema for structured responses as a JSON string -type JsonSchema string +// JsonSchema represents a JSON schema for structured responses +type JsonSchema struct { + Raw string + Parsed map[string]interface{} +} // UnmarshalYAML implements custom YAML unmarshaling for JsonSchema // Only supports JSON string format @@ -84,13 +87,14 @@ func (js *JsonSchema) UnmarshalYAML(node *yaml.Node) error { return err } - // Validate that it's valid JSON - var temp interface{} - if err := json.Unmarshal([]byte(jsonStr), &temp); err != nil { + // Parse and validate the JSON schema + var parsed map[string]interface{} + if err := json.Unmarshal([]byte(jsonStr), &parsed); err != nil { return fmt.Errorf("invalid JSON in jsonSchema: %w", err) } - *js = JsonSchema(jsonStr) + js.Raw = jsonStr + js.Parsed = parsed return nil } @@ -131,17 +135,11 @@ func (f *File) validateResponseFormat() error { return fmt.Errorf("jsonSchema is required when responseFormat is 'json_schema'") } - // Parse and validate the JSON schema - var schema map[string]interface{} - if err := json.Unmarshal([]byte(*f.JsonSchema), &schema); err != nil { - return fmt.Errorf("invalid JSON in jsonSchema: %w", err) - } - - // Check for required fields - if _, ok := schema["name"]; !ok { + // Check for required fields in the already parsed schema + if _, ok := f.JsonSchema.Parsed["name"]; !ok { return fmt.Errorf("jsonSchema must contain 'name' field") } - if _, ok := schema["schema"]; !ok { + if _, ok := f.JsonSchema.Parsed["schema"]; !ok { return fmt.Errorf("jsonSchema must contain 'schema' field") } } @@ -204,7 +202,6 @@ func (f *File) BuildChatCompletionOptions(messages []azuremodels.ChatMessage) az Stream: false, } - // Apply model parameters if f.ModelParameters.MaxTokens != nil { req.MaxTokens = f.ModelParameters.MaxTokens } @@ -215,27 +212,12 @@ func (f *File) BuildChatCompletionOptions(messages []azuremodels.ChatMessage) az req.TopP = f.ModelParameters.TopP } - // Apply response format if f.ResponseFormat != nil { responseFormat := &azuremodels.ResponseFormat{ Type: *f.ResponseFormat, } if f.JsonSchema != nil { - // Parse the JSON schema string into a map - var schemaMap map[string]interface{} - if err := json.Unmarshal([]byte(*f.JsonSchema), &schemaMap); err != nil { - // This should not happen as we validate during unmarshaling - // but we'll handle it gracefully - schemaMap = map[string]interface{}{ - "name": "default_schema", - "strict": true, - "schema": map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{}, - }, - } - } - responseFormat.JsonSchema = &schemaMap + responseFormat.JsonSchema = &f.JsonSchema.Parsed } req.ResponseFormat = responseFormat } diff --git a/pkg/prompt/prompt_test.go b/pkg/prompt/prompt_test.go index 51275725..5967d692 100644 --- a/pkg/prompt/prompt_test.go +++ b/pkg/prompt/prompt_test.go @@ -185,11 +185,8 @@ messages: require.Equal(t, "json_schema", *promptFile.ResponseFormat) require.NotNil(t, promptFile.JsonSchema) - // Parse the JSON schema string to verify its contents - var schema map[string]interface{} - err = json.Unmarshal([]byte(*promptFile.JsonSchema), &schema) - require.NoError(t, err) - + // Verify the schema contents using the already parsed data + schema := promptFile.JsonSchema.Parsed require.Equal(t, "describe_animal", schema["name"]) require.Equal(t, true, schema["strict"]) require.Contains(t, schema, "schema") @@ -270,7 +267,11 @@ messages: promptFile := &File{ Model: "openai/gpt-4o", ResponseFormat: func() *string { s := "json_schema"; return &s }(), - JsonSchema: func() *JsonSchema { js := JsonSchema(jsonSchemaStr); return &js }(), + JsonSchema: func() *JsonSchema { + js := &JsonSchema{Raw: jsonSchemaStr} + json.Unmarshal([]byte(jsonSchemaStr), &js.Parsed) + return js + }(), } messages := []azuremodels.ChatMessage{ From ec3ceed9faf27f0ddbf3727d2921a995bda2009d Mon Sep 17 00:00:00 2001 From: Sean Goedecke Date: Mon, 21 Jul 2025 04:18:57 +0000 Subject: [PATCH 4/4] Check return value of Unmarshal --- pkg/prompt/prompt_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/prompt/prompt_test.go b/pkg/prompt/prompt_test.go index 5967d692..6783d7fd 100644 --- a/pkg/prompt/prompt_test.go +++ b/pkg/prompt/prompt_test.go @@ -269,7 +269,10 @@ messages: ResponseFormat: func() *string { s := "json_schema"; return &s }(), JsonSchema: func() *JsonSchema { js := &JsonSchema{Raw: jsonSchemaStr} - json.Unmarshal([]byte(jsonSchemaStr), &js.Parsed) + err := json.Unmarshal([]byte(jsonSchemaStr), &js.Parsed) + if err != nil { + t.Fatal(err) + } return js }(), }