fix(openai): move image detail property inside image_url object

packages/core/src/llms/type.ts

@@ -194,8 +194,10 @@ export type MessageContentTextDetail = {
 
 export type MessageContentImageDetail = {
   type: "image_url";
-  image_url: { url: string };
-  detail?: "high" | "low" | "auto";
+  image_url: {
+    url: string;
+    detail?: "high" | "low" | "auto";
+  };
 };
 
 export type MessageContentAudioDetail = {

packages/providers/openai/src/llm.ts

@@ -246,7 +246,16 @@ export class OpenAI extends ToolCallLLM<OpenAIAdditionalChatOptions> {
             }
 
             // Keep other types as is (text, image_url, etc.)
-            return item;
+            // return item;
+            if (item.type === "image_url") {
+              return {
+                type: "image_url",
+                image_url: {
+                  url: item.image_url.url,
+                  detail: item.detail, // This might need to be item.image_url.detail after the type change
+                },
+              };
+            }
           }) as ChatCompletionContentPart[],
         } satisfies ChatCompletionUserMessageParam;
       }

packages/providers/openai/tests/llm.test.ts

@@ -148,4 +148,71 @@ describe("Message Formatting", () => {
       expect(OpenAI.toOpenAIMessage(multiToolMessages)).toEqual(expectedOutput);
     });
   });
+  describe("Image Detail Formatting", () => {
+    test("OpenAI formats image messages with detail correctly", () => {
+      const inputMessages: ChatMessage[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "image_url",
+              image_url: {
+                url: "data:image/jpeg;base64,aGVsbG8=",
+                detail: "high",
+              },
+            },
+          ],
+        },
+      ];
+
+      const expectedOutput = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "image_url",
+              image_url: {
+                url: "data:image/jpeg;base64,aGVsbG8=",
+                detail: "high",
+              },
+            },
+          ],
+        },
+      ];
+
+      expect(OpenAI.toOpenAIMessage(inputMessages)).toEqual(expectedOutput);
+    });
+
+    test("OpenAI formats image messages without detail correctly", () => {
+      const inputMessages: ChatMessage[] = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "image_url",
+              image_url: {
+                url: "data:image/jpeg;base64,aGVsbG8=",
+              },
+            },
+          ],
+        },
+      ];
+
+      const expectedOutput = [
+        {
+          role: "user",
+          content: [
+            {
+              type: "image_url",
+              image_url: {
+                url: "data:image/jpeg;base64,aGVsbG8=",
+              },
+            },
+          ],
+        },
+      ];
+
+      expect(OpenAI.toOpenAIMessage(inputMessages)).toEqual(expectedOutput);
+    });
+  });
 });

packages/providers/openai/tests/structure-verify.js

@@ -0,0 +1,32 @@
+console.log("VERIFYING IMAGE DETAIL FIX\n");
+
+// Test the structure change
+console.log("OLD STRUCTURE (Broken):");
+const oldStructure = {
+  type: "image_url",
+  image_url: { url: "data:image/jpeg;base64,test" },
+  detail: "high", // WRONG: detail at top level
+};
+console.log(JSON.stringify(oldStructure, null, 2));
+console.log("Problem: 'detail' is at top level, OpenAI rejects this\n");
+
+console.log("NEW STRUCTURE (Fixed):");
+const newStructure = {
+  type: "image_url",
+  image_url: {
+    url: "data:image/jpeg;base64,test",
+    detail: "high", // CORRECT: detail inside image_url
+  },
+};
+console.log(JSON.stringify(newStructure, null, 2));
+console.log("Fixed: 'detail' is inside image_url object\n");
+
+console.log("WHAT YOU CHANGED:");
+console.log("In packages/core/src/llms/type.ts:");
+console.log("BEFORE: detail?: 'high' | 'low' | 'auto' (at top level)");
+console.log(
+  "AFTER:  image_url: { url: string; detail?: 'high' | 'low' | 'auto' }",
+);
+console.log("        ^-- detail moved inside image_url object");
+
+console.log("\n This matches OpenAI's API requirements!");