fix(ai): accept url image results from OpenAI-style image APIs (#59474)

Author: rekram1-node

packages/ai/src/protocols/openai-images.ts

@@ -13,7 +13,7 @@ import { Framing } from "../route/framing.js"
 import { MediaProtocol } from "../route/media-protocol.js"
 import { MediaRoute } from "../route/media.js"
 import { AIError, mergeJsonRecords, ProviderID, type MediaUsage, type OpenString } from "../schema/index.js"
-import { ProviderShared } from "./shared.js"
+import { optionalNull, ProviderShared } from "./shared.js"
 import { ImageSizing } from "./utils/image-sizing.js"
 import { MediaInput } from "./utils/media-input.js"
 
@@ -64,7 +64,8 @@ const Settings = {
 }
 
 const OpenAIImageResponse = Schema.Struct({
-  data: Schema.Array(Schema.Struct({ b64_json: Schema.String })),
+  // `response_format` picks `b64_json` or `url`; DALL·E and many compatible hosts default to `url`.
+  data: Schema.Array(Schema.Struct({ b64_json: optionalNull(Schema.String), url: optionalNull(Schema.String) })),
   ...Settings,
   usage: Schema.optional(Usage),
 })
@@ -307,7 +308,7 @@ const onDocument = Effect.fnUntraced(function* (frame: Exclude<Frame, string>) {
   const format = decoded.output_format
   const mediaType = format === undefined ? undefined : `image/${format}`
   const images = yield* Effect.forEach(decoded.data, (item, index) =>
-    MediaInput.decodedAsset(invalid, `${route.name} result ${index}`, item.b64_json, mediaType, {
+    MediaInput.imageOutput(invalid, `${route.name} result ${index}`, item, mediaType, {
       info: info(format, decoded.size),
     }),
   )

packages/ai/src/protocols/utils/media-input.ts

@@ -53,7 +53,10 @@ export const decodedAsset = (
     Effect.map((bytes) => Media.bytes(bytes, mediaType, options)),
   )
 
-/** One image of an OpenAI-shaped `data` array, which carries either `b64_json` or a `url`. */
+/**
+ * One image of an OpenAI-shaped `data` array, which carries `b64_json`, a `url`, or both. `b64_json` wins; a `data:`
+ * URL decodes inline with its own media type, and any other URL stays a URL asset for the caller to download.
+ */
 export const imageOutput = (
   invalid: (message: string, cause?: unknown) => AIError,
   label: string,
@@ -62,6 +65,10 @@ export const imageOutput = (
   options?: Media.AssetOptions,
 ) => {
   if (item.b64_json) return decodedAsset(invalid, label, item.b64_json, mediaType, options)
+  if (item.url?.startsWith("data:")) {
+    const asset = Media.parseDataUrl(item.url, options)
+    return asset === undefined ? Effect.fail(invalid(`${label} has an invalid data URL`)) : Effect.succeed(asset)
+  }
   if (item.url) return Effect.succeed(Media.url(item.url, { ...options, mediaType }))
   return Effect.fail(invalid(`${label} has neither image data nor a URL`))
 }

packages/ai/test/image.test.ts

@@ -169,6 +169,48 @@ describe("Image", () => {
     }),
   )
 
+  describe("OpenAI-style url results", () => {
+    const generate = (data: unknown) =>
+      Image.generate({
+        model: OpenAI.configure({ apiKey: "test", baseURL: "https://openai.test/v1" }).image("dall-e-3"),
+        prompt: "A lighthouse",
+      }).pipe(Effect.provide(layer((input) => Effect.succeed(json(input, { data })))))
+
+    it.effect("keeps an http url as a URL asset", () =>
+      Effect.gen(function* () {
+        const response = yield* generate([{ url: "https://cdn.image.test/result.png" }])
+        expect(response.image.source).toEqual({ type: "url", url: "https://cdn.image.test/result.png" })
+      }),
+    )
+
+    it.effect("decodes a data url inline with its media type", () =>
+      Effect.gen(function* () {
+        const response = yield* generate([{ url: "data:image/png;base64,iVBORw==" }])
+        expect(response.image.source).toEqual({ type: "base64", data: "iVBORw==", mediaType: "image/png" })
+        expect(response.image.mediaType).toBe("image/png")
+      }),
+    )
+
+    it.effect("prefers b64_json when url is null", () =>
+      Effect.gen(function* () {
+        const response = yield* generate([{ b64_json: "/9j/4AAQ", url: null }])
+        expect(response.image.source).toEqual({
+          type: "bytes",
+          data: Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]),
+          mediaType: "image/jpeg",
+        })
+      }),
+    )
+
+    it.effect("rejects an item with neither b64_json nor url", () =>
+      Effect.gen(function* () {
+        const error = yield* generate([{}]).pipe(Effect.flip)
+        expect(error.reason._tag).toBe("InvalidProviderOutput")
+        expect(error.message).toContain("has neither image data nor a URL")
+      }),
+    )
+  })
+
   it.effect("classifies an OpenAI stream moderation error event as a content policy rejection", () =>
     Effect.gen(function* () {
       const error = yield* Image.stream({