fix(sdk/python): stream all OpenRouter music requests and label audio by its real container

sdk/python/agentfield/agent.py

@@ -4189,7 +4189,9 @@ async def ai_generate_music(  # pragma: no cover - relies on external services
         Generate music from a text prompt.
 
         Routes to a music-capable provider (OpenRouter with models like
-        google/lyria-3-pro). Returns a MultimodalResponse with audio data.
+        google/lyria-3-pro-preview). Returns a MultimodalResponse with audio
+        data; ``.audio.format`` reports the container the model actually
+        produced, which may differ from the requested ``format``.
 
         Args:
             prompt (str): Text description of the music to generate.
@@ -4204,7 +4206,7 @@ async def ai_generate_music(  # pragma: no cover - relies on external services
             ```python
             result = await app.ai_generate_music("upbeat jazz piano solo")
             if result.has_audio:
-                result.audio.save("jazz.wav")
+                result.audio.save(f"jazz.{result.audio.format}")
             ```
         """
         return await self.ai_handler.ai_generate_music(

sdk/python/agentfield/agent_ai.py

@@ -2043,12 +2043,14 @@ async def ai_generate_music(
         Generate music from a text prompt.
 
         Routes to a music-capable media provider (currently OpenRouter with
-        models like google/lyria-3-pro). Returns a MultimodalResponse with
-        generated audio data (48kHz stereo).
+        models like google/lyria-3-pro-preview). Returns a MultimodalResponse
+        whose ``.audio.format`` reports the container the model actually
+        produced — today google/lyria-3-* always returns MP3 (44.1 kHz stereo)
+        whatever ``format`` is requested.
 
         Args:
             prompt: Text description of the music to generate
-            model: Music model to use (defaults to "google/lyria-3-pro")
+            model: Music model to use (defaults to "google/lyria-3-pro-preview")
             duration: Duration hint in seconds
             **kwargs: Provider-specific parameters (e.g., format="wav")
 
@@ -2058,7 +2060,7 @@ async def ai_generate_music(
         Examples:
             result = await app.ai_generate_music("upbeat jazz piano solo")
             if result.has_audio:
-                result.audio.save("jazz.wav")
+                result.audio.save(f"jazz.{result.audio.format}")
 
             result = await app.ai_generate_music(
                 "calm ambient electronic music",

sdk/python/agentfield/media_providers.py

@@ -2122,6 +2122,102 @@ def _wrap_pcm16_as_wav_b64(pcm_b64: str, *, sample_rate: int = 24000) -> str:
     return base64.b64encode(wav).decode("ascii")
 
 
+# Second byte of an MPEG audio frame header (0xFF sync + MPEG1/2/2.5 Layer III).
+_MPEG_SYNC_SECOND_BYTES = frozenset({0xFA, 0xFB, 0xF2, 0xF3})
+
+# Containers that legitimately satisfy a requested format: Opus is carried in
+# an Ogg container, and "pcm" is the same raw frames as "pcm16". Anything not
+# listed is satisfied only by itself.
+_AUDIO_FORMAT_ALIASES = {
+    "opus": frozenset({"opus", "ogg"}),
+    "pcm": frozenset({"pcm", "pcm16"}),
+    "pcm16": frozenset({"pcm16", "pcm"}),
+}
+
+
+def _sniff_audio_container(data: bytes) -> Optional[str]:
+    """Identify an audio container from the leading magic bytes of ``data``.
+
+    Returns ``"mp3"``, ``"wav"``, ``"flac"`` or ``"ogg"`` when ``data`` starts
+    with that container's signature, and ``None`` when nothing matches — which,
+    for OpenRouter's chat-completions audio modality, means the payload is raw
+    little-endian PCM16 frames carrying no container at all.
+    """
+    if len(data) < 4:
+        return None
+    if data[:3] == b"ID3":
+        return "mp3"
+    if data[0] == 0xFF and data[1] in _MPEG_SYNC_SECOND_BYTES:
+        return "mp3"
+    if data[:4] == b"RIFF" and data[8:12] == b"WAVE":
+        return "wav"
+    if data[:4] == b"fLaC":
+        return "flac"
+    if data[:4] == b"OggS":
+        return "ogg"
+    return None
+
+
+def _label_streamed_audio(
+    b64_data: str, requested_format: str, *, model: str = ""
+) -> tuple[str, str]:
+    """Label streamed OpenRouter audio by what the bytes actually are.
+
+    OpenRouter's music models ignore the requested ``audio.format`` and answer
+    with whatever container the upstream model produces (``google/lyria-3-*``
+    currently always returns MP3). Trusting the requested format therefore
+    mislabels — and, when it triggers a client-side RIFF wrap, corrupts — the
+    result, so the decoded bytes are sniffed instead:
+
+    * a recognised container wins: the payload is returned untouched and
+      labelled with the container that was found;
+    * no recognised container means raw PCM16 frames, which are wrapped in a
+      24 kHz mono RIFF/WAVE header when the caller asked for ``wav`` and passed
+      through labelled ``pcm16`` otherwise.
+
+    A single warning is logged whenever the container actually present differs
+    from the one the caller asked for.
+
+    Returns ``(b64_data, format)``.
+    """
+    import base64
+
+    if not b64_data:
+        return b64_data, requested_format
+
+    try:
+        raw = base64.b64decode(b64_data)
+    except ValueError:
+        # Undecodable payload — nothing to sniff, hand it back as requested.
+        return b64_data, requested_format
+
+    detected = _sniff_audio_container(raw)
+    container = detected or "pcm16"
+
+    if detected is not None:
+        result_format = detected
+    elif requested_format == "wav":
+        b64_data = _wrap_pcm16_as_wav_b64(b64_data, sample_rate=24000)
+        result_format = "wav"
+    else:
+        # No container at all: these are raw PCM16 frames whatever was asked
+        # for. Say so rather than stamping them with a label that is untrue.
+        result_format = "pcm16"
+
+    wrapped_raw_pcm = detected is None and requested_format == "wav"
+    satisfied = _AUDIO_FORMAT_ALIASES.get(requested_format, {requested_format})
+    if container not in satisfied and not wrapped_raw_pcm:
+        from agentfield.logger import log_warn
+
+        log_warn(
+            f"OpenRouter returned {container} audio for "
+            f"{model or 'the requested model'} but format={requested_format!r} "
+            f"was requested; labelling the result as {result_format!r}."
+        )
+
+    return b64_data, result_format
+
+
 class OpenRouterProvider(MediaProvider):
     """
     OpenRouter provider for image generation via chat completions.
@@ -2667,87 +2763,6 @@ async def _stream_openrouter_audio(
         transcript = "".join(transcript_parts)
         return b64_full, transcript
 
-    async def _nonstream_openrouter_audio(
-        self,
-        payload: Dict[str, Any],
-        headers: Dict[str, str],
-        *,
-        timeout: float = 300.0,
-        label: str = "audio",
-    ) -> tuple:
-        """
-        Non-streaming helper for OpenRouter audio (chat-completions) requests.
-
-        Used when the requested wire format is *not* ``pcm16``, because the
-        OpenRouter SSE streaming endpoint only emits ``pcm16`` audio deltas
-        and will reject other formats (mp3 / flac / opus) with a 400.
-
-        The non-streaming response is a single JSON document whose shape
-        mirrors the last SSE event::
-
-            {"choices": [{"message": {"audio": {"data": "...",
-                                                  "transcript": "..."}}]}]}
-
-        Args:
-            payload: JSON body for the chat completions request (``stream``
-                must already be set to ``False`` by the caller).
-            headers: HTTP headers including Authorization
-            timeout: Total request timeout in seconds (default 300)
-            label: Label for error messages ("audio" or "music")
-
-        Returns:
-            Tuple of ``(b64_data: str, transcript: str)``
-        """
-        import json as json_mod
-
-        import aiohttp
-
-        client_timeout = aiohttp.ClientTimeout(total=timeout)
-
-        async with aiohttp.ClientSession(timeout=client_timeout) as session:
-            async with session.post(
-                "https://openrouter.ai/api/v1/chat/completions",
-                json=payload,
-                headers=headers,
-            ) as resp:
-                if resp.status != 200:
-                    body = await resp.text()
-                    raise RuntimeError(
-                        f"OpenRouter {label} request failed ({resp.status}): "
-                        f"{body[:500]}"
-                    )
-                raw = await resp.read()
-
-        try:
-            doc = json_mod.loads(raw.decode("utf-8"))
-        except json_mod.JSONDecodeError as exc:
-            raise RuntimeError(
-                f"OpenRouter {label} non-stream response was not valid JSON: "
-                f"{exc}"
-            ) from exc
-
-        choices = doc.get("choices") or []
-        if not choices:
-            return "", ""
-
-        # OpenRouter non-stream response places the final audio+transcript
-        # under choices[0].message.audio — mirroring the SSE delta schema but
-        # using the full "message" instead of a "delta".
-        message = choices[0].get("message") or {}
-        audio = message.get("audio") or {}
-        b64_data = audio.get("data") or ""
-        transcript = audio.get("transcript") or ""
-
-        if b64_data:
-            total_size = len(b64_data)
-            if total_size > MAX_AUDIO_B64_BYTES:
-                raise RuntimeError(
-                    f"Audio base64 data exceeded "
-                    f"{MAX_AUDIO_B64_BYTES} byte limit"
-                )
-
-        return b64_data, transcript
-
     async def generate_audio(
         self,
         text: str,
@@ -2780,9 +2795,11 @@ async def generate_audio(
                 "hexgrad/kokoro-82m"). Default: ``hexgrad/kokoro-82m``.
             voice: Voice identifier (model-specific — e.g. ``alloy`` for
                 OpenAI, ``af_bella`` for Kokoro)
-            format: Audio format (wav, mp3, flac, opus, pcm16). ``wav`` is
-                synthesized client-side when the upstream endpoint only emits
-                pcm.
+            format: Audio format. ``/audio/speech`` models accept
+                wav / mp3 / flac / opus / aac; chat-audio models
+                (``openai/gpt-audio*``) can only deliver ``pcm16``, so only
+                ``pcm16`` and ``wav`` (wrapped client-side from pcm16) are
+                accepted there — any other value raises ``ValueError``.
             speed: Optional speech speed for ``/audio/speech`` models.
             extra: Optional extra request fields for ``/audio/speech`` models.
             system: Optional system instructions for chat-completions audio
@@ -2791,6 +2808,11 @@ async def generate_audio(
 
         Returns:
             MultimodalResponse with generated audio
+
+        Raises:
+            ValueError: if a chat-audio model is asked for a container other
+                than ``pcm16``/``wav`` (raised before the generation request
+                is sent).
         """
         import os
 
@@ -2854,32 +2876,34 @@ async def generate_audio(
             )
 
         # Chat-completions audio modality path (gpt-audio family).
-        # Streaming on the OpenAI provider only emits pcm16 — fall back to
-        # pcm16 over the wire and re-wrap to user's requested format below.
-        wire_format = "pcm16" if audio_format == "wav" else audio_format
+        # OpenRouter's gateway rejects stream=false for *any* audio-output
+        # request, and the streaming transport only ever emits pcm16 deltas —
+        # so pcm16 is the only container this route can deliver. Refuse the
+        # impossible formats here, before a request is spent on a 400. (The
+        # /audio/speech route above is unaffected and still serves
+        # mp3/flac/opus/aac.)
+        if audio_format not in ("wav", "pcm16"):
+            raise ValueError(
+                f"format={audio_format!r} is not available from OpenRouter "
+                f"chat-audio models: the audio modality delivers pcm16 only. "
+                f"Use format='pcm16', or format='wav' to have the pcm16 "
+                f"stream wrapped in a RIFF container client-side."
+            )
+
+        wire_format = "pcm16"
         messages = [{"role": "user", "content": text}]
         if system is not None:
             messages.insert(0, {"role": "system", "content": system})
-        # OpenRouter SSE streaming only supports pcm16 wire format — any
-        # other format (mp3 / flac / opus) must use the non-streaming JSON
-        # response path, otherwise OpenRouter rejects the request with a
-        # format-related error (see: Issue #584).
-        use_stream = wire_format == "pcm16"
         payload = {
             "model": send_model,
             "messages": messages,
             "modalities": ["text", "audio"],
             "audio": {"voice": voice, "format": wire_format},
-            "stream": use_stream,
+            "stream": True,
         }
-        if use_stream:
-            b64_full, transcript = await self._stream_openrouter_audio(
-                payload, headers, timeout=timeout, label="audio"
-            )
-        else:
-            b64_full, transcript = await self._nonstream_openrouter_audio(
-                payload, headers, timeout=timeout, label="audio"
-            )
+        b64_full, transcript = await self._stream_openrouter_audio(
+            payload, headers, timeout=timeout, label="audio"
+        )
 
         # Re-wrap pcm16 -> wav if user asked for wav.
         if audio_format == "wav" and b64_full:
@@ -2969,17 +2993,31 @@ async def generate_music(
         """
         Generate music via OpenRouter using a music-capable model.
 
-        Uses SSE streaming chat completions with audio modality, similar to
-        generate_audio but targeting music generation models.
+        Always uses SSE streaming chat completions with the audio modality:
+        OpenRouter's music models reject a non-streaming audio request with
+        ``Audio output requires stream: true``, so the transport does not vary
+        with the requested format.
+
+        The requested ``format`` is forwarded as ``audio.format``, but current
+        music models ignore it — ``google/lyria-3-*`` answers with MP3 whatever
+        is asked for. The returned container is therefore detected from the
+        bytes and reported on ``response.audio.format``, and a mismatch with
+        the requested format is logged once. A payload with no container header
+        is treated as raw PCM16 and wrapped in a 24 kHz mono WAV container when
+        ``wav`` was requested.
 
         Args:
             prompt: Text description of the music to generate
-            model: Music model (defaults to "google/lyria-3-pro")
+            model: Music model (defaults to "google/lyria-3-pro-preview")
             duration: Duration hint in seconds (must be >0 and <=600)
-            **kwargs: Additional parameters (timeout overrides default 300s)
+            **kwargs: Additional parameters (``format`` defaults to ``wav``;
+                ``timeout`` overrides the default 300s)
 
         Returns:
-            MultimodalResponse with generated audio (48kHz stereo)
+            MultimodalResponse with generated audio. The container and sample
+            rate are whatever the model produced — ``google/lyria-3-pro-preview``
+            currently returns MP3 at 44.1 kHz stereo — and the detected
+            container is reported on ``response.audio.format``.
         """
         import os
 
@@ -2989,7 +3027,7 @@ async def generate_music(
                 "OpenRouter API key required. Set OPENROUTER_API_KEY env var or pass api_key."
             )
 
-        send_model = model or "google/lyria-3-pro"
+        send_model = model or "google/lyria-3-pro-preview"
         if send_model.startswith("openrouter/"):
             send_model = send_model[len("openrouter/") :]
 
@@ -3006,6 +3044,11 @@ async def generate_music(
         audio_format = kwargs.pop("format", "wav")
         timeout = kwargs.pop("timeout", 300.0)
 
+        # Music models require stream=true for *any* audio output — a
+        # non-streaming request is rejected with "Audio output requires
+        # stream: true" regardless of audio.format. The requested format is
+        # forwarded unchanged (models ignore it today); the container that
+        # actually comes back is detected from the bytes below.
         payload = {
             "model": send_model,
             "messages": [{"role": "user", "content": user_content}],
@@ -3024,9 +3067,13 @@ async def generate_music(
             payload, headers, timeout=timeout, label="music"
         )
 
+        b64_full, result_format = _label_streamed_audio(
+            b64_full, audio_format, model=send_model
+        )
+
         audio_output = AudioOutput(
             data=b64_full if b64_full else None,
-            format=audio_format,
+            format=result_format,
             url=None,
         )