mmc/lingma-openai-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden responses stream termination

Browse Source 
Ensure /v1/responses streaming always emits completion frames on upstream EOF, errors, and cancellation, and add targeted diagnostics for interrupted Lingma streams.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
mmc
2026-04-20 14:55:32 +08:00
parent 12a4d9584e
commit 5e6c1c1a63
3 changed files with 139 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
app/main.py
View File

@@ -859,10 +859,21 @@ async def v1_chat_completions(req: ChatCompletionsRequest, request: Request):
yield "data: [DONE]\n\n"
success = True
except asyncio.CancelledError:
logger.info("chat.stream cancelled by client (inst=%s)", _inst.name)
logger.info(
"chat.stream cancelled by client (inst=%s, session_id=%s)",
_inst.name,
cached_session_id,
)
raise
except Exception as exc:
logger.warning("chat.stream error (inst=%s): %s", _inst.name, exc)
logger.warning(
"chat.stream error (inst=%s, session_id=%s, prompt_tokens=%s, completion_tokens=%s): %s",
_inst.name,
cached_session_id,
prompt_tokens,
completion_tokens_holder["n"],
exc,
)
finally:
# Persist upstream sessionId only on a clean chat/finish.
# Partial streams (cancelled, timed out) leave Lingma's
@@ -1157,6 +1168,23 @@ async def _responses_stream_from_chat_stream(
):
created_at = int(time.time())
usage: dict[str, int] = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
completed_sent = False
def _completed_frame() -> str:
return _sse_data(
{
"type": "response.completed",
"response": {
"id": response_id,
"object": "response",
"created_at": created_at,
"status": "completed",
"model": model,
"usage": usage,
},
}
)
yield _sse_data(
{
"type": "response.created",
@@ -1170,86 +1198,77 @@ async def _responses_stream_from_chat_stream(
}
)
async for part in chat_stream.body_iterator:
chunk = part.decode("utf-8") if isinstance(part, bytes) else str(part)
for frame in chunk.split("\n\n"):
frame = frame.strip()
if not frame or not frame.startswith("data:"):
continue
body = frame[len("data:") :].strip()
if body == "[DONE]":
yield _sse_data(
{
"type": "response.completed",
"response": {
"id": response_id,
"object": "response",
"created_at": created_at,
"status": "completed",
"model": model,
"usage": usage,
},
}
)
yield "data: [DONE]\\n\\n"
return
try:
async for part in chat_stream.body_iterator:
chunk = part.decode("utf-8") if isinstance(part, bytes) else str(part)
for frame in chunk.split("\n\n"):
frame = frame.strip()
if not frame or not frame.startswith("data:"):
continue
body = frame[len("data:") :].strip()
if body == "[DONE]":
yield _completed_frame()
yield "data: [DONE]\n\n"
completed_sent = True
return
try:
payload = json.loads(body)
except Exception:
continue
try:
payload = json.loads(body)
except Exception:
continue
frame_usage = _responses_usage_from_chat(payload.get("usage"))
if any(frame_usage.values()):
usage = frame_usage
frame_usage = _responses_usage_from_chat(payload.get("usage"))
if any(frame_usage.values()):
usage = frame_usage
choices = payload.get("choices")
if not isinstance(choices, list) or not choices:
continue
choice = choices[0] if isinstance(choices[0], dict) else {}
delta = choice.get("delta") if isinstance(choice.get("delta"), dict) else {}
choices = payload.get("choices")
if not isinstance(choices, list) or not choices:
continue
choice = choices[0] if isinstance(choices[0], dict) else {}
delta = choice.get("delta") if isinstance(choice.get("delta"), dict) else {}
text = delta.get("content")
if isinstance(text, str) and text:
yield _sse_data(
{
"type": "response.output_text.delta",
"response_id": response_id,
"delta": text,
}
)
tool_calls = delta.get("tool_calls")
if isinstance(tool_calls, list):
for idx, tool_call in enumerate(tool_calls):
if not isinstance(tool_call, dict):
continue
fn = tool_call.get("function") if isinstance(tool_call.get("function"), dict) else {}
call_id = str(tool_call.get("id") or f"call_{idx}")
text = delta.get("content")
if isinstance(text, str) and text:
yield _sse_data(
{
"type": "response.function_call.delta",
"type": "response.output_text.delta",
"response_id": response_id,
"item_id": call_id,
"name": str(fn.get("name") or "tool"),
"arguments": str(fn.get("arguments") or "{}"),
"delta": text,
}
)
yield _sse_data(
{
"type": "response.completed",
"response": {
"id": response_id,
"object": "response",
"created_at": created_at,
"status": "completed",
"model": model,
"usage": usage,
},
}
)
yield "data: [DONE]\\n\\n"
tool_calls = delta.get("tool_calls")
if isinstance(tool_calls, list):
for idx, tool_call in enumerate(tool_calls):
if not isinstance(tool_call, dict):
continue
fn = tool_call.get("function") if isinstance(tool_call.get("function"), dict) else {}
call_id = str(tool_call.get("id") or f"call_{idx}")
yield _sse_data(
{
"type": "response.function_call.delta",
"response_id": response_id,
"item_id": call_id,
"name": str(fn.get("name") or "tool"),
"arguments": str(fn.get("arguments") or "{}"),
}
)
except asyncio.CancelledError:
if not completed_sent:
yield _completed_frame()
yield "data: [DONE]\n\n"
completed_sent = True
return
except Exception:
if not completed_sent:
yield _completed_frame()
yield "data: [DONE]\n\n"
completed_sent = True
return
if not completed_sent:
yield _completed_frame()
yield "data: [DONE]\n\n"