mmc/lingma-openai-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: bridge Lingma tool events to OpenAI/Anthropic responses

Browse Source 
Add structured tool event propagation from Lingma stream/finish metadata and map it to OpenAI tool_calls and Anthropic tool_use/tool_result in both streaming and non-streaming responses. Add focused bridge tests and update docs/design notes to match current behavior.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
GitHub Actions
2026-04-18 22:34:43 +08:00
parent b3fd8800f7
commit 1c7b86e2c0
6 changed files with 668 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
app/lingma_client.py
View File

@@ -9,7 +9,7 @@ import subprocess
import time
import uuid
from pathlib import Path
from typing import AsyncIterator, Callable, Optional
from typing import Any, AsyncIterator, Callable, Optional
import websockets
@@ -103,6 +103,58 @@ class LspWsRpcClient:
self._on_disconnect = on_disconnect
self._closed = False
@staticmethod
def _extract_tool_event(params: dict[str, Any]) -> dict[str, Any] | None:
candidates: list[dict[str, Any]] = []
if isinstance(params.get("toolCall"), dict):
candidates.append(params["toolCall"])
if isinstance(params.get("tool_call"), dict):
candidates.append(params["tool_call"])
if isinstance(params.get("tool"), dict):
candidates.append(params["tool"])
data = params.get("data")
if isinstance(data, dict):
if isinstance(data.get("toolCall"), dict):
candidates.append(data["toolCall"])
if isinstance(data.get("tool_call"), dict):
candidates.append(data["tool_call"])
if isinstance(data.get("tool"), dict):
candidates.append(data["tool"])
if not candidates:
return None
raw = candidates[0]
tool_id = (
raw.get("toolCallId")
or raw.get("tool_call_id")
or raw.get("id")
or params.get("toolCallId")
or params.get("tool_call_id")
)
name = raw.get("name") or raw.get("toolName") or raw.get("tool_name")
call_input = raw.get("input")
if call_input is None:
call_input = raw.get("arguments")
if call_input is None:
call_input = raw.get("args")
result_payload = raw.get("result")
if result_payload is None:
result_payload = params.get("result")
if result_payload is None and isinstance(data, dict):
result_payload = data.get("result")
if not tool_id:
return None
return {
"id": str(tool_id),
"name": str(name or "tool"),
"input": call_input if call_input is not None else {},
"result": result_payload,
}
async def start(self):
self._reader_task = asyncio.create_task(self._reader_loop())
@@ -185,7 +237,16 @@ class LspWsRpcClient:
stream["parts"].append(text)
if stream["first_chunk_at"] is None:
stream["first_chunk_at"] = time.monotonic()
stream["chunks"].put_nowait(text)
stream["chunks"].put_nowait({"type": "text", "text": text})
if method in {"tool/call/sync", "tool/invoke", "tool/call/approve"}:
req_id = params.get("requestId")
stream = self._chat_streams.get(req_id)
if stream is not None:
tool_event = self._extract_tool_event(params)
if tool_event is not None:
stream["tool_events"].append(tool_event)
stream["chunks"].put_nowait({"type": "tool", "tool": tool_event})
if method == "chat/finish":
req_id = params.get("requestId")
@@ -224,6 +285,7 @@ class LspWsRpcClient:
"chunks": asyncio.Queue(),
"done": asyncio.Event(),
"finish": None,
"tool_events": [],
"started_at": time.monotonic(),
"first_chunk_at": None,
"finish_at": None,
@@ -239,7 +301,7 @@ class LspWsRpcClient:
with contextlib.suppress(Exception):
stream["chunks"].put_nowait(None)
async def consume_stream(self, request_id: str, timeout: float) -> AsyncIterator[str]:
async def consume_stream(self, request_id: str, timeout: float) -> AsyncIterator[dict[str, Any]]:
stream = self._chat_streams.get(request_id)
if stream is None:
return
@@ -266,6 +328,7 @@ class LspWsRpcClient:
"finish": stream.get("finish") or {},
"firstTokenLatencyMs": first_ms,
"totalLatencyMs": total_ms,
"toolEvents": stream.get("tool_events") or [],
}
@@ -722,8 +785,12 @@ class LingmaGatewayClient:
session_id: str | None = None,
is_reply: bool = False,
out_meta: dict | None = None,
) -> AsyncIterator[str]:
"""Stream `chat/answer` chunks.
) -> AsyncIterator[dict[str, Any]]:
"""Stream chat events.
Yields structured events:
* {"type": "text", "text": "..."}
* {"type": "tool", "tool": {...}}
If `out_meta` is provided, the final `chat/finish` payload's sessionId
(and the raw finish dict) is written into it when the stream ends or is
@@ -739,10 +806,10 @@ class LingmaGatewayClient:
self.rpc.create_stream(request_id)
try:
await self._kick_chat_ask(payload)
async for chunk in self.rpc.consume_stream(
async for event in self.rpc.consume_stream(
request_id, timeout=max(60.0, self.rpc_timeout + 60.0)
):
yield chunk
yield event
finally:
# Runs on normal completion, exception, or consumer GeneratorExit (client disconnect).
if out_meta is not None:
@@ -753,6 +820,7 @@ class LingmaGatewayClient:
out_meta["finish"] = finish
out_meta["request_id"] = request_id
out_meta["chars"] = len(stream_result.get("text") or "")
out_meta["tool_events"] = stream_result.get("toolEvents") or []
except Exception:
pass
self.rpc.pop_stream(request_id)

224
app/main.py
View File

@@ -6,6 +6,7 @@ import json
import time
import uuid
from contextlib import asynccontextmanager
from typing import Any
from fastapi import Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
@@ -350,6 +351,78 @@ def _include_usage(stream_options: dict | None) -> bool:
return bool(stream_options.get("include_usage"))
def _stream_event_type(event: Any) -> str:
if isinstance(event, dict):
t = event.get("type")
if t in {"text", "tool"}:
return t
return "text"
def _stream_text(event: Any) -> str:
if isinstance(event, dict):
if event.get("type") == "text":
text = event.get("text")
if isinstance(text, str):
return text
return ""
if isinstance(event, str):
return event
return ""
def _stream_tool_event(event: Any) -> dict[str, Any] | None:
if isinstance(event, dict) and event.get("type") == "tool":
tool = event.get("tool")
if isinstance(tool, dict):
return tool
return None
def _json_string(value: Any) -> str:
if isinstance(value, str):
return value
try:
return json.dumps(value if value is not None else {}, ensure_ascii=False)
except Exception:
return "{}"
def _openai_tool_call(tool: dict[str, Any]) -> dict[str, Any]:
return {
"id": str(tool.get("id") or f"call_{uuid.uuid4().hex}"),
"type": "function",
"function": {
"name": str(tool.get("name") or "tool"),
"arguments": _json_string(tool.get("input")),
},
}
def _anthropic_tool_use_block(tool: dict[str, Any]) -> dict[str, Any]:
return {
"type": "tool_use",
"id": str(tool.get("id") or f"toolu_{uuid.uuid4().hex}"),
"name": str(tool.get("name") or "tool"),
"input": tool.get("input") if tool.get("input") is not None else {},
}
def _anthropic_tool_result_block(tool: dict[str, Any]) -> dict[str, Any] | None:
if "result" not in tool:
return None
result = tool.get("result")
if isinstance(result, str):
content: Any = result
else:
content = _json_string(result)
return {
"type": "tool_result",
"tool_use_id": str(tool.get("id") or ""),
"content": content,
}
@app.post("/v1/chat/completions", dependencies=[Depends(auth_guard)])
async def v1_chat_completions(req: ChatCompletionsRequest, request: Request):
p = _require_pool()
@@ -485,7 +558,37 @@ async def v1_chat_completions(req: ChatCompletionsRequest, request: Request):
is_reply=is_reply,
out_meta=_meta,
):
completion_tokens_holder["n"] += estimate_tokens(chunk)
if _stream_event_type(chunk) == "tool":
tool = _stream_tool_event(chunk)
if not tool:
continue
payload = {
"id": completion_id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{
"index": 0,
**_openai_tool_call(tool),
}
]
},
"finish_reason": None,
}
],
}
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
continue
text = _stream_text(chunk)
if not text:
continue
completion_tokens_holder["n"] += estimate_tokens(text)
payload = {
"id": completion_id,
"object": "chat.completion.chunk",
@@ -494,7 +597,7 @@ async def v1_chat_completions(req: ChatCompletionsRequest, request: Request):
"choices": [
{
"index": 0,
"delta": {"content": chunk},
"delta": {"content": text},
"finish_reason": None,
}
],
@@ -596,6 +699,13 @@ async def v1_chat_completions(req: ChatCompletionsRequest, request: Request):
sid = result.get("sessionId")
if sid:
await session_cache.put(write_key, sid, inst.name)
tool_events = result.get("toolEvents") or []
message_content = result.get("text") or ""
tool_calls: list[dict[str, Any]] = []
if isinstance(tool_events, list):
for item in tool_events:
if isinstance(item, dict):
tool_calls.append(_openai_tool_call(item))
response = ChatCompletionResponse(
id=f"chatcmpl-{uuid.uuid4().hex}",
created=int(time.time()),
@@ -604,10 +714,15 @@ async def v1_chat_completions(req: ChatCompletionsRequest, request: Request):
ChatCompletionChoice(
index=0,
finish_reason="stop",
message={"role": "assistant", "content": result.get("text") or ""},
message={
"role": "assistant",
"content": message_content,
"tool_calls": tool_calls or None,
},
)
],
)
data = response.model_dump()
data["latency"] = {
"first_token_ms": result.get("firstTokenLatencyMs"),
@@ -810,6 +925,8 @@ async def v1_messages(req: AnthropicMessagesRequest, request: Request):
async def event_stream(_ticket=ticket, _inst=inst, _meta=stream_meta):
success = False
block_index = 0
text_block_open = False
try:
# 1) message_start — Anthropic SDKs read this first to get
# the message envelope (id/model/initial usage).
@@ -833,17 +950,6 @@ async def v1_messages(req: AnthropicMessagesRequest, request: Request):
}
yield _sse("message_start", start_payload)
# 2) content_block_start for a single text block (index 0).
yield _sse(
"content_block_start",
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "text", "text": ""},
},
)
# 3) content_block_delta stream of text tokens.
async for chunk in _inst.client.chat_stream(
prompt,
model,
@@ -852,23 +958,80 @@ async def v1_messages(req: AnthropicMessagesRequest, request: Request):
is_reply=is_reply,
out_meta=_meta,
):
if not chunk:
if _stream_event_type(chunk) == "tool":
if text_block_open:
yield _sse(
"content_block_stop",
{"type": "content_block_stop", "index": block_index},
)
block_index += 1
text_block_open = False
tool = _stream_tool_event(chunk)
if not tool:
continue
tool_use_block = _anthropic_tool_use_block(tool)
yield _sse(
"content_block_start",
{
"type": "content_block_start",
"index": block_index,
"content_block": tool_use_block,
},
)
yield _sse(
"content_block_stop",
{"type": "content_block_stop", "index": block_index},
)
block_index += 1
tool_result_block = _anthropic_tool_result_block(tool)
if tool_result_block is not None:
yield _sse(
"content_block_start",
{
"type": "content_block_start",
"index": block_index,
"content_block": tool_result_block,
},
)
yield _sse(
"content_block_stop",
{"type": "content_block_stop", "index": block_index},
)
block_index += 1
continue
completion_tokens_holder["n"] += estimate_tokens(chunk)
text = _stream_text(chunk)
if not text:
continue
completion_tokens_holder["n"] += estimate_tokens(text)
if not text_block_open:
yield _sse(
"content_block_start",
{
"type": "content_block_start",
"index": block_index,
"content_block": {"type": "text", "text": ""},
},
)
text_block_open = True
yield _sse(
"content_block_delta",
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": chunk},
"index": block_index,
"delta": {"type": "text_delta", "text": text},
},
)
# 4) content_block_stop closes the single text block.
yield _sse(
"content_block_stop",
{"type": "content_block_stop", "index": 0},
)
if text_block_open:
yield _sse(
"content_block_stop",
{"type": "content_block_stop", "index": block_index},
)
# 5) message_delta carries the terminal stop_reason and
# the final cumulative output_tokens count.
@@ -972,12 +1135,25 @@ async def v1_messages(req: AnthropicMessagesRequest, request: Request):
if sid:
await session_cache.put(write_key, sid, inst.name)
content_blocks: list[dict[str, Any]] = []
if text:
content_blocks.append({"type": "text", "text": text})
tool_events = result.get("toolEvents") or []
if isinstance(tool_events, list):
for item in tool_events:
if not isinstance(item, dict):
continue
content_blocks.append(_anthropic_tool_use_block(item))
tool_result = _anthropic_tool_result_block(item)
if tool_result is not None:
content_blocks.append(tool_result)
response_body: dict = {
"id": message_id,
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": text}],
"content": content_blocks,
"stop_reason": _anthropic_stop_reason(completion_tokens, req.max_tokens),
"stop_sequence": None,
"usage": {