From 6257f4f6b8d0f3c19b44e2b228f9d868d3a6d540 Mon Sep 17 00:00:00 2001 From: DaiChaoXiong Date: Sun, 9 Aug 2026 22:04:11 +0800 Subject: [PATCH] test(providers): real-endpoint-log regressions for Responses reasoning Based on the user's actual deepseek-v4-flash debug log (Responses protocol): - real_log_captures_reasoning_and_tools: replays turn 1 of the real SSE (reasoning item + reasoning_text.delta + two function calls) and asserts the Done message is [Thinking, ToolCall, ToolCall] with usage mapped (input 850 / output 157 / cache-read 768) - agent_multi_turn_replays_reasoning: drives a 3-turn agent over the real protocol (turn 1 + turn 2 from the log, plus a terminating text turn), asserting thinking lands in the transcript and the second request's replay carries a reasoning item with the captured text and matching call ids MockTransport gains requests() to inspect all recorded requests. --- crates/focus-providers/tests/common/mod.rs | 7 + crates/focus-providers/tests/openai_tests.rs | 219 +++++++++++++++++++ 2 files changed, 226 insertions(+) diff --git a/crates/focus-providers/tests/common/mod.rs b/crates/focus-providers/tests/common/mod.rs index 1523ad8..4cedcf7 100644 --- a/crates/focus-providers/tests/common/mod.rs +++ b/crates/focus-providers/tests/common/mod.rs @@ -61,6 +61,13 @@ impl MockTransport { .expect("no request recorded") } + /// 已记录的全部请求克隆（按顺序）。 + /// Clones of all recorded requests (in order). + #[allow(dead_code)] + pub fn requests(&self) -> Vec { + self.requests.lock().unwrap().clone() + } + /// 已记录的请求数量。 /// Number of recorded requests. #[allow(dead_code)] diff --git a/crates/focus-providers/tests/openai_tests.rs b/crates/focus-providers/tests/openai_tests.rs index 142cc86..0a6df6c 100644 --- a/crates/focus-providers/tests/openai_tests.rs +++ b/crates/focus-providers/tests/openai_tests.rs @@ -618,3 +618,222 @@ fn responses_replays_reasoning_item() { assert_eq!(content[0].get_str("type"), Some("reasoning_text")); assert_eq!(content[0].get_str("text"), Some("思考内容")); } + +// ---- 基于真实端点日志的回归测试 ---- +// ---- regressions built from a real endpoint log (deepseek-v4-flash, +// Responses API with reasoning items) -------------------------------- + +/// 真实日志第 1 轮的 SSE（reasoning + 两个 shell 调用）。 +/// Turn 1 of the real log: reasoning + two shell calls. +const REAL_TURN1: &str = "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"r1\",\"status\":\"in_progress\"}}\n\n\ +event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"reasoning\",\"id\":\"rr1\",\"status\":\"in_progress\",\"content\":[],\"summary\":[]},\"output_index\":0}\n\n\ +event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"rr1\",\"output_index\":0,\"part\":{\"type\":\"reasoning_text\",\"text\":\"\"}}\n\n\ +event: response.reasoning_text.delta\ndata: {\"type\":\"response.reasoning_text.delta\",\"content_index\":0,\"delta\":\"The user is asking in Chinese: \\\"Hello, what does this project do?\\\" Let me explore\",\"item_id\":\"rr1\",\"output_index\":0}\n\n\ +event: response.reasoning_text.delta\ndata: {\"type\":\"response.reasoning_text.delta\",\"content_index\":0,\"delta\":\" the repository to understand what it is.\",\"item_id\":\"rr1\",\"output_index\":0}\n\n\ +event: response.reasoning_text.done\ndata: {\"type\":\"response.reasoning_text.done\",\"content_index\":0,\"item_id\":\"rr1\",\"output_index\":0,\"text\":\"The user is asking in Chinese: \\\"Hello, what does this project do?\\\" Let me explore the repository to understand what it is.\"}\n\n\ +event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\",\"id\":\"f1\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_00_wyQ\",\"name\":\"shell\"},\"output_index\":1}\n\n\ +event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"command\\\": \\\"ls -la /home/focus/Desktop/Project/focus && cat /home/focus/Desktop/Project/focus/README.md 2>/dev/null\",\"item_id\":\"f1\",\"output_index\":1}\n\n\ +event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" | head -50\\\"}\",\"item_id\":\"f1\",\"output_index\":1}\n\n\ +event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\",\"id\":\"f2\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_01_wUr\",\"name\":\"shell\"},\"output_index\":2}\n\n\ +event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"command\\\": \\\"find /home/focus/Desktop/Project/focus -maxdepth 2 -type f | head -50\\\"}\",\"item_id\":\"f2\",\"output_index\":2}\n\n\ +event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":850,\"input_tokens_details\":{\"cached_tokens\":768},\"output_tokens\":157,\"output_tokens_details\":{\"reasoning_tokens\":27},\"total_tokens\":1007}}}\n\n"; + +/// 真实日志第 2 轮的 SSE（reasoning + read + shell）。 +/// Turn 2 of the real log: reasoning + read + shell. +const REAL_TURN2: &str = "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"reasoning\",\"id\":\"rr2\",\"status\":\"in_progress\",\"content\":[],\"summary\":[]},\"output_index\":0}\n\n\ +event: response.reasoning_text.delta\ndata: {\"type\":\"response.reasoning_text.delta\",\"content_index\":0,\"delta\":\"Let me look at the Cargo.toml and AGENTS.md to understand what this project is.\",\"item_id\":\"rr2\",\"output_index\":0}\n\n\ +event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\",\"id\":\"f3\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_00_nK3\",\"name\":\"read\"},\"output_index\":1}\n\n\ +event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"path\\\": \\\"/home/focus/Desktop/Project/focus/Cargo.toml\\\"}\",\"item_id\":\"f3\",\"output_index\":1}\n\n\ +event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\",\"id\":\"f4\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_01_1IR\",\"name\":\"shell\"},\"output_index\":2}\n\n\ +event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"command\\\": \\\"ls /home/focus/Desktop/Project/focus/crates /home/focus/Desktop/Project/focus/docs /home/focus/Desktop/Project/focus/examples\\\"}\",\"item_id\":\"f4\",\"output_index\":2}\n\n\ +event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":1601,\"input_tokens_details\":{\"cached_tokens\":896},\"output_tokens\":142,\"output_tokens_details\":{\"reasoning_tokens\":21},\"total_tokens\":1743}}}\n\n"; + +/// 终止轮：reasoning + 文本回答。 +/// A terminating turn: reasoning + text answer. +const REAL_TURN3: &str = "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"reasoning\",\"id\":\"rr3\",\"status\":\"in_progress\",\"content\":[],\"summary\":[]},\"output_index\":0}\n\n\ +event: response.reasoning_text.delta\ndata: {\"type\":\"response.reasoning_text.delta\",\"content_index\":0,\"delta\":\"Now I can answer.\",\"item_id\":\"rr3\",\"output_index\":0}\n\n\ +event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\",\"id\":\"m1\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]},\"output_index\":1}\n\n\ +event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"m1\",\"output_index\":1,\"delta\":\"这是一个 Rust 项目。\"}\n\n\ +event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":900,\"output_tokens\":30}}}\n\n"; + +/// provider 级：真实第 1 轮 → 捕获推理（思考块）+ 两个工具调用 + usage。 +/// Provider level: real turn 1 → reasoning captured (thinking block) + two +/// tool calls + usage. +#[test] +fn real_log_captures_reasoning_and_tools() { + let mut mock = common::MockTransport::new(); + mock.push_body(REAL_TURN1); + let provider = provider(mock, OpenAiProtocol::Responses); + let events = common::collect(&provider, &request()); + + let msg = final_message(&events); + assert_eq!(msg.content.len(), 3, "thinking + 2 tool calls"); + match &msg.content[0] { + ContentBlock::Thinking(t) => { + assert_eq!( + t.thinking, + "The user is asking in Chinese: \"Hello, what does this project do?\" Let me explore the repository to understand what it is." + ); + } + other => panic!("expected thinking block, got {:?}", other), + } + let tc1 = msg.content[1].as_tool_call().expect("tool call 1"); + assert_eq!(tc1.name, "shell"); + let cmd1 = tc1.arguments.get_str("command").unwrap_or(""); + assert!(cmd1.contains("ls -la"), "got: {}", cmd1); + let tc2 = msg.content[2].as_tool_call().expect("tool call 2"); + assert_eq!(tc2.name, "shell"); + let cmd2 = tc2.arguments.get_str("command").unwrap_or(""); + assert!(cmd2.contains("find"), "got: {}", cmd2); + + // usage 映射：input / output / cache-read。 + // Usage mapping: input / output / cache-read. + assert_eq!(msg.usage.input_tokens, 850); + assert_eq!(msg.usage.output_tokens, 157); + assert_eq!(msg.usage.cache_read_tokens, 768); +} + +/// agent 级全链路：真实日志的三轮 → 思考进入 transcript + 历史回传 reasoning item。 +/// Agent-level end-to-end: three real-log turns → thinking lands in the +/// transcript + the replay carries the reasoning item. +#[test] +fn agent_multi_turn_replays_reasoning() { + use focus_core::event::VecSink; + use focus_core::tool::{ToolEffects, ToolRegistry}; + use focus_core::{Agent, AgentConfig, CoreError}; + + #[derive(Debug)] + struct NoopTool { + name: String, + } + impl Tool for NoopTool { + fn name(&self) -> &str { + &self.name + } + fn description(&self) -> &str { + "does nothing" + } + fn parameters(&self) -> focus_json::JsonValue { + focus_json::JsonValue::obj() + } + fn effects(&self) -> ToolEffects { + ToolEffects::NONE + } + fn execute( + &self, + _id: &str, + _args: &focus_json::JsonValue, + _u: Option<&ToolUpdateSink>, + ) -> Result { + Ok(ToolResult::text("ok")) + } + } + + let mut mock = common::MockTransport::new(); + mock.push_body(REAL_TURN1); + mock.push_body(REAL_TURN2); + mock.push_body(REAL_TURN3); + + let provider = OpenAiProvider::with_transport( + ProviderConfig::new("sk-test"), + OpenAiProtocol::Responses, + Arc::new(mock.clone()), + ); + let config = AgentConfig { + model: "deepseek-v4-flash".into(), + system_prompt: "sys".into(), + max_tokens: Some(4096), + ..Default::default() + }; + let mut registry = ToolRegistry::new(); + registry.register(Box::new(NoopTool { + name: "shell".into(), + })); + registry.register(Box::new(NoopTool { + name: "read".into(), + })); + let mut agent = Agent::new(config, Box::new(provider), registry); + let mut sink = VecSink::new(); + agent + .prompt("你好，本项目是干什么的", &mut sink) + .expect("run failed"); + + // 三轮：user + [assistant, tool, tool] ×2 + [assistant(text)]。 + // Three turns: user + [assistant, tool, tool] ×2 + [assistant(text)]. + let msgs = agent.messages(); + assert_eq!(msgs.len(), 8, "1 user + 3 assistant + 4 tool results"); + + // 第 1 条 assistant 消息：思考块（真实日志文本）在前。 + // The first assistant message: a thinking block (real log text) first. + let first_assistant = msgs + .iter() + .find_map(|m| match m { + Message::Assistant(a) => Some(a), + _ => None, + }) + .expect("assistant message"); + match &first_assistant.content[0] { + ContentBlock::Thinking(t) => { + assert!( + t.thinking.contains("The user is asking in Chinese"), + "got: {}", + t.thinking + ); + } + other => panic!("expected thinking, got {:?}", other), + } + // 第 2 条 assistant 消息的思考（真实日志第 2 轮）。 + // The second assistant's thinking (real log turn 2). + let second_assistant = msgs + .iter() + .filter_map(|m| match m { + Message::Assistant(a) => Some(a), + _ => None, + }) + .nth(1) + .expect("second assistant"); + match &second_assistant.content[0] { + ContentBlock::Thinking(t) => { + assert!( + t.thinking.contains("Cargo.toml and AGENTS.md"), + "got: {}", + t.thinking + ); + } + other => panic!("expected thinking, got {:?}", other), + } + + // 三个请求都发出了。 + // All three requests were sent. + assert_eq!(mock.request_count(), 3); + + // 第 2 个请求（回放第 1 轮历史）必须携带 reasoning item。 + // The second request (replaying turn-1 history) must carry a reasoning + // item. + let reqs = mock.requests(); + let body2: focus_json::JsonValue = + focus_json::parse(&String::from_utf8_lossy(&reqs[1].body)).unwrap(); + let input = body2.get_arr("input").expect("input items"); + let reasoning = input + .iter() + .find(|i| i.get_str("type") == Some("reasoning")) + .expect("reasoning item in replay"); + let content = reasoning.get_arr("content").unwrap(); + let text = content[0].get_str("text").unwrap_or(""); + assert!( + text.contains("The user is asking in Chinese"), + "got: {}", + text + ); + // function_call 与 function_call_output 的 call_id 一致。 + // The function_call and function_call_output call ids match. + let fc = input + .iter() + .find(|i| i.get_str("type") == Some("function_call")) + .expect("function_call item"); + let fco = input + .iter() + .find(|i| i.get_str("type") == Some("function_call_output")) + .expect("function_call_output item"); + assert_eq!(fc.get_str("call_id"), fco.get_str("call_id")); +}