test(providers): real-endpoint-log regressions for Responses reasoning
Based on the user's actual deepseek-v4-flash debug log (Responses protocol): - real_log_captures_reasoning_and_tools: replays turn 1 of the real SSE (reasoning item + reasoning_text.delta + two function calls) and asserts the Done message is [Thinking, ToolCall, ToolCall] with usage mapped (input 850 / output 157 / cache-read 768) - agent_multi_turn_replays_reasoning: drives a 3-turn agent over the real protocol (turn 1 + turn 2 from the log, plus a terminating text turn), asserting thinking lands in the transcript and the second request's replay carries a reasoning item with the captured text and matching call ids MockTransport gains requests() to inspect all recorded requests.
This commit is contained in:
DaiChaoXiong
parent
80333ee127
commit
6257f4f6b8
|
|
@ -61,6 +61,13 @@ impl MockTransport {
|
||||||
.expect("no request recorded")
|
.expect("no request recorded")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 已记录的全部请求克隆（按顺序）。
|
||||||
|
/// Clones of all recorded requests (in order).
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub fn requests(&self) -> Vec<HttpRequest> {
|
||||||
|
self.requests.lock().unwrap().clone()
|
||||||
|
}
|
||||||
|
|
||||||
/// 已记录的请求数量。
|
/// 已记录的请求数量。
|
||||||
/// Number of recorded requests.
|
/// Number of recorded requests.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
|
|
|
||||||
|
|
@ -618,3 +618,222 @@ fn responses_replays_reasoning_item() {
|
||||||
assert_eq!(content[0].get_str("type"), Some("reasoning_text"));
|
assert_eq!(content[0].get_str("type"), Some("reasoning_text"));
|
||||||
assert_eq!(content[0].get_str("text"), Some("思考内容"));
|
assert_eq!(content[0].get_str("text"), Some("思考内容"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- 基于真实端点日志的回归测试 ----
|
||||||
|
// ---- regressions built from a real endpoint log (deepseek-v4-flash,
|
||||||
|
// Responses API with reasoning items) --------------------------------
|
||||||
|
|
||||||
|
/// 真实日志第 1 轮的 SSE（reasoning + 两个 shell 调用）。
|
||||||
|
/// Turn 1 of the real log: reasoning + two shell calls.
|
||||||
|
const REAL_TURN1: &str = "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"r1\",\"status\":\"in_progress\"}}\n\n\
|
||||||
|
event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"reasoning\",\"id\":\"rr1\",\"status\":\"in_progress\",\"content\":[],\"summary\":[]},\"output_index\":0}\n\n\
|
||||||
|
event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"rr1\",\"output_index\":0,\"part\":{\"type\":\"reasoning_text\",\"text\":\"\"}}\n\n\
|
||||||
|
event: response.reasoning_text.delta\ndata: {\"type\":\"response.reasoning_text.delta\",\"content_index\":0,\"delta\":\"The user is asking in Chinese: \\\"Hello, what does this project do?\\\" Let me explore\",\"item_id\":\"rr1\",\"output_index\":0}\n\n\
|
||||||
|
event: response.reasoning_text.delta\ndata: {\"type\":\"response.reasoning_text.delta\",\"content_index\":0,\"delta\":\" the repository to understand what it is.\",\"item_id\":\"rr1\",\"output_index\":0}\n\n\
|
||||||
|
event: response.reasoning_text.done\ndata: {\"type\":\"response.reasoning_text.done\",\"content_index\":0,\"item_id\":\"rr1\",\"output_index\":0,\"text\":\"The user is asking in Chinese: \\\"Hello, what does this project do?\\\" Let me explore the repository to understand what it is.\"}\n\n\
|
||||||
|
event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\",\"id\":\"f1\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_00_wyQ\",\"name\":\"shell\"},\"output_index\":1}\n\n\
|
||||||
|
event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"command\\\": \\\"ls -la /home/focus/Desktop/Project/focus && cat /home/focus/Desktop/Project/focus/README.md 2>/dev/null\",\"item_id\":\"f1\",\"output_index\":1}\n\n\
|
||||||
|
event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" | head -50\\\"}\",\"item_id\":\"f1\",\"output_index\":1}\n\n\
|
||||||
|
event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\",\"id\":\"f2\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_01_wUr\",\"name\":\"shell\"},\"output_index\":2}\n\n\
|
||||||
|
event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"command\\\": \\\"find /home/focus/Desktop/Project/focus -maxdepth 2 -type f | head -50\\\"}\",\"item_id\":\"f2\",\"output_index\":2}\n\n\
|
||||||
|
event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":850,\"input_tokens_details\":{\"cached_tokens\":768},\"output_tokens\":157,\"output_tokens_details\":{\"reasoning_tokens\":27},\"total_tokens\":1007}}}\n\n";
|
||||||
|
|
||||||
|
/// 真实日志第 2 轮的 SSE（reasoning + read + shell）。
|
||||||
|
/// Turn 2 of the real log: reasoning + read + shell.
|
||||||
|
const REAL_TURN2: &str = "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"reasoning\",\"id\":\"rr2\",\"status\":\"in_progress\",\"content\":[],\"summary\":[]},\"output_index\":0}\n\n\
|
||||||
|
event: response.reasoning_text.delta\ndata: {\"type\":\"response.reasoning_text.delta\",\"content_index\":0,\"delta\":\"Let me look at the Cargo.toml and AGENTS.md to understand what this project is.\",\"item_id\":\"rr2\",\"output_index\":0}\n\n\
|
||||||
|
event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\",\"id\":\"f3\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_00_nK3\",\"name\":\"read\"},\"output_index\":1}\n\n\
|
||||||
|
event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"path\\\": \\\"/home/focus/Desktop/Project/focus/Cargo.toml\\\"}\",\"item_id\":\"f3\",\"output_index\":1}\n\n\
|
||||||
|
event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\",\"id\":\"f4\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_01_1IR\",\"name\":\"shell\"},\"output_index\":2}\n\n\
|
||||||
|
event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"command\\\": \\\"ls /home/focus/Desktop/Project/focus/crates /home/focus/Desktop/Project/focus/docs /home/focus/Desktop/Project/focus/examples\\\"}\",\"item_id\":\"f4\",\"output_index\":2}\n\n\
|
||||||
|
event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":1601,\"input_tokens_details\":{\"cached_tokens\":896},\"output_tokens\":142,\"output_tokens_details\":{\"reasoning_tokens\":21},\"total_tokens\":1743}}}\n\n";
|
||||||
|
|
||||||
|
/// 终止轮：reasoning + 文本回答。
|
||||||
|
/// A terminating turn: reasoning + text answer.
|
||||||
|
const REAL_TURN3: &str = "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"reasoning\",\"id\":\"rr3\",\"status\":\"in_progress\",\"content\":[],\"summary\":[]},\"output_index\":0}\n\n\
|
||||||
|
event: response.reasoning_text.delta\ndata: {\"type\":\"response.reasoning_text.delta\",\"content_index\":0,\"delta\":\"Now I can answer.\",\"item_id\":\"rr3\",\"output_index\":0}\n\n\
|
||||||
|
event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\",\"id\":\"m1\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}]},\"output_index\":1}\n\n\
|
||||||
|
event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"m1\",\"output_index\":1,\"delta\":\"这是一个 Rust 项目。\"}\n\n\
|
||||||
|
event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":900,\"output_tokens\":30}}}\n\n";
|
||||||
|
|
||||||
|
/// provider 级：真实第 1 轮 → 捕获推理（思考块）+ 两个工具调用 + usage。
|
||||||
|
/// Provider level: real turn 1 → reasoning captured (thinking block) + two
|
||||||
|
/// tool calls + usage.
|
||||||
|
#[test]
|
||||||
|
fn real_log_captures_reasoning_and_tools() {
|
||||||
|
let mut mock = common::MockTransport::new();
|
||||||
|
mock.push_body(REAL_TURN1);
|
||||||
|
let provider = provider(mock, OpenAiProtocol::Responses);
|
||||||
|
let events = common::collect(&provider, &request());
|
||||||
|
|
||||||
|
let msg = final_message(&events);
|
||||||
|
assert_eq!(msg.content.len(), 3, "thinking + 2 tool calls");
|
||||||
|
match &msg.content[0] {
|
||||||
|
ContentBlock::Thinking(t) => {
|
||||||
|
assert_eq!(
|
||||||
|
t.thinking,
|
||||||
|
"The user is asking in Chinese: \"Hello, what does this project do?\" Let me explore the repository to understand what it is."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
other => panic!("expected thinking block, got {:?}", other),
|
||||||
|
}
|
||||||
|
let tc1 = msg.content[1].as_tool_call().expect("tool call 1");
|
||||||
|
assert_eq!(tc1.name, "shell");
|
||||||
|
let cmd1 = tc1.arguments.get_str("command").unwrap_or("");
|
||||||
|
assert!(cmd1.contains("ls -la"), "got: {}", cmd1);
|
||||||
|
let tc2 = msg.content[2].as_tool_call().expect("tool call 2");
|
||||||
|
assert_eq!(tc2.name, "shell");
|
||||||
|
let cmd2 = tc2.arguments.get_str("command").unwrap_or("");
|
||||||
|
assert!(cmd2.contains("find"), "got: {}", cmd2);
|
||||||
|
|
||||||
|
// usage 映射：input / output / cache-read。
|
||||||
|
// Usage mapping: input / output / cache-read.
|
||||||
|
assert_eq!(msg.usage.input_tokens, 850);
|
||||||
|
assert_eq!(msg.usage.output_tokens, 157);
|
||||||
|
assert_eq!(msg.usage.cache_read_tokens, 768);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// agent 级全链路：真实日志的三轮 → 思考进入 transcript + 历史回传 reasoning item。
|
||||||
|
/// Agent-level end-to-end: three real-log turns → thinking lands in the
|
||||||
|
/// transcript + the replay carries the reasoning item.
|
||||||
|
#[test]
|
||||||
|
fn agent_multi_turn_replays_reasoning() {
|
||||||
|
use focus_core::event::VecSink;
|
||||||
|
use focus_core::tool::{ToolEffects, ToolRegistry};
|
||||||
|
use focus_core::{Agent, AgentConfig, CoreError};
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct NoopTool {
|
||||||
|
name: String,
|
||||||
|
}
|
||||||
|
impl Tool for NoopTool {
|
||||||
|
fn name(&self) -> &str {
|
||||||
|
&self.name
|
||||||
|
}
|
||||||
|
fn description(&self) -> &str {
|
||||||
|
"does nothing"
|
||||||
|
}
|
||||||
|
fn parameters(&self) -> focus_json::JsonValue {
|
||||||
|
focus_json::JsonValue::obj()
|
||||||
|
}
|
||||||
|
fn effects(&self) -> ToolEffects {
|
||||||
|
ToolEffects::NONE
|
||||||
|
}
|
||||||
|
fn execute(
|
||||||
|
&self,
|
||||||
|
_id: &str,
|
||||||
|
_args: &focus_json::JsonValue,
|
||||||
|
_u: Option<&ToolUpdateSink>,
|
||||||
|
) -> Result<ToolResult, CoreError> {
|
||||||
|
Ok(ToolResult::text("ok"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut mock = common::MockTransport::new();
|
||||||
|
mock.push_body(REAL_TURN1);
|
||||||
|
mock.push_body(REAL_TURN2);
|
||||||
|
mock.push_body(REAL_TURN3);
|
||||||
|
|
||||||
|
let provider = OpenAiProvider::with_transport(
|
||||||
|
ProviderConfig::new("sk-test"),
|
||||||
|
OpenAiProtocol::Responses,
|
||||||
|
Arc::new(mock.clone()),
|
||||||
|
);
|
||||||
|
let config = AgentConfig {
|
||||||
|
model: "deepseek-v4-flash".into(),
|
||||||
|
system_prompt: "sys".into(),
|
||||||
|
max_tokens: Some(4096),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut registry = ToolRegistry::new();
|
||||||
|
registry.register(Box::new(NoopTool {
|
||||||
|
name: "shell".into(),
|
||||||
|
}));
|
||||||
|
registry.register(Box::new(NoopTool {
|
||||||
|
name: "read".into(),
|
||||||
|
}));
|
||||||
|
let mut agent = Agent::new(config, Box::new(provider), registry);
|
||||||
|
let mut sink = VecSink::new();
|
||||||
|
agent
|
||||||
|
.prompt("你好，本项目是干什么的", &mut sink)
|
||||||
|
.expect("run failed");
|
||||||
|
|
||||||
|
// 三轮：user + [assistant, tool, tool] ×2 + [assistant(text)]。
|
||||||
|
// Three turns: user + [assistant, tool, tool] ×2 + [assistant(text)].
|
||||||
|
let msgs = agent.messages();
|
||||||
|
assert_eq!(msgs.len(), 8, "1 user + 3 assistant + 4 tool results");
|
||||||
|
|
||||||
|
// 第 1 条 assistant 消息：思考块（真实日志文本）在前。
|
||||||
|
// The first assistant message: a thinking block (real log text) first.
|
||||||
|
let first_assistant = msgs
|
||||||
|
.iter()
|
||||||
|
.find_map(|m| match m {
|
||||||
|
Message::Assistant(a) => Some(a),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.expect("assistant message");
|
||||||
|
match &first_assistant.content[0] {
|
||||||
|
ContentBlock::Thinking(t) => {
|
||||||
|
assert!(
|
||||||
|
t.thinking.contains("The user is asking in Chinese"),
|
||||||
|
"got: {}",
|
||||||
|
t.thinking
|
||||||
|
);
|
||||||
|
}
|
||||||
|
other => panic!("expected thinking, got {:?}", other),
|
||||||
|
}
|
||||||
|
// 第 2 条 assistant 消息的思考（真实日志第 2 轮）。
|
||||||
|
// The second assistant's thinking (real log turn 2).
|
||||||
|
let second_assistant = msgs
|
||||||
|
.iter()
|
||||||
|
.filter_map(|m| match m {
|
||||||
|
Message::Assistant(a) => Some(a),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.nth(1)
|
||||||
|
.expect("second assistant");
|
||||||
|
match &second_assistant.content[0] {
|
||||||
|
ContentBlock::Thinking(t) => {
|
||||||
|
assert!(
|
||||||
|
t.thinking.contains("Cargo.toml and AGENTS.md"),
|
||||||
|
"got: {}",
|
||||||
|
t.thinking
|
||||||
|
);
|
||||||
|
}
|
||||||
|
other => panic!("expected thinking, got {:?}", other),
|
||||||
|
}
|
||||||
|
|
||||||
|
// 三个请求都发出了。
|
||||||
|
// All three requests were sent.
|
||||||
|
assert_eq!(mock.request_count(), 3);
|
||||||
|
|
||||||
|
// 第 2 个请求（回放第 1 轮历史）必须携带 reasoning item。
|
||||||
|
// The second request (replaying turn-1 history) must carry a reasoning
|
||||||
|
// item.
|
||||||
|
let reqs = mock.requests();
|
||||||
|
let body2: focus_json::JsonValue =
|
||||||
|
focus_json::parse(&String::from_utf8_lossy(&reqs[1].body)).unwrap();
|
||||||
|
let input = body2.get_arr("input").expect("input items");
|
||||||
|
let reasoning = input
|
||||||
|
.iter()
|
||||||
|
.find(|i| i.get_str("type") == Some("reasoning"))
|
||||||
|
.expect("reasoning item in replay");
|
||||||
|
let content = reasoning.get_arr("content").unwrap();
|
||||||
|
let text = content[0].get_str("text").unwrap_or("");
|
||||||
|
assert!(
|
||||||
|
text.contains("The user is asking in Chinese"),
|
||||||
|
"got: {}",
|
||||||
|
text
|
||||||
|
);
|
||||||
|
// function_call 与 function_call_output 的 call_id 一致。
|
||||||
|
// The function_call and function_call_output call ids match.
|
||||||
|
let fc = input
|
||||||
|
.iter()
|
||||||
|
.find(|i| i.get_str("type") == Some("function_call"))
|
||||||
|
.expect("function_call item");
|
||||||
|
let fco = input
|
||||||
|
.iter()
|
||||||
|
.find(|i| i.get_str("type") == Some("function_call_output"))
|
||||||
|
.expect("function_call_output item");
|
||||||
|
assert_eq!(fc.get_str("call_id"), fco.get_str("call_id"));
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue