test(extensions): 验证容器 CPU 失败与实例替换
This commit is contained in:
@@ -681,15 +681,20 @@ mod tests {
|
|||||||
}
|
}
|
||||||
#[test]
|
#[test]
|
||||||
fn real_container_cannot_reach_ipv4_or_ipv6_loopback_listeners() {
|
fn real_container_cannot_reach_ipv4_or_ipv6_loopback_listeners() {
|
||||||
real_native_protocol_probes(false);
|
real_native_protocol_probes(false, false);
|
||||||
}
|
}
|
||||||
#[cfg(feature = "desktop")]
|
#[cfg(feature = "desktop")]
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "real MCP tools/call 60-second deadline acceptance; run explicitly"]
|
#[ignore = "real MCP tools/call 60-second deadline acceptance; run explicitly"]
|
||||||
fn real_mcp_tool_deadline_terminates_hung_server_and_host_can_save() {
|
fn real_mcp_tool_deadline_terminates_hung_server_and_host_can_save() {
|
||||||
real_native_protocol_probes(true);
|
real_native_protocol_probes(true, false);
|
||||||
}
|
}
|
||||||
fn real_native_protocol_probes(_mcp_deadline: bool) {
|
#[test]
|
||||||
|
#[ignore = "real AppContainer MCP descendant CPU pressure; run explicitly"]
|
||||||
|
fn real_mcp_cpu_pressure_reports_resource_error_and_reaps_tree() {
|
||||||
|
real_native_protocol_probes(false, true);
|
||||||
|
}
|
||||||
|
fn real_native_protocol_probes(_mcp_deadline: bool, cpu: bool) {
|
||||||
use std::{
|
use std::{
|
||||||
net::{TcpListener, UdpSocket},
|
net::{TcpListener, UdpSocket},
|
||||||
os::windows::fs::OpenOptionsExt,
|
os::windows::fs::OpenOptionsExt,
|
||||||
@@ -966,6 +971,9 @@ mod tests {
|
|||||||
if _mcp_deadline {
|
if _mcp_deadline {
|
||||||
mcp_modes.push("mcp_deadline");
|
mcp_modes.push("mcp_deadline");
|
||||||
}
|
}
|
||||||
|
if cpu {
|
||||||
|
mcp_modes.push("mcp_cpu");
|
||||||
|
}
|
||||||
for mode in mcp_modes {
|
for mode in mcp_modes {
|
||||||
use std::sync::{
|
use std::sync::{
|
||||||
atomic::{AtomicBool, Ordering},
|
atomic::{AtomicBool, Ordering},
|
||||||
@@ -1084,6 +1092,40 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert!(session.take_tools_changed());
|
assert!(session.take_tools_changed());
|
||||||
}
|
}
|
||||||
|
"mcp_cpu" => {
|
||||||
|
assert_eq!(result.unwrap_err().code, "EXTENSION_RESOURCE_CPU_EXCEEDED");
|
||||||
|
let elapsed = tool_started.elapsed();
|
||||||
|
assert!(elapsed < std::time::Duration::from_secs(20));
|
||||||
|
eprintln!("AppContainer MCP descendant CPU error: {elapsed:?}");
|
||||||
|
assert_eq!(
|
||||||
|
session.refresh_tools(&cancel).err().unwrap().code,
|
||||||
|
"EXTENSION_MCP_SESSION_FAILED"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
running.check_authorization().unwrap_err().code,
|
||||||
|
"EXTENSION_RESOURCE_CPU_EXCEEDED"
|
||||||
|
);
|
||||||
|
let vault = tempfile::tempdir().unwrap();
|
||||||
|
let mut workspace =
|
||||||
|
crate::workspace::Workspace::open(vault.path()).unwrap();
|
||||||
|
workspace
|
||||||
|
.write(
|
||||||
|
"cpu-recovery.md",
|
||||||
|
"",
|
||||||
|
b"Host saved after descendant CPU termination",
|
||||||
|
"local",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
drop(workspace);
|
||||||
|
assert_eq!(
|
||||||
|
crate::workspace::Workspace::open(vault.path())
|
||||||
|
.unwrap()
|
||||||
|
.read("cpu-recovery.md")
|
||||||
|
.unwrap()
|
||||||
|
.content,
|
||||||
|
"Host saved after descendant CPU termination"
|
||||||
|
);
|
||||||
|
}
|
||||||
"mcp_deadline" => {
|
"mcp_deadline" => {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
result.unwrap_err().code,
|
result.unwrap_err().code,
|
||||||
|
|||||||
@@ -511,6 +511,14 @@ mod tests {
|
|||||||
}
|
}
|
||||||
#[test]
|
#[test]
|
||||||
fn native_worker_routes_reviews_cancels_calls_and_reaps_generations() {
|
fn native_worker_routes_reviews_cancels_calls_and_reaps_generations() {
|
||||||
|
native_worker_lifecycle(false);
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
#[ignore = "real background MCP descendant CPU exhaustion and restart; run explicitly"]
|
||||||
|
fn native_cpu_failure_is_reported_reaped_and_replacement_can_start() {
|
||||||
|
native_worker_lifecycle(true);
|
||||||
|
}
|
||||||
|
fn native_worker_lifecycle(cpu: bool) {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let package = temp.path().join("package");
|
let package = temp.path().join("package");
|
||||||
std::fs::create_dir(&package).unwrap();
|
std::fs::create_dir(&package).unwrap();
|
||||||
@@ -724,6 +732,87 @@ mod tests {
|
|||||||
.unwrap(),
|
.unwrap(),
|
||||||
0
|
0
|
||||||
);
|
);
|
||||||
|
if cpu {
|
||||||
|
let exhausted = unsafe { registry.start(make("mcp_cpu")) }.unwrap();
|
||||||
|
wait_for(|| exhausted.snapshot().status != Status::Starting);
|
||||||
|
assert_eq!(exhausted.snapshot().status, Status::Ready);
|
||||||
|
let old_identity =
|
||||||
|
serde_json::to_value(exhausted.snapshot().identity.unwrap()).unwrap();
|
||||||
|
let review = exhausted
|
||||||
|
.review("echo".into(), json!({}))
|
||||||
|
.unwrap()
|
||||||
|
.wait(Duration::from_secs(5))
|
||||||
|
.unwrap();
|
||||||
|
let started = Instant::now();
|
||||||
|
assert_eq!(
|
||||||
|
exhausted
|
||||||
|
.invoke_confirmed(review.review_id)
|
||||||
|
.unwrap()
|
||||||
|
.wait(Duration::from_secs(20))
|
||||||
|
.unwrap_err()
|
||||||
|
.code,
|
||||||
|
"EXTENSION_RESOURCE_CPU_EXCEEDED"
|
||||||
|
);
|
||||||
|
eprintln!("background CPU call error after {:?}", started.elapsed());
|
||||||
|
wait_for(|| {
|
||||||
|
registry.reap();
|
||||||
|
registry.entries.is_empty()
|
||||||
|
});
|
||||||
|
assert_eq!(exhausted.snapshot().status, Status::Failed);
|
||||||
|
assert_eq!(
|
||||||
|
exhausted.snapshot().error.as_deref(),
|
||||||
|
Some("EXTENSION_RESOURCE_CPU_EXCEEDED")
|
||||||
|
);
|
||||||
|
assert_eq!(exhausted.snapshot().tool_count, 0);
|
||||||
|
assert!(exhausted.review("echo".into(), json!({})).is_err());
|
||||||
|
assert_eq!(
|
||||||
|
exhausted
|
||||||
|
.control
|
||||||
|
.job
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.active_processes()
|
||||||
|
.unwrap(),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
crate::extension_container::test_acl_entries(&acl_file),
|
||||||
|
file_acl
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
crate::extension_container::test_acl_entries(&acl_root),
|
||||||
|
root_acl
|
||||||
|
);
|
||||||
|
let replacement = unsafe { registry.start(make("mcp")) }.unwrap();
|
||||||
|
wait_for(|| replacement.snapshot().status != Status::Starting);
|
||||||
|
assert_eq!(replacement.snapshot().status, Status::Ready);
|
||||||
|
assert_ne!(
|
||||||
|
serde_json::to_value(replacement.snapshot().identity.unwrap()).unwrap()
|
||||||
|
["instance_id"],
|
||||||
|
old_identity["instance_id"]
|
||||||
|
);
|
||||||
|
let review = replacement
|
||||||
|
.review("echo".into(), json!({}))
|
||||||
|
.unwrap()
|
||||||
|
.wait(Duration::from_secs(5))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
replacement
|
||||||
|
.invoke_confirmed(review.review_id)
|
||||||
|
.unwrap()
|
||||||
|
.wait(Duration::from_secs(5))
|
||||||
|
.unwrap()["content"][0]["text"],
|
||||||
|
"native MCP success"
|
||||||
|
);
|
||||||
|
replacement.stop();
|
||||||
|
wait_for(|| {
|
||||||
|
registry.reap();
|
||||||
|
registry.entries.is_empty()
|
||||||
|
});
|
||||||
|
assert_eq!(replacement.snapshot().status, Status::Stopped);
|
||||||
|
}
|
||||||
let called = Arc::new(AtomicBool::new(false));
|
let called = Arc::new(AtomicBool::new(false));
|
||||||
let marker = Arc::clone(&called);
|
let marker = Arc::clone(&called);
|
||||||
let mut denied = make("mcp");
|
let mut denied = make("mcp");
|
||||||
|
|||||||
+18
-2
@@ -3,6 +3,22 @@ use std::net::{SocketAddr, TcpStream, UdpSocket};
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
fn main() {
|
fn main() {
|
||||||
let args: Vec<_> = std::env::args().collect();
|
let args: Vec<_> = std::env::args().collect();
|
||||||
|
if args.get(1).is_some_and(|s| s == "cpu_burn") {
|
||||||
|
std::thread::scope(|scope| {
|
||||||
|
for _ in 0..8 {
|
||||||
|
scope.spawn(|| {
|
||||||
|
let until = std::time::Instant::now() + Duration::from_secs(120);
|
||||||
|
let mut value = 1u64;
|
||||||
|
while std::time::Instant::now() < until {
|
||||||
|
for _ in 0..10000 {
|
||||||
|
value = std::hint::black_box(value.wrapping_mul(6364136223846793005).wrapping_add(1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
if args.get(1).is_some_and(|s| s.starts_with("mcp")) {
|
if args.get(1).is_some_and(|s| s.starts_with("mcp")) {
|
||||||
use std::io::{BufRead, Write};
|
use std::io::{BufRead, Write};
|
||||||
fn read(reader: &mut impl BufRead) -> String { let mut line = String::new(); reader.read_line(&mut line).unwrap(); line }
|
fn read(reader: &mut impl BufRead) -> String { let mut line = String::new(); reader.read_line(&mut line).unwrap(); line }
|
||||||
@@ -25,8 +41,8 @@ fn main() {
|
|||||||
}
|
}
|
||||||
let mut request = read(&mut input);
|
let mut request = read(&mut input);
|
||||||
assert!(request.contains("tools/call"));
|
assert!(request.contains("tools/call"));
|
||||||
if args[1] == "mcp_cancel" || args[1] == "mcp_deadline" {
|
if args[1] == "mcp_cancel" || args[1] == "mcp_deadline" || args[1] == "mcp_cpu" {
|
||||||
let _child = std::process::Command::new(std::env::current_exe().unwrap()).arg("wait").spawn().unwrap();
|
let _child = std::process::Command::new(std::env::current_exe().unwrap()).arg(if args[1] == "mcp_cpu" { "cpu_burn" } else { "wait" }).spawn().unwrap();
|
||||||
std::thread::sleep(Duration::from_secs(120));
|
std::thread::sleep(Duration::from_secs(120));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user