fix: 修正Core故障退避与熔断计数

This commit is contained in:
2026-09-12 02:29:04 +08:00
parent 41b8091839
commit bdba729a92
2 changed files with 112 additions and 8 deletions
+34 -8
View File
@@ -208,6 +208,25 @@ pub struct RequestSession {
}
impl CoreSupervisor {
fn record_failure(&mut self, now: Instant) {
self.attempts
.retain(|time| now.duration_since(*time) < Duration::from_secs(300));
self.attempts.push_back(now);
self.next_attempt = Some(now + Duration::from_secs(1 << (self.attempts.len() - 1).min(4)));
}
fn reap_failed_session(&mut self) -> bool {
let failed = self
.session
.as_mut()
.is_some_and(|session| !matches!(session.child.try_wait(), Ok(None)));
if failed {
self.session.take();
self.record_failure(Instant::now());
}
failed
}
pub fn new(
executable: PathBuf,
arguments: Vec<String>,
@@ -238,9 +257,17 @@ impl CoreSupervisor {
}
pub fn available(&mut self) -> bool {
self.session
.as_mut()
.is_some_and(|s| matches!(s.child.try_wait(), Ok(None)))
self.reap_failed_session();
self.session.is_some()
}
/// 返回受 Host 管理的 Core 启动进程 ID,用于诊断和故障注入。
pub fn process_id(&mut self) -> Option<u32> {
if self.available() {
self.session.as_ref().map(|session| session.child.id())
} else {
None
}
}
pub fn request_session(&mut self, path: &str) -> Result<RequestSession> {
@@ -260,18 +287,15 @@ impl CoreSupervisor {
if self.available() {
return Ok(());
}
self.session.take();
let now = Instant::now();
self.attempts
.retain(|t| now.duration_since(*t) < Duration::from_secs(300));
if self.attempts.len() >= 5 {
if self.attempts.len() > 5 {
return Err("CORE_RESTART_LIMIT".into());
}
if self.next_attempt.is_some_and(|t| now < t) {
return Err("CORE_RESTART_BACKOFF".into());
}
self.attempts.push_back(now);
self.next_attempt = Some(now + Duration::from_secs(1 << (self.attempts.len() - 1)));
if let Some(manifest) = &self.bundle_manifest {
verify_bundle(&self.working_dir, manifest)?;
}
@@ -281,8 +305,10 @@ impl CoreSupervisor {
&self.working_dir,
&self.data_dir,
self.broker.clone(),
)?;
)
.inspect_err(|_| self.record_failure(Instant::now()))?;
self.session = Some(session);
self.next_attempt = None;
Ok(())
}
+78
View File
@@ -2,9 +2,29 @@
use notesagent_host::core::CoreSupervisor;
use notesagent_host::credentials::{CredentialBroker, CredentialId, Scope};
use std::path::Path;
#[cfg(windows)]
use std::process::Command;
use std::sync::{Arc, Mutex};
#[cfg(windows)]
use std::time::{Duration, Instant};
use zeroize::Zeroizing;
#[cfg(windows)]
fn file_sha256(path: &Path) -> String {
use sha2::{Digest, Sha256};
let bytes = std::fs::read(path).unwrap();
format!("{:x}", Sha256::digest(bytes))
}
#[cfg(windows)]
fn terminate_process_tree(pid: u32) {
let status = Command::new("taskkill")
.args(["/PID", &pid.to_string(), "/T", "/F"])
.status()
.unwrap();
assert!(status.success(), "无法终止 Core 进程树 {pid}");
}
#[test]
fn real_python_core_authenticates_and_rotates_generation() {
let backend = Path::new(env!("CARGO_MANIFEST_DIR"))
@@ -48,6 +68,64 @@ fn real_python_core_authenticates_and_rotates_generation() {
assert!(std::net::TcpStream::connect(endpoint).is_err());
}
#[cfg(windows)]
#[test]
fn six_real_core_crashes_back_off_then_open_the_circuit_without_touching_local_edits() {
let backend = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../backend")
.canonicalize()
.unwrap();
let python = backend.join(".venv/Scripts/python.exe");
assert!(python.is_file(), "需要已锁定的后端虚拟环境");
let temp = tempfile::tempdir().unwrap();
let note = temp.path().join("local-edit.md");
std::fs::write(&note, "Core 故障期间仍由 Host 保存的本地修改。\n").unwrap();
let expected_hash = file_sha256(&note);
let mut core = CoreSupervisor::new(
python,
vec!["-m".into(), "app.sidecar".into()],
backend,
temp.path().join("core"),
);
let first = core.request_session("/health").unwrap();
let first_endpoint = first
.url
.trim_start_matches("http://")
.trim_end_matches("/health")
.to_string();
for crash in 0..6 {
terminate_process_tree(core.process_id().unwrap());
let stopped = Instant::now();
while core.available() {
assert!(stopped.elapsed() < Duration::from_secs(2));
std::thread::sleep(Duration::from_millis(10));
}
assert_eq!(file_sha256(&note), expected_hash);
if crash == 5 {
assert_eq!(
core.request_session("/health").err().unwrap(),
"CORE_RESTART_LIMIT"
);
break;
}
let expected_delay = Duration::from_secs(1 << crash);
assert_eq!(
core.request_session("/health").err().unwrap(),
"CORE_RESTART_BACKOFF"
);
while stopped.elapsed() < expected_delay {
std::thread::sleep(Duration::from_millis(20));
}
core.request_session("/health").unwrap();
}
let shutdown = Instant::now();
drop(core);
assert!(shutdown.elapsed() < Duration::from_secs(10));
assert!(std::net::TcpStream::connect(first_endpoint).is_err());
}
#[test]
fn real_core_credential_api_uses_host_stronghold_without_plaintext_response() {
use std::io::{Read, Write};