fix: 修正Core故障退避与熔断计数
This commit is contained in:
@@ -208,6 +208,25 @@ pub struct RequestSession {
|
||||
}
|
||||
|
||||
impl CoreSupervisor {
|
||||
fn record_failure(&mut self, now: Instant) {
|
||||
self.attempts
|
||||
.retain(|time| now.duration_since(*time) < Duration::from_secs(300));
|
||||
self.attempts.push_back(now);
|
||||
self.next_attempt = Some(now + Duration::from_secs(1 << (self.attempts.len() - 1).min(4)));
|
||||
}
|
||||
|
||||
fn reap_failed_session(&mut self) -> bool {
|
||||
let failed = self
|
||||
.session
|
||||
.as_mut()
|
||||
.is_some_and(|session| !matches!(session.child.try_wait(), Ok(None)));
|
||||
if failed {
|
||||
self.session.take();
|
||||
self.record_failure(Instant::now());
|
||||
}
|
||||
failed
|
||||
}
|
||||
|
||||
pub fn new(
|
||||
executable: PathBuf,
|
||||
arguments: Vec<String>,
|
||||
@@ -238,9 +257,17 @@ impl CoreSupervisor {
|
||||
}
|
||||
|
||||
pub fn available(&mut self) -> bool {
|
||||
self.session
|
||||
.as_mut()
|
||||
.is_some_and(|s| matches!(s.child.try_wait(), Ok(None)))
|
||||
self.reap_failed_session();
|
||||
self.session.is_some()
|
||||
}
|
||||
|
||||
/// 返回受 Host 管理的 Core 启动进程 ID,用于诊断和故障注入。
|
||||
pub fn process_id(&mut self) -> Option<u32> {
|
||||
if self.available() {
|
||||
self.session.as_ref().map(|session| session.child.id())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn request_session(&mut self, path: &str) -> Result<RequestSession> {
|
||||
@@ -260,18 +287,15 @@ impl CoreSupervisor {
|
||||
if self.available() {
|
||||
return Ok(());
|
||||
}
|
||||
self.session.take();
|
||||
let now = Instant::now();
|
||||
self.attempts
|
||||
.retain(|t| now.duration_since(*t) < Duration::from_secs(300));
|
||||
if self.attempts.len() >= 5 {
|
||||
if self.attempts.len() > 5 {
|
||||
return Err("CORE_RESTART_LIMIT".into());
|
||||
}
|
||||
if self.next_attempt.is_some_and(|t| now < t) {
|
||||
return Err("CORE_RESTART_BACKOFF".into());
|
||||
}
|
||||
self.attempts.push_back(now);
|
||||
self.next_attempt = Some(now + Duration::from_secs(1 << (self.attempts.len() - 1)));
|
||||
if let Some(manifest) = &self.bundle_manifest {
|
||||
verify_bundle(&self.working_dir, manifest)?;
|
||||
}
|
||||
@@ -281,8 +305,10 @@ impl CoreSupervisor {
|
||||
&self.working_dir,
|
||||
&self.data_dir,
|
||||
self.broker.clone(),
|
||||
)?;
|
||||
)
|
||||
.inspect_err(|_| self.record_failure(Instant::now()))?;
|
||||
self.session = Some(session);
|
||||
self.next_attempt = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -2,9 +2,29 @@
|
||||
use notesagent_host::core::CoreSupervisor;
|
||||
use notesagent_host::credentials::{CredentialBroker, CredentialId, Scope};
|
||||
use std::path::Path;
|
||||
#[cfg(windows)]
|
||||
use std::process::Command;
|
||||
use std::sync::{Arc, Mutex};
|
||||
#[cfg(windows)]
|
||||
use std::time::{Duration, Instant};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
#[cfg(windows)]
|
||||
fn file_sha256(path: &Path) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
let bytes = std::fs::read(path).unwrap();
|
||||
format!("{:x}", Sha256::digest(bytes))
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn terminate_process_tree(pid: u32) {
|
||||
let status = Command::new("taskkill")
|
||||
.args(["/PID", &pid.to_string(), "/T", "/F"])
|
||||
.status()
|
||||
.unwrap();
|
||||
assert!(status.success(), "无法终止 Core 进程树 {pid}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_python_core_authenticates_and_rotates_generation() {
|
||||
let backend = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
@@ -48,6 +68,64 @@ fn real_python_core_authenticates_and_rotates_generation() {
|
||||
assert!(std::net::TcpStream::connect(endpoint).is_err());
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn six_real_core_crashes_back_off_then_open_the_circuit_without_touching_local_edits() {
|
||||
let backend = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../backend")
|
||||
.canonicalize()
|
||||
.unwrap();
|
||||
let python = backend.join(".venv/Scripts/python.exe");
|
||||
assert!(python.is_file(), "需要已锁定的后端虚拟环境");
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let note = temp.path().join("local-edit.md");
|
||||
std::fs::write(¬e, "Core 故障期间仍由 Host 保存的本地修改。\n").unwrap();
|
||||
let expected_hash = file_sha256(¬e);
|
||||
let mut core = CoreSupervisor::new(
|
||||
python,
|
||||
vec!["-m".into(), "app.sidecar".into()],
|
||||
backend,
|
||||
temp.path().join("core"),
|
||||
);
|
||||
|
||||
let first = core.request_session("/health").unwrap();
|
||||
let first_endpoint = first
|
||||
.url
|
||||
.trim_start_matches("http://")
|
||||
.trim_end_matches("/health")
|
||||
.to_string();
|
||||
for crash in 0..6 {
|
||||
terminate_process_tree(core.process_id().unwrap());
|
||||
let stopped = Instant::now();
|
||||
while core.available() {
|
||||
assert!(stopped.elapsed() < Duration::from_secs(2));
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert_eq!(file_sha256(¬e), expected_hash);
|
||||
if crash == 5 {
|
||||
assert_eq!(
|
||||
core.request_session("/health").err().unwrap(),
|
||||
"CORE_RESTART_LIMIT"
|
||||
);
|
||||
break;
|
||||
}
|
||||
let expected_delay = Duration::from_secs(1 << crash);
|
||||
assert_eq!(
|
||||
core.request_session("/health").err().unwrap(),
|
||||
"CORE_RESTART_BACKOFF"
|
||||
);
|
||||
while stopped.elapsed() < expected_delay {
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
core.request_session("/health").unwrap();
|
||||
}
|
||||
|
||||
let shutdown = Instant::now();
|
||||
drop(core);
|
||||
assert!(shutdown.elapsed() < Duration::from_secs(10));
|
||||
assert!(std::net::TcpStream::connect(first_endpoint).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_core_credential_api_uses_host_stronghold_without_plaintext_response() {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
Reference in New Issue
Block a user