fix: 修正Core故障退避与熔断计数
This commit is contained in:
@@ -208,6 +208,25 @@ pub struct RequestSession {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl CoreSupervisor {
|
impl CoreSupervisor {
|
||||||
|
fn record_failure(&mut self, now: Instant) {
|
||||||
|
self.attempts
|
||||||
|
.retain(|time| now.duration_since(*time) < Duration::from_secs(300));
|
||||||
|
self.attempts.push_back(now);
|
||||||
|
self.next_attempt = Some(now + Duration::from_secs(1 << (self.attempts.len() - 1).min(4)));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reap_failed_session(&mut self) -> bool {
|
||||||
|
let failed = self
|
||||||
|
.session
|
||||||
|
.as_mut()
|
||||||
|
.is_some_and(|session| !matches!(session.child.try_wait(), Ok(None)));
|
||||||
|
if failed {
|
||||||
|
self.session.take();
|
||||||
|
self.record_failure(Instant::now());
|
||||||
|
}
|
||||||
|
failed
|
||||||
|
}
|
||||||
|
|
||||||
pub fn new(
|
pub fn new(
|
||||||
executable: PathBuf,
|
executable: PathBuf,
|
||||||
arguments: Vec<String>,
|
arguments: Vec<String>,
|
||||||
@@ -238,9 +257,17 @@ impl CoreSupervisor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn available(&mut self) -> bool {
|
pub fn available(&mut self) -> bool {
|
||||||
self.session
|
self.reap_failed_session();
|
||||||
.as_mut()
|
self.session.is_some()
|
||||||
.is_some_and(|s| matches!(s.child.try_wait(), Ok(None)))
|
}
|
||||||
|
|
||||||
|
/// 返回受 Host 管理的 Core 启动进程 ID,用于诊断和故障注入。
|
||||||
|
pub fn process_id(&mut self) -> Option<u32> {
|
||||||
|
if self.available() {
|
||||||
|
self.session.as_ref().map(|session| session.child.id())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn request_session(&mut self, path: &str) -> Result<RequestSession> {
|
pub fn request_session(&mut self, path: &str) -> Result<RequestSession> {
|
||||||
@@ -260,18 +287,15 @@ impl CoreSupervisor {
|
|||||||
if self.available() {
|
if self.available() {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
self.session.take();
|
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
self.attempts
|
self.attempts
|
||||||
.retain(|t| now.duration_since(*t) < Duration::from_secs(300));
|
.retain(|t| now.duration_since(*t) < Duration::from_secs(300));
|
||||||
if self.attempts.len() >= 5 {
|
if self.attempts.len() > 5 {
|
||||||
return Err("CORE_RESTART_LIMIT".into());
|
return Err("CORE_RESTART_LIMIT".into());
|
||||||
}
|
}
|
||||||
if self.next_attempt.is_some_and(|t| now < t) {
|
if self.next_attempt.is_some_and(|t| now < t) {
|
||||||
return Err("CORE_RESTART_BACKOFF".into());
|
return Err("CORE_RESTART_BACKOFF".into());
|
||||||
}
|
}
|
||||||
self.attempts.push_back(now);
|
|
||||||
self.next_attempt = Some(now + Duration::from_secs(1 << (self.attempts.len() - 1)));
|
|
||||||
if let Some(manifest) = &self.bundle_manifest {
|
if let Some(manifest) = &self.bundle_manifest {
|
||||||
verify_bundle(&self.working_dir, manifest)?;
|
verify_bundle(&self.working_dir, manifest)?;
|
||||||
}
|
}
|
||||||
@@ -281,8 +305,10 @@ impl CoreSupervisor {
|
|||||||
&self.working_dir,
|
&self.working_dir,
|
||||||
&self.data_dir,
|
&self.data_dir,
|
||||||
self.broker.clone(),
|
self.broker.clone(),
|
||||||
)?;
|
)
|
||||||
|
.inspect_err(|_| self.record_failure(Instant::now()))?;
|
||||||
self.session = Some(session);
|
self.session = Some(session);
|
||||||
|
self.next_attempt = None;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,29 @@
|
|||||||
use notesagent_host::core::CoreSupervisor;
|
use notesagent_host::core::CoreSupervisor;
|
||||||
use notesagent_host::credentials::{CredentialBroker, CredentialId, Scope};
|
use notesagent_host::credentials::{CredentialBroker, CredentialId, Scope};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
#[cfg(windows)]
|
||||||
|
use std::process::Command;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
#[cfg(windows)]
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
use zeroize::Zeroizing;
|
use zeroize::Zeroizing;
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
fn file_sha256(path: &Path) -> String {
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
let bytes = std::fs::read(path).unwrap();
|
||||||
|
format!("{:x}", Sha256::digest(bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
fn terminate_process_tree(pid: u32) {
|
||||||
|
let status = Command::new("taskkill")
|
||||||
|
.args(["/PID", &pid.to_string(), "/T", "/F"])
|
||||||
|
.status()
|
||||||
|
.unwrap();
|
||||||
|
assert!(status.success(), "无法终止 Core 进程树 {pid}");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn real_python_core_authenticates_and_rotates_generation() {
|
fn real_python_core_authenticates_and_rotates_generation() {
|
||||||
let backend = Path::new(env!("CARGO_MANIFEST_DIR"))
|
let backend = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
@@ -48,6 +68,64 @@ fn real_python_core_authenticates_and_rotates_generation() {
|
|||||||
assert!(std::net::TcpStream::connect(endpoint).is_err());
|
assert!(std::net::TcpStream::connect(endpoint).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(windows)]
|
||||||
|
#[test]
|
||||||
|
fn six_real_core_crashes_back_off_then_open_the_circuit_without_touching_local_edits() {
|
||||||
|
let backend = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("../../backend")
|
||||||
|
.canonicalize()
|
||||||
|
.unwrap();
|
||||||
|
let python = backend.join(".venv/Scripts/python.exe");
|
||||||
|
assert!(python.is_file(), "需要已锁定的后端虚拟环境");
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let note = temp.path().join("local-edit.md");
|
||||||
|
std::fs::write(¬e, "Core 故障期间仍由 Host 保存的本地修改。\n").unwrap();
|
||||||
|
let expected_hash = file_sha256(¬e);
|
||||||
|
let mut core = CoreSupervisor::new(
|
||||||
|
python,
|
||||||
|
vec!["-m".into(), "app.sidecar".into()],
|
||||||
|
backend,
|
||||||
|
temp.path().join("core"),
|
||||||
|
);
|
||||||
|
|
||||||
|
let first = core.request_session("/health").unwrap();
|
||||||
|
let first_endpoint = first
|
||||||
|
.url
|
||||||
|
.trim_start_matches("http://")
|
||||||
|
.trim_end_matches("/health")
|
||||||
|
.to_string();
|
||||||
|
for crash in 0..6 {
|
||||||
|
terminate_process_tree(core.process_id().unwrap());
|
||||||
|
let stopped = Instant::now();
|
||||||
|
while core.available() {
|
||||||
|
assert!(stopped.elapsed() < Duration::from_secs(2));
|
||||||
|
std::thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
assert_eq!(file_sha256(¬e), expected_hash);
|
||||||
|
if crash == 5 {
|
||||||
|
assert_eq!(
|
||||||
|
core.request_session("/health").err().unwrap(),
|
||||||
|
"CORE_RESTART_LIMIT"
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let expected_delay = Duration::from_secs(1 << crash);
|
||||||
|
assert_eq!(
|
||||||
|
core.request_session("/health").err().unwrap(),
|
||||||
|
"CORE_RESTART_BACKOFF"
|
||||||
|
);
|
||||||
|
while stopped.elapsed() < expected_delay {
|
||||||
|
std::thread::sleep(Duration::from_millis(20));
|
||||||
|
}
|
||||||
|
core.request_session("/health").unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let shutdown = Instant::now();
|
||||||
|
drop(core);
|
||||||
|
assert!(shutdown.elapsed() < Duration::from_secs(10));
|
||||||
|
assert!(std::net::TcpStream::connect(first_endpoint).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn real_core_credential_api_uses_host_stronghold_without_plaintext_response() {
|
fn real_core_credential_api_uses_host_stronghold_without_plaintext_response() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
|
|||||||
Reference in New Issue
Block a user