test(sandbox): verify repeated restart and failed capture lifecycle

This commit is contained in:
2026-10-08 13:52:01 +08:00
parent 32d94f3a66
commit fc61e9d053
3 changed files with 99 additions and 4 deletions
+31 -3
View File
@@ -98,7 +98,10 @@ pub(super) fn run_case(case: &str) -> io::Result<serde_json::Value> {
std::thread::sleep(Duration::from_secs(30)); std::thread::sleep(Duration::from_secs(30));
return Ok(serde_json::json!({"residual_deadline":true})); return Ok(serde_json::json!({"residual_deadline":true}));
} }
if !matches!(case, "restart-prepared" | "restart-live") { if !matches!(
case,
"restart-prepared" | "restart-live" | "restart-capture"
) {
return Err(io::Error::other("unknown fixed restart case")); return Err(io::Error::other("unknown fixed restart case"));
} }
let fixture = Fixture::new("/kf/bin/isolation-probe")?; let fixture = Fixture::new("/kf/bin/isolation-probe")?;
@@ -132,13 +135,38 @@ pub(super) fn run_case(case: &str) -> io::Result<serde_json::Value> {
let (mut lease, dir) = CaptureLease::prepare(&config, &spec)?; let (mut lease, dir) = CaptureLease::prepare(&config, &spec)?;
let (pump, stdio) = crate::io::PipePump::prepare(target.take_stdin(), &spec, dir)?; let (pump, stdio) = crate::io::PipePump::prepare(target.take_stdin(), &spec, dir)?;
target.root_mut().target.prepare_execution(&spec, stdio)?; target.root_mut().target.prepare_execution(&spec, stdio)?;
let _child = ready(&config, &spec, "prepared")?; let capture_case = case == "restart-capture";
let _child = ready(
&config,
&spec,
if capture_case { "capture" } else { "prepared" },
)?;
let deadline = Instant::now() + Duration::from_secs(15); let deadline = Instant::now() + Duration::from_secs(15);
while Instant::now() < deadline { while Instant::now() < deadline {
if capture_case && std::path::Path::new("/kf/restart-cleanup").exists() {
break;
}
std::thread::sleep(Duration::from_millis(10)); std::thread::sleep(Duration::from_millis(10));
} }
drop(pump); drop(pump);
lease.cleanup()?; let capture = lease.cleanup();
if capture_case {
// The controller owns this exact fixture and repairs its foreign bare
// capture entry only after proving the failed UID remains reserved.
// Keep the delegated anchor alive for the fresh restart helper.
std::mem::forget(fixture);
let result = target.refuse_at(crate::outcome::ExecutionStage::Exec, None);
let reaped = owned.finish()?;
let e = capture
.err()
.ok_or_else(|| io::Error::other("expected real capture deletion failure"))?;
return Ok(serde_json::json!({
"case":case,"failure":result.failure,"cleanup_error":result.cleanup_failure,
"capture_cleanup_errno":e.raw_os_error(),"reaped_owned_orphans":reaped,
"user_exec_released":false,"owned_children_finished":true
}));
}
capture?;
let result = target.refuse_at(crate::outcome::ExecutionStage::Exec, None); let result = target.refuse_at(crate::outcome::ExecutionStage::Exec, None);
owned.finish()?; owned.finish()?;
fs::remove_file("/kf/restart-ready")?; fs::remove_file("/kf/restart-ready")?;
+1 -1
View File
@@ -217,7 +217,7 @@ fn incomplete_target_cleanup_keeps_identity_reserved() {
drop(next); drop(next);
assert_eq!(fs::read(task.join("foreign")).unwrap(), b"keep"); assert_eq!(fs::read(task.join("foreign")).unwrap(), b"keep");
// VM operator restores the test baseline only after proving both task // VM operator restores the test baseline only after proving both task
// cgroups/children gone. Production restart recovery is still unimplemented. // cgroups/children gone. Fresh-process restart recovery is tested separately.
fs::remove_file(task.join("foreign")).unwrap(); fs::remove_file(task.join("foreign")).unwrap();
fs::remove_dir(task).unwrap(); fs::remove_dir(task).unwrap();
fs::remove_file(Path::new(LEASES).join("uid-100000/owner.json")).unwrap(); fs::remove_file(Path::new(LEASES).join("uid-100000/owner.json")).unwrap();
+67
View File
@@ -229,10 +229,77 @@ impl Drop for Owner {
let _ = fs::remove_dir_all(&self.publications); let _ = fs::remove_dir_all(&self.publications);
let _ = fs::remove_file("/kf/restart-ready"); let _ = fs::remove_file("/kf/restart-ready");
let _ = fs::remove_file("/kf/restart-runtime.json"); let _ = fs::remove_file("/kf/restart-runtime.json");
let _ = fs::remove_file("/kf/restart-cleanup");
let _ = rustix::process::set_child_subreaper(self.previous); let _ = rustix::process::set_child_subreaper(self.previous);
} }
} }
#[test] #[test]
#[ignore = "independent capped VM: repeated real owner death reuses the same run and UID only after full recovery"]
fn repeated_restarts_reuse_only_fully_reclaimed_identity() {
for round in 0..3 {
let mut owner = Owner::start("live");
assert_eq!(record()["uid"], 100000);
assert_eq!(record()["run_id"], "restart-live");
owner.crash();
assert_eq!(recover()["items"][0]["status"], "reclaimed");
owner.finish();
println!(
"KF_LIFECYCLE_REUSE_PROOF round={round} same_uid=100000 same_run=restart-live actual_user_exec=1 all_resources_gone=1"
);
drop(owner);
}
}
#[test]
#[ignore = "independent capped VM: actual capture unlink ENOTEMPTY retains UID after all target processes and root are removed"]
fn capture_cleanup_failure_retains_identity_until_restart_repair() {
let mut owner = Owner::start("capture");
let capture = format!("/var/lib/kf-judge/captures/kf-{}", owner.run_id);
// The controller sees the bare mountpoint; the supervisor has a private
// mounted tmpfs at the same path. Never delete this foreign file implicitly.
fs::write(
format!("{capture}/foreign"),
b"capture deletion must preserve me",
)
.unwrap();
fs::write("/kf/restart-cleanup", b"now").unwrap();
let output = owner.child.take().unwrap().wait_with_output().unwrap();
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let v: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(v["capture_cleanup_errno"], 39);
assert_eq!(v["failure"]["stage"], "exec");
assert_eq!(v["cleanup_error"]["stage"], "cleanup");
assert_eq!(v["cleanup_error"]["code"], "cleanup_incomplete");
assert_eq!(v["user_exec_released"], false);
assert_eq!(v["owned_children_finished"], true);
owner.reap();
assert!(!std::path::Path::new(&owner.group()).exists());
assert!(!std::path::Path::new(&owner.task()).exists());
assert_eq!(record()["uid"], 100000);
assert_eq!(
fs::read(format!("{capture}/foreign")).unwrap(),
b"capture deletion must preserve me"
);
let recovery = recover();
assert_eq!(recovery["items"][0]["status"], "cleanup_failed");
assert_eq!(
recovery["items"][0]["cleanup_error"]["code"],
"cleanup_incomplete"
);
assert!(std::path::Path::new(LEASE).exists());
fs::remove_file(format!("{capture}/foreign")).unwrap();
assert_eq!(recover()["items"][0]["status"], "reclaimed");
empty();
assert_eq!(recover()["items"], serde_json::json!([]));
println!(
"KF_LIFECYCLE_CAPTURE_PROOF {}",
serde_json::json!({"target_result":v,"reserved_uid":100000,"foreign_preserved":true,"all_owned_pidfds_exited":owner.members.len(),"root_and_group_removed_before_uid_release":true,"fresh_service_recovery_after_explicit_repair":true,"repeat_cleanup_empty":true})
);
}
#[test]
#[ignore = "independent capped VM: real SIGKILL and service restart before/after user exec with surviving residual"] #[ignore = "independent capped VM: real SIGKILL and service restart before/after user exec with surviving residual"]
fn actual_owner_death_reclaims_prepared_and_live_tasks() { fn actual_owner_death_reclaims_prepared_and_live_tasks() {
for stage in ["prepared", "live"] { for stage in ["prepared", "live"] {