test(sandbox): verify repeated restart and failed capture lifecycle
This commit is contained in:
@@ -98,7 +98,10 @@ pub(super) fn run_case(case: &str) -> io::Result<serde_json::Value> {
|
||||
std::thread::sleep(Duration::from_secs(30));
|
||||
return Ok(serde_json::json!({"residual_deadline":true}));
|
||||
}
|
||||
if !matches!(case, "restart-prepared" | "restart-live") {
|
||||
if !matches!(
|
||||
case,
|
||||
"restart-prepared" | "restart-live" | "restart-capture"
|
||||
) {
|
||||
return Err(io::Error::other("unknown fixed restart case"));
|
||||
}
|
||||
let fixture = Fixture::new("/kf/bin/isolation-probe")?;
|
||||
@@ -132,13 +135,38 @@ pub(super) fn run_case(case: &str) -> io::Result<serde_json::Value> {
|
||||
let (mut lease, dir) = CaptureLease::prepare(&config, &spec)?;
|
||||
let (pump, stdio) = crate::io::PipePump::prepare(target.take_stdin(), &spec, dir)?;
|
||||
target.root_mut().target.prepare_execution(&spec, stdio)?;
|
||||
let _child = ready(&config, &spec, "prepared")?;
|
||||
let capture_case = case == "restart-capture";
|
||||
let _child = ready(
|
||||
&config,
|
||||
&spec,
|
||||
if capture_case { "capture" } else { "prepared" },
|
||||
)?;
|
||||
let deadline = Instant::now() + Duration::from_secs(15);
|
||||
while Instant::now() < deadline {
|
||||
if capture_case && std::path::Path::new("/kf/restart-cleanup").exists() {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
drop(pump);
|
||||
lease.cleanup()?;
|
||||
let capture = lease.cleanup();
|
||||
if capture_case {
|
||||
// The controller owns this exact fixture and repairs its foreign bare
|
||||
// capture entry only after proving the failed UID remains reserved.
|
||||
// Keep the delegated anchor alive for the fresh restart helper.
|
||||
std::mem::forget(fixture);
|
||||
let result = target.refuse_at(crate::outcome::ExecutionStage::Exec, None);
|
||||
let reaped = owned.finish()?;
|
||||
let e = capture
|
||||
.err()
|
||||
.ok_or_else(|| io::Error::other("expected real capture deletion failure"))?;
|
||||
return Ok(serde_json::json!({
|
||||
"case":case,"failure":result.failure,"cleanup_error":result.cleanup_failure,
|
||||
"capture_cleanup_errno":e.raw_os_error(),"reaped_owned_orphans":reaped,
|
||||
"user_exec_released":false,"owned_children_finished":true
|
||||
}));
|
||||
}
|
||||
capture?;
|
||||
let result = target.refuse_at(crate::outcome::ExecutionStage::Exec, None);
|
||||
owned.finish()?;
|
||||
fs::remove_file("/kf/restart-ready")?;
|
||||
|
||||
@@ -217,7 +217,7 @@ fn incomplete_target_cleanup_keeps_identity_reserved() {
|
||||
drop(next);
|
||||
assert_eq!(fs::read(task.join("foreign")).unwrap(), b"keep");
|
||||
// VM operator restores the test baseline only after proving both task
|
||||
// cgroups/children gone. Production restart recovery is still unimplemented.
|
||||
// cgroups/children gone. Fresh-process restart recovery is tested separately.
|
||||
fs::remove_file(task.join("foreign")).unwrap();
|
||||
fs::remove_dir(task).unwrap();
|
||||
fs::remove_file(Path::new(LEASES).join("uid-100000/owner.json")).unwrap();
|
||||
|
||||
@@ -229,10 +229,77 @@ impl Drop for Owner {
|
||||
let _ = fs::remove_dir_all(&self.publications);
|
||||
let _ = fs::remove_file("/kf/restart-ready");
|
||||
let _ = fs::remove_file("/kf/restart-runtime.json");
|
||||
let _ = fs::remove_file("/kf/restart-cleanup");
|
||||
let _ = rustix::process::set_child_subreaper(self.previous);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
#[ignore = "independent capped VM: repeated real owner death reuses the same run and UID only after full recovery"]
|
||||
fn repeated_restarts_reuse_only_fully_reclaimed_identity() {
|
||||
for round in 0..3 {
|
||||
let mut owner = Owner::start("live");
|
||||
assert_eq!(record()["uid"], 100000);
|
||||
assert_eq!(record()["run_id"], "restart-live");
|
||||
owner.crash();
|
||||
assert_eq!(recover()["items"][0]["status"], "reclaimed");
|
||||
owner.finish();
|
||||
println!(
|
||||
"KF_LIFECYCLE_REUSE_PROOF round={round} same_uid=100000 same_run=restart-live actual_user_exec=1 all_resources_gone=1"
|
||||
);
|
||||
drop(owner);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
#[ignore = "independent capped VM: actual capture unlink ENOTEMPTY retains UID after all target processes and root are removed"]
|
||||
fn capture_cleanup_failure_retains_identity_until_restart_repair() {
|
||||
let mut owner = Owner::start("capture");
|
||||
let capture = format!("/var/lib/kf-judge/captures/kf-{}", owner.run_id);
|
||||
// The controller sees the bare mountpoint; the supervisor has a private
|
||||
// mounted tmpfs at the same path. Never delete this foreign file implicitly.
|
||||
fs::write(
|
||||
format!("{capture}/foreign"),
|
||||
b"capture deletion must preserve me",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write("/kf/restart-cleanup", b"now").unwrap();
|
||||
let output = owner.child.take().unwrap().wait_with_output().unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let v: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
|
||||
assert_eq!(v["capture_cleanup_errno"], 39);
|
||||
assert_eq!(v["failure"]["stage"], "exec");
|
||||
assert_eq!(v["cleanup_error"]["stage"], "cleanup");
|
||||
assert_eq!(v["cleanup_error"]["code"], "cleanup_incomplete");
|
||||
assert_eq!(v["user_exec_released"], false);
|
||||
assert_eq!(v["owned_children_finished"], true);
|
||||
owner.reap();
|
||||
assert!(!std::path::Path::new(&owner.group()).exists());
|
||||
assert!(!std::path::Path::new(&owner.task()).exists());
|
||||
assert_eq!(record()["uid"], 100000);
|
||||
assert_eq!(
|
||||
fs::read(format!("{capture}/foreign")).unwrap(),
|
||||
b"capture deletion must preserve me"
|
||||
);
|
||||
let recovery = recover();
|
||||
assert_eq!(recovery["items"][0]["status"], "cleanup_failed");
|
||||
assert_eq!(
|
||||
recovery["items"][0]["cleanup_error"]["code"],
|
||||
"cleanup_incomplete"
|
||||
);
|
||||
assert!(std::path::Path::new(LEASE).exists());
|
||||
fs::remove_file(format!("{capture}/foreign")).unwrap();
|
||||
assert_eq!(recover()["items"][0]["status"], "reclaimed");
|
||||
empty();
|
||||
assert_eq!(recover()["items"], serde_json::json!([]));
|
||||
println!(
|
||||
"KF_LIFECYCLE_CAPTURE_PROOF {}",
|
||||
serde_json::json!({"target_result":v,"reserved_uid":100000,"foreign_preserved":true,"all_owned_pidfds_exited":owner.members.len(),"root_and_group_removed_before_uid_release":true,"fresh_service_recovery_after_explicit_repair":true,"repeat_cleanup_empty":true})
|
||||
);
|
||||
}
|
||||
#[test]
|
||||
#[ignore = "independent capped VM: real SIGKILL and service restart before/after user exec with surviving residual"]
|
||||
fn actual_owner_death_reclaims_prepared_and_live_tasks() {
|
||||
for stage in ["prepared", "live"] {
|
||||
|
||||
Reference in New Issue
Block a user