test(sandbox): verify repeated restart and failed capture lifecycle
This commit is contained in:
@@ -98,7 +98,10 @@ pub(super) fn run_case(case: &str) -> io::Result<serde_json::Value> {
|
|||||||
std::thread::sleep(Duration::from_secs(30));
|
std::thread::sleep(Duration::from_secs(30));
|
||||||
return Ok(serde_json::json!({"residual_deadline":true}));
|
return Ok(serde_json::json!({"residual_deadline":true}));
|
||||||
}
|
}
|
||||||
if !matches!(case, "restart-prepared" | "restart-live") {
|
if !matches!(
|
||||||
|
case,
|
||||||
|
"restart-prepared" | "restart-live" | "restart-capture"
|
||||||
|
) {
|
||||||
return Err(io::Error::other("unknown fixed restart case"));
|
return Err(io::Error::other("unknown fixed restart case"));
|
||||||
}
|
}
|
||||||
let fixture = Fixture::new("/kf/bin/isolation-probe")?;
|
let fixture = Fixture::new("/kf/bin/isolation-probe")?;
|
||||||
@@ -132,13 +135,38 @@ pub(super) fn run_case(case: &str) -> io::Result<serde_json::Value> {
|
|||||||
let (mut lease, dir) = CaptureLease::prepare(&config, &spec)?;
|
let (mut lease, dir) = CaptureLease::prepare(&config, &spec)?;
|
||||||
let (pump, stdio) = crate::io::PipePump::prepare(target.take_stdin(), &spec, dir)?;
|
let (pump, stdio) = crate::io::PipePump::prepare(target.take_stdin(), &spec, dir)?;
|
||||||
target.root_mut().target.prepare_execution(&spec, stdio)?;
|
target.root_mut().target.prepare_execution(&spec, stdio)?;
|
||||||
let _child = ready(&config, &spec, "prepared")?;
|
let capture_case = case == "restart-capture";
|
||||||
|
let _child = ready(
|
||||||
|
&config,
|
||||||
|
&spec,
|
||||||
|
if capture_case { "capture" } else { "prepared" },
|
||||||
|
)?;
|
||||||
let deadline = Instant::now() + Duration::from_secs(15);
|
let deadline = Instant::now() + Duration::from_secs(15);
|
||||||
while Instant::now() < deadline {
|
while Instant::now() < deadline {
|
||||||
|
if capture_case && std::path::Path::new("/kf/restart-cleanup").exists() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
std::thread::sleep(Duration::from_millis(10));
|
std::thread::sleep(Duration::from_millis(10));
|
||||||
}
|
}
|
||||||
drop(pump);
|
drop(pump);
|
||||||
lease.cleanup()?;
|
let capture = lease.cleanup();
|
||||||
|
if capture_case {
|
||||||
|
// The controller owns this exact fixture and repairs its foreign bare
|
||||||
|
// capture entry only after proving the failed UID remains reserved.
|
||||||
|
// Keep the delegated anchor alive for the fresh restart helper.
|
||||||
|
std::mem::forget(fixture);
|
||||||
|
let result = target.refuse_at(crate::outcome::ExecutionStage::Exec, None);
|
||||||
|
let reaped = owned.finish()?;
|
||||||
|
let e = capture
|
||||||
|
.err()
|
||||||
|
.ok_or_else(|| io::Error::other("expected real capture deletion failure"))?;
|
||||||
|
return Ok(serde_json::json!({
|
||||||
|
"case":case,"failure":result.failure,"cleanup_error":result.cleanup_failure,
|
||||||
|
"capture_cleanup_errno":e.raw_os_error(),"reaped_owned_orphans":reaped,
|
||||||
|
"user_exec_released":false,"owned_children_finished":true
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
capture?;
|
||||||
let result = target.refuse_at(crate::outcome::ExecutionStage::Exec, None);
|
let result = target.refuse_at(crate::outcome::ExecutionStage::Exec, None);
|
||||||
owned.finish()?;
|
owned.finish()?;
|
||||||
fs::remove_file("/kf/restart-ready")?;
|
fs::remove_file("/kf/restart-ready")?;
|
||||||
|
|||||||
@@ -217,7 +217,7 @@ fn incomplete_target_cleanup_keeps_identity_reserved() {
|
|||||||
drop(next);
|
drop(next);
|
||||||
assert_eq!(fs::read(task.join("foreign")).unwrap(), b"keep");
|
assert_eq!(fs::read(task.join("foreign")).unwrap(), b"keep");
|
||||||
// VM operator restores the test baseline only after proving both task
|
// VM operator restores the test baseline only after proving both task
|
||||||
// cgroups/children gone. Production restart recovery is still unimplemented.
|
// cgroups/children gone. Fresh-process restart recovery is tested separately.
|
||||||
fs::remove_file(task.join("foreign")).unwrap();
|
fs::remove_file(task.join("foreign")).unwrap();
|
||||||
fs::remove_dir(task).unwrap();
|
fs::remove_dir(task).unwrap();
|
||||||
fs::remove_file(Path::new(LEASES).join("uid-100000/owner.json")).unwrap();
|
fs::remove_file(Path::new(LEASES).join("uid-100000/owner.json")).unwrap();
|
||||||
|
|||||||
@@ -229,10 +229,77 @@ impl Drop for Owner {
|
|||||||
let _ = fs::remove_dir_all(&self.publications);
|
let _ = fs::remove_dir_all(&self.publications);
|
||||||
let _ = fs::remove_file("/kf/restart-ready");
|
let _ = fs::remove_file("/kf/restart-ready");
|
||||||
let _ = fs::remove_file("/kf/restart-runtime.json");
|
let _ = fs::remove_file("/kf/restart-runtime.json");
|
||||||
|
let _ = fs::remove_file("/kf/restart-cleanup");
|
||||||
let _ = rustix::process::set_child_subreaper(self.previous);
|
let _ = rustix::process::set_child_subreaper(self.previous);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#[test]
|
#[test]
|
||||||
|
#[ignore = "independent capped VM: repeated real owner death reuses the same run and UID only after full recovery"]
|
||||||
|
fn repeated_restarts_reuse_only_fully_reclaimed_identity() {
|
||||||
|
for round in 0..3 {
|
||||||
|
let mut owner = Owner::start("live");
|
||||||
|
assert_eq!(record()["uid"], 100000);
|
||||||
|
assert_eq!(record()["run_id"], "restart-live");
|
||||||
|
owner.crash();
|
||||||
|
assert_eq!(recover()["items"][0]["status"], "reclaimed");
|
||||||
|
owner.finish();
|
||||||
|
println!(
|
||||||
|
"KF_LIFECYCLE_REUSE_PROOF round={round} same_uid=100000 same_run=restart-live actual_user_exec=1 all_resources_gone=1"
|
||||||
|
);
|
||||||
|
drop(owner);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
|
#[ignore = "independent capped VM: actual capture unlink ENOTEMPTY retains UID after all target processes and root are removed"]
|
||||||
|
fn capture_cleanup_failure_retains_identity_until_restart_repair() {
|
||||||
|
let mut owner = Owner::start("capture");
|
||||||
|
let capture = format!("/var/lib/kf-judge/captures/kf-{}", owner.run_id);
|
||||||
|
// The controller sees the bare mountpoint; the supervisor has a private
|
||||||
|
// mounted tmpfs at the same path. Never delete this foreign file implicitly.
|
||||||
|
fs::write(
|
||||||
|
format!("{capture}/foreign"),
|
||||||
|
b"capture deletion must preserve me",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::write("/kf/restart-cleanup", b"now").unwrap();
|
||||||
|
let output = owner.child.take().unwrap().wait_with_output().unwrap();
|
||||||
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"{}",
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
let v: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
|
||||||
|
assert_eq!(v["capture_cleanup_errno"], 39);
|
||||||
|
assert_eq!(v["failure"]["stage"], "exec");
|
||||||
|
assert_eq!(v["cleanup_error"]["stage"], "cleanup");
|
||||||
|
assert_eq!(v["cleanup_error"]["code"], "cleanup_incomplete");
|
||||||
|
assert_eq!(v["user_exec_released"], false);
|
||||||
|
assert_eq!(v["owned_children_finished"], true);
|
||||||
|
owner.reap();
|
||||||
|
assert!(!std::path::Path::new(&owner.group()).exists());
|
||||||
|
assert!(!std::path::Path::new(&owner.task()).exists());
|
||||||
|
assert_eq!(record()["uid"], 100000);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(format!("{capture}/foreign")).unwrap(),
|
||||||
|
b"capture deletion must preserve me"
|
||||||
|
);
|
||||||
|
let recovery = recover();
|
||||||
|
assert_eq!(recovery["items"][0]["status"], "cleanup_failed");
|
||||||
|
assert_eq!(
|
||||||
|
recovery["items"][0]["cleanup_error"]["code"],
|
||||||
|
"cleanup_incomplete"
|
||||||
|
);
|
||||||
|
assert!(std::path::Path::new(LEASE).exists());
|
||||||
|
fs::remove_file(format!("{capture}/foreign")).unwrap();
|
||||||
|
assert_eq!(recover()["items"][0]["status"], "reclaimed");
|
||||||
|
empty();
|
||||||
|
assert_eq!(recover()["items"], serde_json::json!([]));
|
||||||
|
println!(
|
||||||
|
"KF_LIFECYCLE_CAPTURE_PROOF {}",
|
||||||
|
serde_json::json!({"target_result":v,"reserved_uid":100000,"foreign_preserved":true,"all_owned_pidfds_exited":owner.members.len(),"root_and_group_removed_before_uid_release":true,"fresh_service_recovery_after_explicit_repair":true,"repeat_cleanup_empty":true})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
#[test]
|
||||||
#[ignore = "independent capped VM: real SIGKILL and service restart before/after user exec with surviving residual"]
|
#[ignore = "independent capped VM: real SIGKILL and service restart before/after user exec with surviving residual"]
|
||||||
fn actual_owner_death_reclaims_prepared_and_live_tasks() {
|
fn actual_owner_death_reclaims_prepared_and_live_tasks() {
|
||||||
for stage in ["prepared", "live"] {
|
for stage in ["prepared", "live"] {
|
||||||
|
|||||||
Reference in New Issue
Block a user