#![cfg(all(target_os = "linux", feature = "vm-acceptance"))] #[allow(dead_code)] mod support; use rustix::{ fd::{AsFd, OwnedFd}, process::{WaitId, WaitIdOptions}, }; use std::{ fs, os::unix::{ fs::{PermissionsExt, symlink}, process::ExitStatusExt, }, process::{Child, Command, Stdio}, time::{Duration, Instant}, }; const LEASE: &str = "/var/lib/kf-judge/identities/uid-100000/owner.json"; fn command(case: &str) -> Command { let mut c = Command::new(support::binary()); c.args(["vm-case", case]) .env_clear() .env("KF_SANDBOX_INDEPENDENT_VM", "1"); c } fn recover() -> serde_json::Value { let o = command("restart-recover").output().unwrap(); assert!( o.status.success(), "{} {}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr) ); let v: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap(); println!("KF_RESTART_RECOVERY_PROOF {v}"); v } fn record() -> serde_json::Value { serde_json::from_slice(&fs::read(LEASE).unwrap()).unwrap() } fn store(v: &serde_json::Value) { fs::write(LEASE, serde_json::to_vec(v).unwrap()).unwrap(); } fn empty() { for dir in ["tasks", "identities", "captures"] { assert_eq!( fs::read_dir(format!("/var/lib/kf-judge/{dir}")) .unwrap() .count(), 0 ); } } struct Owner { child: Option, cg: String, publications: String, run_id: String, members: Vec, previous: Option, residual: u32, } impl Owner { fn start(stage: &str) -> Self { support::require_vm(); assert!(support::children().is_empty()); let previous = rustix::process::child_subreaper().unwrap(); rustix::process::set_child_subreaper(rustix::process::Pid::from_raw(1)).unwrap(); let _ = fs::remove_file("/kf/restart-ready"); let mut child = command(&format!("restart-{stage}")) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .unwrap(); let owner = child.id(); let cg = format!("/sys/fs/cgroup/kf-acceptance-{owner}"); let deadline = Instant::now() + Duration::from_secs(8); let ready: serde_json::Value = loop { assert!( child.try_wait().unwrap().is_none(), "supervisor exited before ready" ); assert!( Instant::now() < deadline, "crash fixture readiness deadline" ); if let Ok(bytes) = fs::read("/kf/restart-ready") { break serde_json::from_slice(&bytes).unwrap(); } std::thread::sleep(Duration::from_millis(5)); }; assert_eq!(ready["owner_pid"], owner); assert_eq!(ready["stage"], stage); let run_id = ready["run_id"].as_str().unwrap().to_owned(); if stage == "live" { loop { let pids = fs::read_to_string(format!("{cg}/kf-{run_id}/cgroup.procs")).unwrap(); if pids.split_whitespace().any(|p| { fs::read(format!("/proc/{p}/root/work/peer-ready")).is_ok_and(|s| s == b"ready") }) { break; } assert!(Instant::now() < deadline); std::thread::sleep(Duration::from_millis(5)); } } let members = fs::read_to_string(format!("{cg}/kf-{run_id}/cgroup.procs")) .unwrap() .split_whitespace() .map(|n| { rustix::process::pidfd_open( rustix::process::Pid::from_raw(n.parse().unwrap()).unwrap(), rustix::process::PidfdFlags::empty(), ) .unwrap() }) .collect(); let r = record(); assert_eq!(r["owner"]["pid"], owner); assert_eq!(r["version"], 2); assert_eq!(r["service"], "kf-sandbox-task-v2"); for key in ["task", "cgroup", "capture"] { assert!(r["resources"][key]["ino"].as_u64().unwrap() > 0); } let mut cfg: kf_sandbox::runtime::RuntimeConfig = serde_json::from_str(include_str!( "../../../../deploy/sandbox.runtime.example.json" )) .unwrap(); cfg.delegated_cgroup_root = cg.clone(); fs::write( "/kf/restart-runtime.json", serde_json::to_vec(&cfg).unwrap(), ) .unwrap(); fs::set_permissions( "/kf/restart-runtime.json", fs::Permissions::from_mode(0o600), ) .unwrap(); Self { child: Some(child), cg, publications: format!("/kf/execution-publications-{owner}"), run_id, members, previous, residual: ready["residual_pid"].as_u64().unwrap() as u32, } } fn crash(&mut self) { self.child.as_mut().unwrap().kill().unwrap(); assert_eq!(self.child.take().unwrap().wait().unwrap().signal(), Some(9)); assert!( fs::read_to_string(format!("/proc/{}/stat", self.residual)).is_ok(), "real partial-preparation residual must survive owner SIGKILL" ); } fn group(&self) -> String { format!("{}/kf-{}", self.cg, self.run_id) } fn task(&self) -> String { format!("/var/lib/kf-judge/tasks/kf-{}", self.run_id) } fn reap(&self) { let deadline = Instant::now() + Duration::from_secs(3); for fd in &self.members { loop { match rustix::process::waitid( WaitId::PidFd(fd.as_fd()), WaitIdOptions::EXITED | WaitIdOptions::NOHANG, ) { Ok(Some(_)) | Err(rustix::io::Errno::CHILD) => break, Ok(None) => { assert!(Instant::now() < deadline); std::thread::sleep(Duration::from_millis(5)); } Err(e) => panic!("owned pidfd wait: {e}"), } } let mut p = [rustix::event::PollFd::new(fd, rustix::event::PollFlags::IN)]; rustix::event::poll( &mut p, Some(&rustix::event::Timespec { tv_sec: 0, tv_nsec: 0, }), ) .unwrap(); assert!( p[0].revents() .intersects(rustix::event::PollFlags::IN | rustix::event::PollFlags::HUP) ); } assert!( support::children().is_empty(), "all owned adopted children reaped" ); } fn finish(&self) { self.reap(); assert!(!std::path::Path::new(&self.group()).exists()); empty(); assert_eq!(recover()["items"], serde_json::json!([])); println!( "KF_RESTART_KERNEL_PROOF {}", serde_json::json!({"run_id":self.run_id,"residual_pid":self.residual,"owner_killed_sigkill":true,"kernel_pidfd_exits_verified":self.members.len(),"only_owned_adopted_children_reaped":true,"resources_removed":true,"repeat_cleanup_empty":true}) ); } } impl Drop for Owner { fn drop(&mut self) { if let Some(child) = self.child.as_mut() { let _ = child.kill(); let _ = child.wait(); } let _ = fs::write(format!("{}/cgroup.kill", self.cg), "1"); let deadline = Instant::now() + Duration::from_secs(3); while Instant::now() < deadline { match rustix::process::wait(rustix::process::WaitOptions::NOHANG) { Ok(Some(_)) => {} Err(rustix::io::Errno::CHILD) => break, _ => std::thread::sleep(Duration::from_millis(5)), } } let _ = fs::remove_dir(self.group()); let _ = fs::remove_dir(&self.cg); // Fixture input publications belong to the trusted caller, not to // sandbox recovery. Remove only this known test-owned publication. let _ = fs::remove_dir_all(&self.publications); let _ = fs::remove_file("/kf/restart-ready"); let _ = fs::remove_file("/kf/restart-runtime.json"); let _ = fs::remove_file("/kf/restart-cleanup"); let _ = rustix::process::set_child_subreaper(self.previous); } } #[test] #[ignore = "independent capped VM: repeated real owner death reuses the same run and UID only after full recovery"] fn repeated_restarts_reuse_only_fully_reclaimed_identity() { for round in 0..3 { let mut owner = Owner::start("live"); assert_eq!(record()["uid"], 100000); assert_eq!(record()["run_id"], "restart-live"); owner.crash(); assert_eq!(recover()["items"][0]["status"], "reclaimed"); owner.finish(); println!( "KF_LIFECYCLE_REUSE_PROOF round={round} same_uid=100000 same_run=restart-live actual_user_exec=1 all_resources_gone=1" ); drop(owner); } } #[test] #[ignore = "independent capped VM: actual capture unlink ENOTEMPTY retains UID after all target processes and root are removed"] fn capture_cleanup_failure_retains_identity_until_restart_repair() { let mut owner = Owner::start("capture"); let capture = format!("/var/lib/kf-judge/captures/kf-{}", owner.run_id); // The controller sees the bare mountpoint; the supervisor has a private // mounted tmpfs at the same path. Never delete this foreign file implicitly. fs::write( format!("{capture}/foreign"), b"capture deletion must preserve me", ) .unwrap(); fs::write("/kf/restart-cleanup", b"now").unwrap(); let output = owner.child.take().unwrap().wait_with_output().unwrap(); assert!( output.status.success(), "{}", String::from_utf8_lossy(&output.stderr) ); let v: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); assert_eq!(v["capture_cleanup_errno"], 39); assert_eq!(v["failure"]["stage"], "exec"); assert_eq!(v["cleanup_error"]["stage"], "cleanup"); assert_eq!(v["cleanup_error"]["code"], "cleanup_incomplete"); assert_eq!(v["user_exec_released"], false); assert_eq!(v["owned_children_finished"], true); owner.reap(); assert!(!std::path::Path::new(&owner.group()).exists()); assert!(!std::path::Path::new(&owner.task()).exists()); assert_eq!(record()["uid"], 100000); assert_eq!( fs::read(format!("{capture}/foreign")).unwrap(), b"capture deletion must preserve me" ); let recovery = recover(); assert_eq!(recovery["items"][0]["status"], "cleanup_failed"); assert_eq!( recovery["items"][0]["cleanup_error"]["code"], "cleanup_incomplete" ); assert!(std::path::Path::new(LEASE).exists()); fs::remove_file(format!("{capture}/foreign")).unwrap(); assert_eq!(recover()["items"][0]["status"], "reclaimed"); empty(); assert_eq!(recover()["items"], serde_json::json!([])); println!( "KF_LIFECYCLE_CAPTURE_PROOF {}", serde_json::json!({"target_result":v,"reserved_uid":100000,"foreign_preserved":true,"all_owned_pidfds_exited":owner.members.len(),"root_and_group_removed_before_uid_release":true,"fresh_service_recovery_after_explicit_repair":true,"repeat_cleanup_empty":true}) ); } #[test] #[ignore = "independent capped VM: real SIGKILL and service restart before/after user exec with surviving residual"] fn actual_owner_death_reclaims_prepared_and_live_tasks() { for stage in ["prepared", "live"] { let mut owner = Owner::start(stage); assert_eq!(recover()["items"][0]["status"], "active"); assert!(owner.child.as_mut().unwrap().try_wait().unwrap().is_none()); owner.crash(); let v = recover(); assert_eq!(v["items"][0]["status"], "reclaimed", "{v}"); assert!(v["items"][0]["stopped_processes"].as_u64().unwrap() > 0); owner.finish(); drop(owner); empty(); } } #[test] #[ignore = "independent capped VM: failed resource deletion must preserve separate error, UID and foreign bytes"] fn cleanup_failure_is_independent_and_reserved_until_explicit_repair() { let mut owner = Owner::start("live"); fs::write( format!("{}/foreign", owner.task()), b"preserve this foreign content", ) .unwrap(); owner.crash(); let v = recover(); assert_eq!(v["items"][0]["status"], "cleanup_failed"); assert_eq!(v["items"][0]["cleanup_error"]["stage"], "cleanup"); assert_eq!(v["items"][0]["cleanup_error"]["code"], "cleanup_incomplete"); assert_eq!( fs::read(format!("{}/foreign", owner.task())).unwrap(), b"preserve this foreign content" ); assert!(std::path::Path::new(LEASE).exists()); assert!(!std::path::Path::new(&owner.group()).exists()); owner.reap(); assert_eq!(recover()["items"][0]["status"], "cleanup_failed"); fs::remove_file(format!("{}/foreign", owner.task())).unwrap(); assert_eq!(recover()["items"][0]["status"], "reclaimed"); owner.finish(); } #[test] #[ignore = "independent capped VM: replaced cgroup inode, symlink, boot/PID identity and unknown markers stay protected"] fn replaced_resources_and_unrecognized_markers_are_not_adopted() { let mut owner = Owner::start("prepared"); owner.crash(); let saved = record(); // A reused owner PID/start pair is live even without a surviving flock. let stat = fs::read_to_string("/proc/self/stat").unwrap(); let ticks = stat .rsplit_once(") ") .unwrap() .1 .split_whitespace() .nth(19) .unwrap() .parse::() .unwrap(); let mut v = saved.clone(); v["owner"]["pid"] = std::process::id().into(); v["owner"]["start_ticks"] = ticks.into(); store(&v); assert_eq!(recover()["items"][0]["status"], "active"); store(&saved); v = saved.clone(); v["owner"]["boot_id"] = "00000000-0000-0000-0000-000000000000".into(); store(&v); assert_eq!(recover()["items"][0]["status"], "cleanup_failed"); store(&saved); // Startup must inspect a foreign FIFO without reading or blocking on it. let held_record = format!("{LEASE}.held"); fs::rename(LEASE, &held_record).unwrap(); rustix::fs::mknodat( rustix::fs::CWD, LEASE, rustix::fs::FileType::Fifo, rustix::fs::Mode::from_raw_mode(0o600), 0, ) .unwrap(); let before = Instant::now(); assert_eq!(recover()["items"][0]["status"], "unrecognized"); assert!(before.elapsed() < Duration::from_secs(2)); fs::remove_file(LEASE).unwrap(); fs::rename(held_record, LEASE).unwrap(); v = saved.clone(); v["resources"]["cgroup"] = serde_json::Value::Null; store(&v); assert_eq!(recover()["items"][0]["status"], "cleanup_failed"); store(&saved); // cgroup v2 deliberately forbids rename. The VM operator first stops and // removes its known original fixture, then creates a genuinely different // cgroup at the same name while retaining the old recorded inode. fs::write(format!("{}/cgroup.kill", owner.group()), "1").unwrap(); owner.reap(); fs::remove_dir(owner.group()).unwrap(); fs::create_dir(owner.group()).unwrap(); fs::set_permissions(owner.group(), fs::Permissions::from_mode(0o700)).unwrap(); let mut foreign = command("restart-residual") .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() .unwrap(); fs::write( format!("{}/cgroup.procs", owner.group()), foreign.id().to_string(), ) .unwrap(); assert_eq!(recover()["items"][0]["status"], "cleanup_failed"); assert!(foreign.try_wait().unwrap().is_none()); foreign.kill().unwrap(); foreign.wait().unwrap(); fs::remove_dir(owner.group()).unwrap(); let old_task = format!("{}-held", owner.task()); fs::rename(owner.task(), &old_task).unwrap(); fs::create_dir("/kf/protected-restart-directory").unwrap(); fs::write( "/kf/protected-restart-directory/foreign", b"do not traverse", ) .unwrap(); symlink("/kf/protected-restart-directory", owner.task()).unwrap(); assert_eq!(recover()["items"][0]["status"], "cleanup_failed"); assert_eq!( fs::read("/kf/protected-restart-directory/foreign").unwrap(), b"do not traverse" ); fs::remove_file(owner.task()).unwrap(); fs::rename(old_task, owner.task()).unwrap(); fs::remove_file("/kf/protected-restart-directory/foreign").unwrap(); fs::remove_dir("/kf/protected-restart-directory").unwrap(); let unknown = "/var/lib/kf-judge/identities/uid-100001"; fs::create_dir(unknown).unwrap(); fs::set_permissions(unknown, fs::Permissions::from_mode(0o700)).unwrap(); let legacy = b"{\"version\":1,\"run_id\":\"foreign\",\"uid\":100001,\"gid\":100001}"; fs::write(format!("{unknown}/owner.json"), legacy).unwrap(); fs::set_permissions( format!("{unknown}/owner.json"), fs::Permissions::from_mode(0o600), ) .unwrap(); let v = recover(); assert_eq!(v["items"][0]["status"], "reclaimed"); assert_eq!(v["items"][1]["status"], "unrecognized"); assert_eq!(fs::read(format!("{unknown}/owner.json")).unwrap(), legacy); fs::remove_file(format!("{unknown}/owner.json")).unwrap(); fs::remove_dir(unknown).unwrap(); owner.finish(); println!( "KF_RESTART_OWNERSHIP_PROOF live_owner_pid_start=1 boot_mismatch_refused=1 unrecorded_resource_refused=1 fifo_not_opened=1 replaced_cgroup_and_process_untouched=1 symlink_not_followed=1 legacy_marker_untouched=1" ); }