diff --git a/judge/crates/sandbox/src/acceptance/isolation.rs b/judge/crates/sandbox/src/acceptance/isolation.rs index a766d8d..19453aa 100644 --- a/judge/crates/sandbox/src/acceptance/isolation.rs +++ b/judge/crates/sandbox/src/acceptance/isolation.rs @@ -2,11 +2,20 @@ use super::{Fixture, describe}; use crate::{ launcher::run::{self, MonitorMode}, - spec::{ExecutionSpec, InputArtifact, SpecPolicy}, + spec::{ExecutionProfile, ExecutionSpec, InputArtifact, ResourceLimits, SpecPolicy}, }; use std::{fs, io, os::unix::fs::symlink}; pub(super) fn run_case(case: &str) -> io::Result { + if case == "isolation-peer" + || case.strip_prefix("isolation-").is_some_and(|s| { + ["danger-", "identity-", "cross-"] + .iter() + .any(|p| s.starts_with(p)) + }) + { + return matrix_case(case); + } if !matches!(case, "isolation-files" | "isolation-inherited-fd") { return Err(io::Error::other("unknown isolation case")); } @@ -84,3 +93,103 @@ pub(super) fn run_case(case: &str) -> io::Result { value["outside_sentinels_unchanged"] = true.into(); Ok(value) } + +fn matrix_case(case: &str) -> io::Result { + let (mode, profile) = if case == "isolation-peer" { + ("peer", ExecutionProfile::Native) + } else { + let (mode, profile) = case + .strip_prefix("isolation-") + .and_then(|s| s.split_once('-')) + .ok_or_else(|| io::Error::other("fixed matrix case"))?; + if !matches!(mode, "danger" | "identity" | "cross") { + return Err(io::Error::other("fixed matrix mode")); + } + ( + mode, + match profile { + "native" => ExecutionProfile::Native, + "c" => ExecutionProfile::CompileC, + "cpp" => ExecutionProfile::CompileCpp, + _ => return Err(io::Error::other("fixed matrix profile")), + }, + ) + }; + let fixture = Fixture::new("/kf/bin/isolation-probe")?; + let config = fixture.config()?; + let mut request: ExecutionSpec = + serde_json::from_str(include_str!("../../tests/fixtures/spec.valid.json"))?; + request.run_id = case.into(); + request.profile = profile; + request.stdin_artifact = None; + request.limits = ResourceLimits::for_profile(profile); + request.limits.cpu_time_us = 2_000_000; + request.limits.wall_time_us = if mode == "peer" { + 30_000_000 + } else { + 8_000_000 + }; + let label = match profile { + ExecutionProfile::Native => "native", + ExecutionProfile::CompileC => "compile_c", + ExecutionProfile::CompileCpp => "compile_cpp", + }; + if profile == ExecutionProfile::Native { + request.argv.extend([mode.into(), label.into()]); + } else { + // Exercise user-controlled compiler wrappers through the real GNU + // driver. The image probe is fixed; we never replace GCC/G++ itself. + fs::write( + format!("{}/source-001", fixture.publications), + b"int isolated_source;\n", + )?; + request.inputs = vec![InputArtifact { + artifact_id: "source-001".into(), + path: "probe.c".into(), + }]; + request.argv = vec![ + if profile == ExecutionProfile::CompileC { + "/usr/bin/gcc" + } else { + "/usr/bin/g++" + } + .into(), + "-fsyntax-only".into(), + "/work/probe.c".into(), + "-wrapper".into(), + format!("/usr/bin/kf-isolation-probe,{mode},{label}"), + ]; + } + if mode == "cross" { + let frame = fs::read("/kf/isolation-peer-frame")?; + if frame.len() > 512 { + return Err(io::Error::other("peer frame size")); + } + fs::write(format!("{}/peer-001", fixture.publications), frame)?; + request.inputs.push(InputArtifact { + artifact_id: "peer-001".into(), + path: "peer".into(), + }); + } + let spec = request + .validate(&SpecPolicy::default()) + .map_err(io::Error::other)?; + let completed = run::execute( + &config, + &spec, + &std::env::current_exe()?, + &fixture.root()?, + MonitorMode::Normal, + ) + .map_err(|e| io::Error::other(format!("matrix initialization: {:?}", e.failure)))?; + let mut value = describe(case, &spec, &completed, false)?; + let mut stdout = String::new(); + use std::io::Read; + completed + .stdout + .reader()? + .take(2048) + .read_to_string(&mut stdout)?; + value["probe_stdout"] = stdout.into(); + Ok(value) +} diff --git a/judge/crates/sandbox/tests/fixtures/isolation_probe.c b/judge/crates/sandbox/tests/fixtures/isolation_probe.c index 6607240..42d8498 100644 --- a/judge/crates/sandbox/tests/fixtures/isolation_probe.c +++ b/judge/crates/sandbox/tests/fixtures/isolation_probe.c @@ -11,6 +11,9 @@ #include #include #include +#define main isolated_danger_main +#include "seccomp_danger_probe.c" +#undef main static void check(int yes, const char *why) { if (!yes) { fprintf(stderr,"ISOLATION_FAIL %s errno=%d\n",why,errno); exit(1); } } @@ -67,4 +70,65 @@ static void files(void) { errno=0;check(syscall(SYS_chroot,"/")==-1&&errno==EPERM,"chroot reconstruction refused"); printf("FILES_OK proc dev input stdin fd root image\n"); } -int main(int argc,char **argv){check(argc>=2,"fixed mode");if(!strcmp(argv[1],"files")){files();return 0;}return 100;} +static void identity(void) { + check(getuid()>=100000&&getuid()==geteuid()&&getgid()==getegid()&&getgroups(0,NULL)==0,"identity and groups"); + check(prctl(PR_GET_NO_NEW_PRIVS,0,0,0,0)==1&&prctl(PR_GET_SECCOMP)==2,"identity NNP/filter"); + FILE *f=fopen("/proc/self/status","r");check(f!=NULL,"status for caps");char line[256];int caps=0; + while(fgets(line,sizeof line,f))if(!strncmp(line,"Cap",3)){ + char name[32];unsigned long long value; + check(sscanf(line,"%31s %llx",name,&value)==2&&value==0,"all capability sets zero");caps++; + } + fclose(f);check(caps==5,"five capability sets"); +} +static void peer(void) { + identity();int fd=open("peer-secret",O_WRONLY|O_CREAT|O_EXCL,0600);check(fd>=0,"peer secret create"); + check(write(fd,"peer-private-content",20)==20,"peer secret write");close(fd); + fd=open("peer-ready",O_WRONLY|O_CREAT|O_EXCL,0600);check(fd>=0&&write(fd,"ready",5)==5,"peer readiness");close(fd); + struct stat st;while(stat("peer-stop",&st)<0){check(errno==ENOENT,"peer stop state");usleep(10000);} + fd=open("peer-secret",O_RDONLY|O_CLOEXEC);char data[32];check(fd>=0&&read(fd,data,sizeof data)==20&&!memcmp(data,"peer-private-content",20),"peer secret preserved");close(fd); + printf("PEER_OK uid=%u\n",getuid()); +} +static void cross(void) { + identity();FILE *f=fopen("/work/peer","r");check(f!=NULL,"peer metadata"); + int pid,init,parent;unsigned uid;unsigned long long ns[6]; + check(fscanf(f,"%d %d %d %u %llu %llu %llu %llu %llu %llu",&pid,&init,&parent,&uid,&ns[0],&ns[1],&ns[2],&ns[3],&ns[4],&ns[5])==10,"peer frame");fclose(f); + check(pid>2&&init>2&&parent>2&&uid>=100000&&uid!=getuid(),"live peer distinct identity"); + const char *names[]={"mnt","pid","net","ipc","uts","cgroup"};char path[160]; + for(int i=0;i<6;i++){ + snprintf(path,sizeof path,"/proc/self/ns/%s",names[i]);struct stat st;check(!stat(path,&st)&&(unsigned long long)st.st_ino!=ns[i],"distinct peer namespace"); + } + snprintf(path,sizeof path,"/proc/%d/root/work/peer-secret",pid);absent(path); + snprintf(path,sizeof path,"/proc/%d/mem",pid);absent(path); + snprintf(path,sizeof path,"/proc/%d/fd/3",pid);absent(path); + snprintf(path,sizeof path,"/proc/%d/ns/mnt",init);absent(path); + snprintf(path,sizeof path,"/proc/%d/root/kf/isolation-peer-frame",parent);absent(path); + absent("/var/lib/kf-judge/tasks/kf-isolation-peer/root/work/peer-secret"); + DENY(syscall(SYS_kill,pid,0));DENY(syscall(SYS_kill,-1,SIGTERM)); + DENY(syscall(SYS_ptrace,PTRACE_ATTACH,pid,NULL,NULL)); + DENY(syscall(SYS_process_vm_readv,pid,NULL,0,NULL,0,0)); + DENY(syscall(SYS_process_vm_writev,pid,NULL,0,NULL,0,0)); + errno=0;check(syscall(SYS_pidfd_open,pid,0)==-1&&errno==ESRCH,"peer invisible to pidfd"); + int fd=(int)syscall(SYS_pidfd_open,1,0);check(fd>=3,"own visible init pidfd"); + DENY(syscall(SYS_pidfd_getfd,fd,3,0)); + /* GCC may pass its own harmless descriptors to a wrapper. Reuse FD3 + explicitly to test the number-based sendmsg exception on a non-socket. */ + if(fd!=3){check(dup3(fd,3,O_CLOEXEC)==3,"reuse FD3 without control capability");close(fd);fd=3;} + struct msghdr msg={0};errno=0;check(sendmsg(fd,&msg,MSG_NOSIGNAL)==-1&&errno==ENOTSOCK,"FD3 cannot regain control endpoint");close(fd); + DENY(syscall(SYS_symlink,"/proc/1/root","/work/escape"));DENY(syscall(SYS_link,"/work/peer","/work/stolen")); + printf("CROSS_OK peer=%u self=%u denied=%u\n",uid,getuid(),denied); +} +int main(int argc,char **argv){ + check(argc>=2,"fixed mode"); + if(!strcmp(argv[1],"files")){files();return 0;} + if(!strcmp(argv[1],"peer")){peer();return 0;} + if(!strcmp(argv[1],"danger")){char *args[]={argv[0],argc>=3?argv[2]:"native",NULL};return isolated_danger_main(2,args);} + if(!strcmp(argv[1],"cross")){cross();return 0;} + if(!strcmp(argv[1],"identity")){ + identity();struct stat st;struct statvfs v; + check(!stat("/usr/bin/kf-isolation-setuid",&st)&&st.st_uid==0&&(st.st_mode&06000)==06000,"actual setuid/setgid file"); + check(!statvfs("/usr/bin/kf-isolation-setuid",&v)&&(v.f_flag&ST_NOSUID),"image nosuid"); + char *args[]={"/usr/bin/kf-isolation-setuid","post-setuid",NULL};char *env[]={NULL};execve(args[0],args,env);check(0,"setuid exec"); + } + if(!strcmp(argv[1],"post-setuid")){identity();printf("IDENTITY_OK uid=%u euid=%u\n",getuid(),geteuid());return 0;} + return 100; +} diff --git a/judge/crates/sandbox/tests/isolation_matrix_vm.rs b/judge/crates/sandbox/tests/isolation_matrix_vm.rs new file mode 100644 index 0000000..8120d83 --- /dev/null +++ b/judge/crates/sandbox/tests/isolation_matrix_vm.rs @@ -0,0 +1,237 @@ +#![cfg(all(target_os = "linux", feature = "vm-acceptance"))] +#[allow(dead_code)] +mod support; +use std::{ + fs, + io::Write, + os::unix::fs::PermissionsExt, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; + +fn command(case: &str) -> Command { + let mut c = Command::new(support::binary()); + c.args(["vm-case", case]) + .env_clear() + .env("KF_SANDBOX_INDEPENDENT_VM", "1"); + c +} +fn parsed(output: std::process::Output) -> serde_json::Value { + assert!( + output.status.success(), + "{} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let v: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + v["outcome"], + serde_json::json!({"kind":"exited","code":0}), + "{v}" + ); + assert_eq!(v["report"]["cleanup"], "complete"); + assert_eq!(v["report"]["errors"], serde_json::json!([])); + assert_eq!(v["report"]["limits"], serde_json::json!([])); + println!("KF_ISOLATION_MATRIX_PROOF {v}"); + v +} +fn empty() { + for dir in ["tasks", "identities", "captures"] { + assert_eq!( + fs::read_dir(format!("/var/lib/kf-judge/{dir}")) + .unwrap() + .count(), + 0 + ); + } +} +#[test] +#[ignore = "independent capped VM: real compiler wrappers, dangerous syscalls and actual setuid image"] +fn all_profiles_keep_network_namespaces_process_and_privilege_boundaries() { + support::require_vm(); + for profile in ["native", "c", "cpp"] { + let v = parsed( + command(&format!("isolation-danger-{profile}")) + .output() + .unwrap(), + ); + let s = v["probe_stdout"].as_str().unwrap(); + assert!( + s.contains("denied=49") + && s.contains("network=1 ptrace=1 namespaces=1 privilege=1 signal=1 device=1"), + "{s}" + ); + empty(); + let v = parsed( + command(&format!("isolation-identity-{profile}")) + .output() + .unwrap(), + ); + assert_eq!(v["probe_stdout"], "IDENTITY_OK uid=100000 euid=100000\n"); + empty(); + } +} +struct Peer { + child: Option, + work: String, +} +impl Drop for Peer { + fn drop(&mut self) { + if let Some(child) = self.child.as_mut() { + if !self.work.is_empty() { + let _ = fs::write(format!("{}/peer-stop", self.work), b"stop"); + } + let deadline = Instant::now() + Duration::from_secs(10); + while child.try_wait().ok().flatten().is_none() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + if child.try_wait().ok().flatten().is_none() { + let _ = child.kill(); + } + let _ = child.wait(); + } + let _ = fs::remove_file("/kf/isolation-peer-frame"); + } +} +#[test] +#[ignore = "independent capped VM: simultaneously live production tasks with distinct leased UID and six namespaces"] +fn live_peer_files_and_processes_remain_inaccessible_to_every_profile() { + support::require_vm(); + let child = command("isolation-peer") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let owner = child.id(); + let mut peer = Peer { + child: Some(child), + work: String::new(), + }; + let cg = format!("/sys/fs/cgroup/kf-acceptance-{owner}/kf-isolation-peer"); + let deadline = Instant::now() + Duration::from_secs(8); + let (pid, uid, init) = loop { + assert!( + peer.child.as_mut().unwrap().try_wait().unwrap().is_none(), + "peer exited before readiness" + ); + assert!(Instant::now() < deadline, "peer readiness deadline"); + let members = fs::read_to_string(format!("{cg}/cgroup.procs")).unwrap_or_default(); + let mut found = None; + let mut init = 0; + for id in members.split_whitespace() { + let Ok(status) = fs::read_to_string(format!("/proc/{id}/status")) else { + continue; + }; + let ns = status + .lines() + .find_map(|l| l.strip_prefix("NSpid:")) + .unwrap_or("") + .split_whitespace() + .last(); + if ns == Some("1") { + init = id.parse::().unwrap(); + } + if ns == Some("2") + && fs::read(format!("/proc/{id}/root/work/peer-ready")).is_ok_and(|s| s == b"ready") + { + let uid = status + .lines() + .find_map(|l| l.strip_prefix("Uid:")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse::() + .unwrap(); + found = Some((id.parse::().unwrap(), uid)); + } + } + if let Some((pid, uid)) = found + && init > 1 + { + break (pid, uid, init); + } + std::thread::sleep(Duration::from_millis(5)); + }; + assert_eq!(uid, 100000); + peer.work = format!("/proc/{pid}/root/work"); + let pidfd = rustix::process::pidfd_open( + rustix::process::Pid::from_raw(pid as i32).unwrap(), + rustix::process::PidfdFlags::empty(), + ) + .unwrap(); + assert!( + rustix::io::fcntl_getfd(&pidfd) + .unwrap() + .contains(rustix::io::FdFlags::CLOEXEC) + ); + let names = ["mnt", "pid", "net", "ipc", "uts", "cgroup"]; + let mut ids = Vec::new(); + for name in names { + let target = fs::read_link(format!("/proc/{pid}/ns/{name}")).unwrap(); + let s = target.to_str().unwrap(); + ids.push( + s.split_once('[') + .unwrap() + .1 + .trim_end_matches(']') + .parse::() + .unwrap(), + ); + } + let frame = format!( + "{pid} {init} {owner} {uid} {}\n", + ids.iter().map(u64::to_string).collect::>().join(" ") + ); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open("/kf/isolation-peer-frame") + .unwrap(); + file.set_permissions(fs::Permissions::from_mode(0o600)) + .unwrap(); + file.write_all(frame.as_bytes()).unwrap(); + drop(file); + for profile in ["native", "c", "cpp"] { + let v = parsed( + command(&format!("isolation-cross-{profile}")) + .output() + .unwrap(), + ); + assert_eq!( + v["probe_stdout"], + "CROSS_OK peer=100000 self=100001 denied=8\n" + ); + assert!(peer.child.as_mut().unwrap().try_wait().unwrap().is_none()); + assert_eq!( + fs::read(format!("{}/peer-secret", peer.work)).unwrap(), + b"peer-private-content" + ); + assert!( + fs::read_to_string(format!("{cg}/cgroup.events")) + .unwrap() + .contains("populated 1") + ); + assert_eq!( + fs::read_dir("/var/lib/kf-judge/identities") + .unwrap() + .count(), + 1 + ); + } + println!( + "KF_LIVE_PEER_KERNEL_PROOF {}", + serde_json::json!({"owner_pid":owner,"main_pid":pid,"init_pid":init,"uid":uid,"namespaces":ids,"three_attacker_profiles_uid":100001,"secret_unchanged_after_each_attack":true,"peer_cgroup_still_populated":true}) + ); + fs::write(format!("{}/peer-stop", peer.work), b"stop").unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + while peer.child.as_mut().unwrap().try_wait().unwrap().is_none() { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(5)); + } + let v = parsed(peer.child.take().unwrap().wait_with_output().unwrap()); + assert_eq!(v["probe_stdout"], "PEER_OK uid=100000\n"); + drop(pidfd); + drop(peer); + empty(); +}