diff --git a/judge/README.md b/judge/README.md index f032dce..9f3090e 100644 --- a/judge/README.md +++ b/judge/README.md @@ -127,6 +127,37 @@ FD、栈和其他信号仍按原退出状态记录，缺少专门证据时不猜 默认仍使用 Cargo 构建的路径。此变量只由测试读取，启动器子进程仍清空环境。 在 VM 内直接验证 GNU 工具链或这些控制测试，只能证明测试载荷可用，不能作为沙箱隔离验收。 +## 真实目标阻塞与任务 cgroup + +`launcher/blocked.rs` 通过清空环境的 `Command` 新建可信 bootstrap 进程，bootstrap +在单线程入口设置父线程死亡时的 SIGKILL、核对实际父 PID，再自行 SIGSTOP。父端持有 +唯一 Child 和 pidfd，用 waitid/NOWAIT 及实际进程状态确认停止；调用方须保留默认 SIGCHLD +处理（包括不使用 SA_NOCLDWAIT），不能由其他线程/信号处理器回收该 Child。代码额外拒绝 +可观察到的 SIGCHLD 忽略/自定义处理。没有自定义 pre_exec 或 Rust fork 子进程初始化。 + +`cgroup.rs` 只接受可信运行配置中的委派子树，逐级 NOFOLLOW 打开并验证 cgroup v2、 +helper 所有权、不可被组/其他用户写入、空 domain 及必需控制器。它设置并读回总体与每任务的 +memory.max、交换空间 0、pids.max、cpu.max 和 memory.oom.group，限制任务目录数和层级。 +每任务使用排他 mkdir，不采用已存在目录。CPU 带宽只是节流；CPU 总时间监督仍未实现。 + +`PreparedTarget` 将真实停止的 bootstrap 纳入已配置限额、已冻结的任务 cgroup，重复核对 +限额、成员和冻结状态。当前没有 thaw/continue/exec 公共方法，不使用请求中的 argv 启动用户程序； +启动器二进制路径来自可信调用方。缺少隔离时明确返回 namespaces/capability_unavailable， +通过 cgroup.kill 和 pidfd 终止并回收后返回，清理错误单独保留。部署方仍须认证配置文件和 +实际委派；尚未安装权限 helper、接入完整监督器或启用 CLI run。现有控制入口的 Initialize +仍返回准备拒绝。未来隔离初始化与最终 exec 放行需要独立的验证和控制阶段。 + +普通 `make check-judge` 对 `blocked_vm.rs` 的权限用例明确报告 ignored。仅在无宿主共享挂载、 +无业务凭据且外部已验证整体资源上限的独立 QEMU VM 内，由可信运行者设置 +`KF_SANDBOX_INDEPENDENT_VM=1` 和启动器路径，执行测试二进制的 +`--ignored --test-threads=1 --nocapture`。测试还要求 VM 内的专用标记、QEMU 厂商信息、 +网络仅 lo 及无宿主路径；不具备条件时拒绝，不能将 ignored 计为验收通过。 +固定 C 探针仅在该 VM 中验证真实内存 OOM、fork 拒绝与 CPU 节流，测试过程没有执行提交程序。 + +参见 [cgroup v2](https://docs.kernel.org/admin-guide/cgroup-v2.html)、 +[pidfd_open](https://man7.org/linux/man-pages/man2/pidfd_open.2.html)、 +[PR_SET_PDEATHSIG](https://man7.org/linux/man-pages/man2/PR_SET_PDEATHSIG.2const.html)。 + ## 可信运行配置与启动前检查 ```bash diff --git a/judge/crates/sandbox/Cargo.toml b/judge/crates/sandbox/Cargo.toml index 423b393..c8230ea 100644 --- a/judge/crates/sandbox/Cargo.toml +++ b/judge/crates/sandbox/Cargo.toml @@ -10,7 +10,7 @@ kf-contracts.workspace = true clap.workspace = true serde.workspace = true serde_json.workspace = true -rustix = { version = "1.1.5", features = ["net", "process"] } +rustix = { version = "1.1.5", features = ["net", "process", "fs"] } [lints] workspace = true diff --git a/judge/crates/sandbox/src/cgroup.rs b/judge/crates/sandbox/src/cgroup.rs new file mode 100644 index 0000000..d7311c5 --- /dev/null +++ b/judge/crates/sandbox/src/cgroup.rs @@ -0,0 +1,412 @@ +//! Descriptor-relative management of one operator-delegated cgroup v2 subtree. +//! No request controls host paths; this module never changes the system root. + +use crate::launcher::blocked::BlockedTarget; +use crate::runtime::ValidatedRuntimeConfig; +use crate::spec::{SpecPolicy, ValidatedSpec}; +use rustix::fd::OwnedFd; +use rustix::fs::{AtFlags, Mode, OFlags}; +use std::fs::File; +use std::io::{self, Read, Write}; +use std::path::{Component, Path}; +use std::time::{Duration, Instant}; + +const CGROUP2_MAGIC: u64 = 0x6367_7270; +const MAX_INTERFACE_BYTES: u64 = 4096; +const DIR_FLAGS: OFlags = OFlags::RDONLY + .union(OFlags::DIRECTORY) + .union(OFlags::NOFOLLOW) + .union(OFlags::CLOEXEC); + +#[derive(Debug)] +pub struct CgroupError { + pub source: io::Error, + pub cleanup_error: Option, +} + +impl From for CgroupError { + fn from(source: io::Error) -> Self { + Self { + source, + cleanup_error: None, + } + } +} + +impl std::fmt::Display for CgroupError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "cgroup operation failed: {:?}", self.source.kind()) + } +} +impl std::error::Error for CgroupError {} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AppliedLimits { + pub memory_bytes: u64, + pub swap_bytes: u64, + pub processes: u32, + pub cpu_quota_us: u64, + pub cpu_period_us: u64, + pub oom_group: bool, +} + +pub struct DelegatedCgroup { + fd: OwnedFd, + policy: SpecPolicy, + quota: u64, + period: u64, +} + +impl DelegatedCgroup { + /// Config comes from the operator. Structural validation is not proof of + /// config-file ownership, service delegation or an exec permit. + /// The root must be empty, owned by this helper and not group/other writable. + pub fn open(config: &ValidatedRuntimeConfig) -> Result { + let config_data = config.config(); + let fd = open_directory(Path::new(&config_data.delegated_cgroup_root))?; + let stat = rustix::fs::fstat(&fd).map_err(io::Error::from)?; + if stat.st_uid != rustix::process::geteuid().as_raw() || stat.st_mode & 0o022 != 0 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "unsafe delegated owner/mode", + ) + .into()); + } + if rustix::fs::fstatfs(&fd).map_err(io::Error::from)?.f_type as u64 != CGROUP2_MAGIC { + return Err(invalid("not a cgroup v2 directory").into()); + } + if read(&fd, "cgroup.type")?.trim() != "domain" + || !read(&fd, "cgroup.procs")?.trim().is_empty() + { + return Err(invalid("delegated root must be an empty domain").into()); + } + let controllers = read(&fd, "cgroup.controllers")?; + if !["cpu", "memory", "pids"] + .iter() + .all(|name| controllers.split_whitespace().any(|value| value == *name)) + { + return Err(invalid("required controllers unavailable").into()); + } + write(&fd, "cgroup.subtree_control", "+cpu +memory +pids")?; + let enabled = read(&fd, "cgroup.subtree_control")?; + if !["cpu", "memory", "pids"] + .iter() + .all(|name| enabled.split_whitespace().any(|value| value == *name)) + { + return Err(invalid("required controllers not enabled").into()); + } + // Bound concurrent task directories and prevent deeper cgroup creation. + write_verify(&fd, "cgroup.max.depth", "1")?; + write_verify( + &fd, + "cgroup.max.descendants", + &config_data.max_concurrent_runs.to_string(), + )?; + let budget = &config_data.aggregate; + let aggregate = AppliedLimits { + memory_bytes: budget.memory_bytes, + swap_bytes: 0, + processes: budget.processes, + cpu_quota_us: budget.cpu_quota_us, + cpu_period_us: budget.cpu_period_us, + oom_group: true, + }; + apply_limits(&fd, &aggregate)?; + Ok(Self { + fd, + policy: config.spec_policy(), + quota: budget.task_cpu_quota_us, + period: budget.cpu_period_us, + }) + } + + /// Always mkdir exclusive: never adopt, overwrite or reuse an existing task. + pub fn create_task(&self, spec: &ValidatedSpec) -> Result { + spec.spec() + .clone() + .validate(&self.policy) + .map_err(|_| invalid("request exceeds delegated policy"))?; + let name = format!("kf-{}", spec.spec().run_id); + rustix::fs::mkdirat(&self.fd, name.as_str(), Mode::from_raw_mode(0o700)) + .map_err(io::Error::from)?; + let opened = rustix::fs::openat(&self.fd, name.as_str(), DIR_FLAGS, Mode::empty()); + let fd = match opened { + Ok(fd) => fd, + Err(error) => { + return Err(CgroupError { + source: error.into(), + cleanup_error: rustix::fs::unlinkat( + &self.fd, + name.as_str(), + AtFlags::REMOVEDIR, + ) + .err() + .map(Into::into), + }); + } + }; + let parent = match rustix::io::fcntl_dupfd_cloexec(&self.fd, 3) { + Ok(fd) => fd, + Err(error) => { + return Err(CgroupError { + source: error.into(), + cleanup_error: rustix::fs::unlinkat( + &self.fd, + name.as_str(), + AtFlags::REMOVEDIR, + ) + .err() + .map(Into::into), + }); + } + }; + let limits = AppliedLimits { + memory_bytes: spec.spec().limits.memory_bytes, + swap_bytes: 0, + processes: spec.spec().limits.processes, + cpu_quota_us: self.quota, + cpu_period_us: self.period, + oom_group: true, + }; + let mut task = TaskCgroup { + parent, + fd, + name, + limits, + removed: false, + }; + let setup = (|| { + if read(&task.fd, "cgroup.type")?.trim() != "domain" { + return Err(invalid("invalid task domain")); + } + apply_limits(&task.fd, &task.limits)?; + write_verify(&task.fd, "cgroup.max.depth", "0")?; + write_verify(&task.fd, "cgroup.max.descendants", "0")?; + // Make fatal termination cover the entire task and retain a kernel + // freeze gate even if somebody sends the bootstrap SIGCONT. + write_verify(&task.fd, "cgroup.freeze", "1")?; + task.wait_event("frozen", "1", Duration::from_secs(1))?; + Ok::<_, io::Error>(()) + })(); + if let Err(source) = setup { + let cleanup_error = task.kill_and_remove(Duration::from_secs(1)).err(); + return Err(CgroupError { + source, + cleanup_error, + }); + } + Ok(task) + } +} + +pub struct TaskCgroup { + parent: OwnedFd, + fd: OwnedFd, + name: String, + limits: AppliedLimits, + removed: bool, +} + +impl TaskCgroup { + pub fn name(&self) -> &str { + &self.name + } + + pub fn applied_limits(&self) -> io::Result { + self.live()?; + verify_limits(&self.fd, &self.limits)?; + Ok(self.limits.clone()) + } + + /// PID comes from the unreaped direct Child; callers cannot enroll arbitrary PIDs. + pub fn enroll(&mut self, target: &mut BlockedTarget) -> Result<(), CgroupError> { + self.live()?; + target.verify_stopped()?; + self.applied_limits()?; + self.wait_event("frozen", "1", Duration::from_secs(1))?; + if !read(&self.fd, "cgroup.procs")?.trim().is_empty() { + return Err(invalid("task already populated").into()); + } + write(&self.fd, "cgroup.procs", &target.pid().to_string())?; + let members = read(&self.fd, "cgroup.procs")?; + if members.trim() != target.pid().to_string() { + return Err(invalid("task enrollment mismatch").into()); + } + target.verify_stopped()?; + self.wait_event("frozen", "1", Duration::from_secs(1))?; + Ok(()) + } + + /// Entire group must be empty before descriptor-relative removal. Repeated + /// cleanup is harmless; explicit calls report errors independently. + pub fn kill_and_remove(&mut self, timeout: Duration) -> io::Result<()> { + if self.removed { + return Ok(()); + } + if timeout.is_zero() || timeout > Duration::from_secs(5) { + return Err(invalid("invalid cleanup deadline")); + } + write(&self.fd, "cgroup.kill", "1")?; + self.wait_event("populated", "0", timeout)?; + rustix::fs::unlinkat(&self.parent, self.name.as_str(), AtFlags::REMOVEDIR)?; + self.removed = true; + Ok(()) + } + + fn live(&self) -> io::Result<()> { + if self.removed { + Err(io::Error::new( + io::ErrorKind::NotFound, + "task cgroup removed", + )) + } else { + Ok(()) + } + } + + fn wait_event(&self, key: &str, expected: &str, timeout: Duration) -> io::Result<()> { + let deadline = Instant::now() + timeout; + loop { + let events = read(&self.fd, "cgroup.events")?; + if events + .lines() + .any(|line| line.split_once(' ') == Some((key, expected))) + { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "cgroup event deadline", + )); + } + std::thread::sleep(Duration::from_millis(2)); + } + } +} + +impl Drop for TaskCgroup { + fn drop(&mut self) { + if !self.removed { + let _ = self.kill_and_remove(Duration::from_millis(250)); + } + } +} + +fn open_directory(path: &Path) -> io::Result { + // The validated root is absolute and strictly below /sys/fs/cgroup. Walk + // every component with NOFOLLOW; a symlink ancestor cannot redirect us. + if !path.starts_with("/sys/fs/cgroup") || path == Path::new("/sys/fs/cgroup") { + return Err(invalid("not a delegated cgroup subtree")); + } + let mut fd = rustix::fs::open("/", DIR_FLAGS, Mode::empty())?; + for component in path.components() { + match component { + Component::RootDir => {} + Component::Normal(name) => { + fd = rustix::fs::openat(&fd, name, DIR_FLAGS, Mode::empty())? + } + _ => return Err(invalid("noncanonical cgroup root")), + } + } + Ok(fd) +} + +fn read(fd: &OwnedFd, name: &str) -> io::Result { + let file = rustix::fs::openat( + fd, + name, + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?; + let mut text = String::new(); + File::from(file) + .take(MAX_INTERFACE_BYTES + 1) + .read_to_string(&mut text)?; + if text.len() as u64 > MAX_INTERFACE_BYTES { + return Err(invalid("cgroup interface too large")); + } + Ok(text) +} + +fn write(fd: &OwnedFd, name: &str, value: &str) -> io::Result<()> { + let file = rustix::fs::openat( + fd, + name, + OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?; + // Kernfs values (not streams): one complete write, never retry a short suffix. + if File::from(file).write(value.as_bytes())? != value.len() { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + "short cgroup write", + )); + } + Ok(()) +} + +fn write_verify(fd: &OwnedFd, name: &str, value: &str) -> io::Result<()> { + write(fd, name, value)?; + if read(fd, name)?.trim() != value { + return Err(invalid("cgroup readback mismatch")); + } + Ok(()) +} + +fn apply_limits(fd: &OwnedFd, limits: &AppliedLimits) -> io::Result<()> { + write_verify(fd, "memory.max", &limits.memory_bytes.to_string())?; + write_verify(fd, "memory.swap.max", &limits.swap_bytes.to_string())?; + write_verify( + fd, + "memory.oom.group", + if limits.oom_group { "1" } else { "0" }, + )?; + write_verify(fd, "pids.max", &limits.processes.to_string())?; + write_verify( + fd, + "cpu.max", + &format!("{} {}", limits.cpu_quota_us, limits.cpu_period_us), + ) +} + +fn verify_limits(fd: &OwnedFd, limits: &AppliedLimits) -> io::Result<()> { + for (name, expected) in [ + ("memory.max", limits.memory_bytes.to_string()), + ("memory.swap.max", limits.swap_bytes.to_string()), + ("pids.max", limits.processes.to_string()), + ( + "memory.oom.group", + if limits.oom_group { "1" } else { "0" }.into(), + ), + ( + "cpu.max", + format!("{} {}", limits.cpu_quota_us, limits.cpu_period_us), + ), + ] { + if read(fd, name)?.trim() != expected { + return Err(invalid("configured limits changed")); + } + } + Ok(()) +} + +fn invalid(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, message) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn system_root_and_host_paths_are_rejected_before_open() { + for path in [ + "/sys/fs/cgroup", + "/tmp/cgroup", + "/sys/fs/cgroup/../cgroup", + "/sys/fs/cgroup/./task", + ] { + assert!(open_directory(Path::new(path)).is_err()); + } + } +} diff --git a/judge/crates/sandbox/src/launcher/blocked.rs b/judge/crates/sandbox/src/launcher/blocked.rs new file mode 100644 index 0000000..ff172f2 --- /dev/null +++ b/judge/crates/sandbox/src/launcher/blocked.rs @@ -0,0 +1,285 @@ +//! Real kernel stop before task enrollment. There is deliberately no exec/release API. + +use crate::cgroup::{DelegatedCgroup, TaskCgroup}; +use crate::outcome::{ErrorCode, ExecutionError, ExecutionStage}; +use crate::runtime::ValidatedRuntimeConfig; +use crate::spec::ValidatedSpec; +use rustix::fd::{AsFd, OwnedFd}; +use rustix::process::{Pid, PidfdFlags, Signal, WaitId, WaitIdOptions}; +use std::io; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +const STARTUP: Duration = Duration::from_secs(5); +const CLEANUP: Duration = Duration::from_secs(2); + +/// Runs only after a fresh exec, without custom pre_exec/fork callbacks. A parent +/// dying before/during prctl is caught by the subsequent kernel parent check. +/// PDEATHSIG refers to the spawning thread: the supervisor must keep it alive. +pub fn bootstrap(expected_parent: u32) -> io::Result<()> { + super::entry::ensure_single_threaded()?; + rustix::process::set_parent_process_death_signal(Some(Signal::KILL))?; + let parent = rustix::process::getppid(); + if expected_parent <= 1 + || parent.map(|pid| pid.as_raw_pid() as u32) != Some(expected_parent) + || rustix::process::parent_process_death_signal()? != Some(Signal::KILL) + { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "bootstrap parent mismatch", + )); + } + rustix::process::kill_process(rustix::process::getpid(), Signal::STOP)?; + // An unexpected SIGCONT cannot run initialization or a user command. Future + // isolation initialization and the separate final exec gate are still absent. + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "bootstrap has no execution permit", + )) +} + +/// Sole child owner; callers must not ignore SIGCHLD or reap this child elsewhere. +/// pidfd is obtained before any wait; NOWAIT leaves reaping to the Child handle. +pub struct BlockedTarget { + child: Child, + pidfd: OwnedFd, + reaped: bool, +} + +impl BlockedTarget { + /// `executable` is the operator's trusted sandbox binary, never a request argv. + pub fn spawn(executable: &Path) -> io::Result { + // A ignored/caught SIGCHLD could reap/recycle a PID before pidfd_open. + // Fresh helpers retain the default disposition; reject inherited changes. + let status = std::fs::read_to_string("/proc/self/status")?; + for key in ["SigIgn:", "SigCgt:"] { + let bits = status + .lines() + .find_map(|line| line.strip_prefix(key)) + .and_then(|value| u64::from_str_radix(value.trim(), 16).ok()) + .ok_or_else(|| io::Error::other("cannot verify SIGCHLD policy"))?; + if bits & (1 << (rustix::process::Signal::CHILD.as_raw() - 1)) != 0 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "SIGCHLD must have default disposition", + )); + } + } + let mut child = Command::new(executable) + .arg("bootstrap") + .arg("--parent-pid") + .arg(std::process::id().to_string()) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + let Some(pid) = Pid::from_raw(child.id() as i32) else { + let _ = child.kill(); + let _ = child.wait(); + return Err(io::Error::other("invalid child PID")); + }; + let pidfd = match rustix::process::pidfd_open(pid, PidfdFlags::empty()) { + Ok(fd) => fd, + Err(error) => { + // The unreaped direct child cannot have its PID recycled. + let _ = child.kill(); + let _ = child.wait(); + return Err(error.into()); + } + }; + let mut target = Self { + child, + pidfd, + reaped: false, + }; + let deadline = Instant::now() + STARTUP; + loop { + let state = rustix::process::waitid( + WaitId::PidFd(target.pidfd.as_fd()), + WaitIdOptions::STOPPED + | WaitIdOptions::EXITED + | WaitIdOptions::NOHANG + | WaitIdOptions::NOWAIT, + )?; + if let Some(state) = state { + if state.stopping_signal() == Some(Signal::STOP.as_raw()) { + target.verify_stopped()?; + return Ok(target); + } + return Err(io::Error::other("bootstrap exited before stop")); + } + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "bootstrap stop deadline", + )); + } + std::thread::sleep(Duration::from_millis(2)); + } + } + + pub fn pid(&self) -> u32 { + self.child.id() + } + + pub fn poll_exit(&mut self) -> io::Result> { + let status = self.child.try_wait()?; + if status.is_some() { + self.reaped = true; + } + Ok(status) + } + + pub fn verify_stopped(&mut self) -> io::Result<()> { + if self.reaped || self.child.try_wait()?.is_some() { + self.reaped = true; + return Err(io::Error::other("bootstrap is not alive")); + } + let status = std::fs::read_to_string(format!("/proc/{}/status", self.pid()))?; + if !status.lines().any(|line| line == "State:\tT (stopped)") { + return Err(io::Error::other("bootstrap is not kernel stopped")); + } + Ok(()) + } + + pub fn terminate(&mut self) -> io::Result { + if self.reaped { + return self.child.wait(); + } + match rustix::process::pidfd_send_signal(&self.pidfd, Signal::KILL) { + Ok(()) | Err(rustix::io::Errno::SRCH) => {} + Err(error) => return Err(error.into()), + } + let deadline = Instant::now() + CLEANUP; + loop { + if let Some(status) = self.child.try_wait()? { + self.reaped = true; + return Ok(status); + } + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "bootstrap reap deadline", + )); + } + std::thread::sleep(Duration::from_millis(2)); + } + } +} + +impl Drop for BlockedTarget { + fn drop(&mut self) { + if !self.reaped { + let _ = self.terminate(); + } + } +} + +#[derive(Debug)] +pub struct InitializationFailure { + pub failure: ExecutionError, + pub cleanup_failure: Option, +} + +fn failure(stage: ExecutionStage, code: ErrorCode, error: Option<&io::Error>) -> ExecutionError { + ExecutionError { + stage, + code, + errno: error + .and_then(|e| e.raw_os_error()) + .and_then(|n| u16::try_from(n).ok()), + } +} + +/// Owns the real stopped child and its frozen task cgroup. No lifecycle event, +/// JSON field or public method can turn this into a user-code execution permit. +pub struct PreparedTarget { + target: BlockedTarget, + cgroup: TaskCgroup, +} + +impl PreparedTarget { + pub fn prepare( + config: &ValidatedRuntimeConfig, + spec: &ValidatedSpec, + executable: &Path, + ) -> Result { + let mut target = + BlockedTarget::spawn(executable).map_err(|error| InitializationFailure { + failure: failure( + ExecutionStage::Preparation, + ErrorCode::OperationFailed, + Some(&error), + ), + cleanup_failure: None, + })?; + let setup = (|| { + let root = DelegatedCgroup::open(config)?; + let mut group = root.create_task(spec)?; + if let Err(mut error) = group.enroll(&mut target) { + error.cleanup_error = group.kill_and_remove(CLEANUP).err(); + return Err(error); + } + Ok::<_, crate::cgroup::CgroupError>(group) + })(); + match setup { + Ok(cgroup) => Ok(Self { target, cgroup }), + Err(error) => { + let reap_error = target.terminate().err(); + Err(InitializationFailure { + failure: failure( + ExecutionStage::Cgroup, + ErrorCode::OperationFailed, + Some(&error.source), + ), + cleanup_failure: error.cleanup_error.as_ref().or(reap_error.as_ref()).map( + |e| { + failure( + ExecutionStage::Cleanup, + ErrorCode::CleanupIncomplete, + Some(e), + ) + }, + ), + }) + } + } + } + + pub fn pid(&self) -> u32 { + self.target.pid() + } + pub fn cgroup(&self) -> &TaskCgroup { + &self.cgroup + } + + /// Missing isolation is a real initialization failure, never readiness. + /// Kill/reap happens before a refusal is returned to the caller. + pub fn refuse_unimplemented_isolation(mut self) -> InitializationFailure { + let group_error = self.cgroup.kill_and_remove(CLEANUP).err(); + let reap_error = self.target.terminate().err(); + InitializationFailure { + failure: failure( + ExecutionStage::Namespaces, + ErrorCode::CapabilityUnavailable, + None, + ), + cleanup_failure: group_error.as_ref().or(reap_error.as_ref()).map(|e| { + failure( + ExecutionStage::Cleanup, + ErrorCode::CleanupIncomplete, + Some(e), + ) + }), + } + } +} + +impl Drop for PreparedTarget { + fn drop(&mut self) { + let _ = self.cgroup.kill_and_remove(CLEANUP); + let _ = self.target.terminate(); + } +} diff --git a/judge/crates/sandbox/src/launcher/mod.rs b/judge/crates/sandbox/src/launcher/mod.rs index 9e5b18a..f0d2caf 100644 --- a/judge/crates/sandbox/src/launcher/mod.rs +++ b/judge/crates/sandbox/src/launcher/mod.rs @@ -6,6 +6,8 @@ pub mod lifecycle; +#[cfg(target_os = "linux")] +pub mod blocked; #[cfg(target_os = "linux")] pub mod entry; #[cfg(target_os = "linux")] diff --git a/judge/crates/sandbox/src/lib.rs b/judge/crates/sandbox/src/lib.rs index 7e5657b..33bb2dd 100644 --- a/judge/crates/sandbox/src/lib.rs +++ b/judge/crates/sandbox/src/lib.rs @@ -8,3 +8,6 @@ pub mod outcome; pub mod probe; pub mod runtime; pub mod spec; + +#[cfg(target_os = "linux")] +pub mod cgroup; diff --git a/judge/crates/sandbox/src/main.rs b/judge/crates/sandbox/src/main.rs index 8036895..1514965 100644 --- a/judge/crates/sandbox/src/main.rs +++ b/judge/crates/sandbox/src/main.rs @@ -35,6 +35,13 @@ enum Command { #[arg(long)] control_socket: PathBuf, }, + /// Internal bootstrap: stops itself and cannot execute a user command. + #[cfg(target_os = "linux")] + #[command(hide = true)] + Bootstrap { + #[arg(long)] + parent_pid: u32, + }, } fn main() -> ExitCode { @@ -69,6 +76,16 @@ fn main() -> ExitCode { ), } } + #[cfg(target_os = "linux")] + Command::Bootstrap { parent_pid } => { + match kf_sandbox::launcher::blocked::bootstrap(parent_pid) { + Ok(()) => ExitCode::from(2), + Err(error) => spec_error( + "bootstrap_refused", + Some(serde_json::json!({"io_kind": format!("{:?}", error.kind())})), + ), + } + } } } diff --git a/judge/crates/sandbox/tests/blocked_vm.rs b/judge/crates/sandbox/tests/blocked_vm.rs new file mode 100644 index 0000000..7d1f293 --- /dev/null +++ b/judge/crates/sandbox/tests/blocked_vm.rs @@ -0,0 +1,316 @@ +#![cfg(target_os = "linux")] + +//! These tests are skipped by ordinary cargo test. Only an independently capped +//! VM runner may opt in; absence of that environment must fail, never pass. + +use kf_sandbox::cgroup::DelegatedCgroup; +use kf_sandbox::launcher::blocked::{BlockedTarget, PreparedTarget}; +use kf_sandbox::outcome::{ErrorCode, ExecutionStage}; +use kf_sandbox::runtime::{RuntimeConfig, ValidatedRuntimeConfig}; +use kf_sandbox::spec::{ExecutionSpec, SpecPolicy, ValidatedSpec}; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::os::unix::process::ExitStatusExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +static ATTEMPT: AtomicU64 = AtomicU64::new(0); + +fn require_vm() { + assert_eq!( + std::env::var("KF_SANDBOX_INDEPENDENT_VM").as_deref(), + Ok("1") + ); + assert_eq!( + fs::read_to_string("/kf/INDEPENDENT_VM").unwrap().trim(), + "isolated-test-guest" + ); + assert!( + fs::read_to_string("/sys/class/dmi/id/sys_vendor") + .unwrap() + .contains("QEMU") + ); + for path in ["/mnt/hgfs", "/home/kronecker", "/root/.ssh"] { + assert!(!Path::new(path).exists()); + } + let mounts = fs::read_to_string("/proc/self/mountinfo").unwrap(); + for forbidden in ["hgfs", "9p", "virtiofs"] { + assert!(!mounts.contains(forbidden)); + } + let net: Vec<_> = fs::read_dir("/sys/class/net") + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(net, ["lo"]); +} + +struct Root { + path: PathBuf, + config: RuntimeConfig, +} + +impl Root { + fn new() -> Self { + require_vm(); + let path = PathBuf::from(format!( + "/sys/fs/cgroup/kf-vm-{}-{}", + std::process::id(), + ATTEMPT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + let mut config: RuntimeConfig = serde_json::from_str(include_str!( + "../../../../deploy/sandbox.runtime.example.json" + )) + .unwrap(); + config.delegated_cgroup_root = path.to_str().unwrap().into(); + config.aggregate.task_cpu_quota_us = 10_000; + Self { path, config } + } + fn validated(&self) -> ValidatedRuntimeConfig { + self.config.clone().validate().unwrap() + } +} + +impl Drop for Root { + fn drop(&mut self) { + if self.path.exists() { + let _ = fs::write(self.path.join("cgroup.kill"), "1"); + if let Ok(entries) = fs::read_dir(&self.path) { + for entry in entries.flatten() { + if entry.file_type().is_ok_and(|kind| kind.is_dir()) { + let _ = fs::remove_dir(entry.path()); + } + } + } + let _ = fs::remove_dir(&self.path); + } + } +} + +fn spec(id: &str, memory: u64, processes: u32) -> ValidatedSpec { + let mut spec: ExecutionSpec = + serde_json::from_str(include_str!("fixtures/spec.valid.json")).unwrap(); + spec.run_id = id.into(); + spec.limits.memory_bytes = memory; + spec.limits.processes = processes; + spec.validate(&SpecPolicy::default()).unwrap() +} + +fn binary() -> PathBuf { + std::env::var_os("KF_SANDBOX_TEST_BIN") + .expect("VM sandbox path") + .into() +} +fn children() -> Vec { + let mut children = Vec::new(); + for task in fs::read_dir("/proc/self/task").unwrap() { + children.extend( + fs::read_to_string(task.unwrap().path().join("children")) + .unwrap() + .split_whitespace() + .map(|pid| pid.parse::().unwrap()), + ); + } + children.sort_unstable(); + children +} + +#[test] +#[ignore = "requires independent VM with externally verified whole-VM caps"] +fn stopped_target_is_enrolled_frozen_and_reaped_before_namespace_refusal() { + let root = Root::new(); + let target = + PreparedTarget::prepare(&root.validated(), &spec("gated", 64 << 20, 8), &binary()).unwrap(); + let pid = target.pid(); + let path = root.path.join(target.cgroup().name()); + let limits = target.cgroup().applied_limits().unwrap(); + assert_eq!(limits.memory_bytes, 64 << 20); + assert_eq!(limits.swap_bytes, 0); + assert_eq!(limits.processes, 8); + assert_eq!( + (limits.cpu_quota_us, limits.cpu_period_us), + (10_000, 100_000) + ); + assert!(limits.oom_group); + for _ in 0..3 { + assert!( + fs::read_to_string(format!("/proc/{pid}/status")) + .unwrap() + .contains("State:\tT (stopped)") + ); + assert_eq!( + fs::read_to_string(path.join("cgroup.freeze")) + .unwrap() + .trim(), + "1" + ); + assert_eq!( + fs::read_to_string(path.join("cgroup.procs")) + .unwrap() + .trim(), + pid.to_string() + ); + std::thread::sleep(Duration::from_millis(10)); + } + let failure = target.refuse_unimplemented_isolation(); + assert_eq!(failure.failure.stage, ExecutionStage::Namespaces); + assert_eq!(failure.failure.code, ErrorCode::CapabilityUnavailable); + assert!(failure.cleanup_failure.is_none()); + assert!(!Path::new(&format!("/proc/{pid}")).exists()); + assert!(!path.exists()); + println!( + "KF_GATED_PROOF pid={pid} memory={} swap=0 pids=8 cpu=10000/100000 frozen=1 refusal=namespaces reaped=1 removed=1", + limits.memory_bytes + ); +} + +#[test] +#[ignore = "requires independent VM with externally verified whole-VM caps"] +fn unsafe_delegation_fails_at_cgroup_and_does_not_leave_bootstrap() { + let root = Root::new(); + let before = children(); + fs::set_permissions(&root.path, fs::Permissions::from_mode(0o777)).unwrap(); + let result = + PreparedTarget::prepare(&root.validated(), &spec("unsafe", 64 << 20, 8), &binary()); + let Err(failure) = result else { + panic!("unsafe root must fail"); + }; + assert_eq!(failure.failure.stage, ExecutionStage::Cgroup); + assert!(failure.cleanup_failure.is_none()); + assert_eq!(children(), before); + assert!(!root.path.join("kf-unsafe").exists()); + println!("KF_CGROUP_FAILURE_PROOF stage=cgroup bootstrap_reaped=1"); +} + +#[test] +#[ignore = "requires independent VM with externally verified whole-VM caps"] +fn exclusive_tasks_and_kernel_directory_count_prevent_adoption_and_overbooking() { + let root = Root::new(); + let config = root.validated(); + let delegate = DelegatedCgroup::open(&config).unwrap(); + let mut one = delegate.create_task(&spec("one", 64 << 20, 8)).unwrap(); + assert!(delegate.create_task(&spec("one", 64 << 20, 8)).is_err()); + let mut two = delegate.create_task(&spec("two", 64 << 20, 8)).unwrap(); + assert!(delegate.create_task(&spec("three", 64 << 20, 8)).is_err()); + assert_eq!(one.applied_limits().unwrap().memory_bytes, 64 << 20); + one.kill_and_remove(Duration::from_secs(1)).unwrap(); + one.kill_and_remove(Duration::from_secs(1)).unwrap(); + two.kill_and_remove(Duration::from_secs(1)).unwrap(); + assert!(!root.path.join("kf-one").exists()); + assert!(!root.path.join("kf-two").exists()); + println!("KF_EXCLUSIVE_PROOF collision_rejected=1 max_tasks=2 idempotent_cleanup=1"); +} + +#[test] +#[ignore = "requires independent VM with externally verified whole-VM caps"] +fn changed_limits_are_rejected_while_child_is_still_stopped() { + let root = Root::new(); + let delegate = DelegatedCgroup::open(&root.validated()).unwrap(); + let mut group = delegate.create_task(&spec("changed", 64 << 20, 8)).unwrap(); + let path = root.path.join(group.name()); + fs::write(path.join("memory.max"), (32 << 20).to_string()).unwrap(); + let mut child = BlockedTarget::spawn(&binary()).unwrap(); + assert!(group.enroll(&mut child).is_err()); + child.verify_stopped().unwrap(); + assert!( + fs::read_to_string(path.join("cgroup.procs")) + .unwrap() + .trim() + .is_empty() + ); + group.kill_and_remove(Duration::from_secs(1)).unwrap(); + assert_eq!(child.terminate().unwrap().signal(), Some(9)); + println!("KF_CHANGED_LIMIT_PROOF enrollment_rejected=1 blocked=1 terminated=1"); +} + +#[test] +#[ignore = "requires independent VM with externally verified whole-VM caps"] +fn request_cannot_exceed_operator_ceiling() { + let root = Root::new(); + let delegate = DelegatedCgroup::open(&root.validated()).unwrap(); + assert!( + delegate + .create_task(&spec("excess", 1024 << 20, 8)) + .is_err() + ); + assert!(!root.path.join("kf-excess").exists()); + println!("KF_CEILING_PROOF request_excess_rejected=1"); +} + +fn counter(raw: &str, key: &str) -> u64 { + raw.lines() + .find_map(|line| { + line.split_once(' ') + .filter(|(name, _)| *name == key) + .map(|(_, value)| value.parse().unwrap()) + }) + .unwrap() +} + +fn limit_probe(kind: &str) { + let root = Root::new(); + let delegate = DelegatedCgroup::open(&root.validated()).unwrap(); + let processes = if kind == "pids" { 1 } else { 8 }; + let mut task = delegate + .create_task(&spec(kind, 32 << 20, processes)) + .unwrap(); + let path = root.path.join(task.name()); + let mut child = BlockedTarget::spawn(Path::new(&format!("/kf/bin/limit-{kind}"))).unwrap(); + task.enroll(&mut child).unwrap(); + // VM-only fixed probes: deliberately exercise controllers after admission. + // Production APIs expose no thaw/continue or user execution permit. + fs::write(path.join("cgroup.freeze"), "0").unwrap(); + let pid = rustix::process::Pid::from_raw(child.pid() as i32).unwrap(); + let pidfd = rustix::process::pidfd_open(pid, rustix::process::PidfdFlags::empty()).unwrap(); + rustix::process::pidfd_send_signal(&pidfd, rustix::process::Signal::CONT).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let status = loop { + if let Some(status) = child.poll_exit().unwrap() { + break status; + } + assert!(Instant::now() < deadline, "bounded fixed resource probe"); + std::thread::sleep(Duration::from_millis(5)); + }; + let cpu = fs::read_to_string(path.join("cpu.stat")).unwrap(); + let memory = fs::read_to_string(path.join("memory.events")).unwrap(); + let pids = fs::read_to_string(path.join("pids.events")).unwrap(); + match kind { + "pids" => { + assert!(status.success()); + assert!(counter(&pids, "max") >= 1); + } + "memory" => { + assert_eq!(status.signal(), Some(9)); + assert!(counter(&memory, "oom_kill") >= 1); + } + "cpu" => { + assert!(status.success()); + assert!(counter(&cpu, "usage_usec") >= 150_000); + assert!(counter(&cpu, "nr_throttled") >= 1); + } + _ => unreachable!(), + } + println!( + "KF_LIMIT_PROOF kind={kind} status={status} cpu={cpu:?} memory={memory:?} pids={pids:?}" + ); + task.kill_and_remove(Duration::from_secs(1)).unwrap(); + assert!(!path.exists()); +} + +#[test] +#[ignore = "requires independent VM with externally verified whole-VM caps"] +fn pids_controller_actually_denies_fork() { + limit_probe("pids"); +} +#[test] +#[ignore = "requires independent VM with externally verified whole-VM caps"] +fn memory_controller_actually_kills_at_limit() { + limit_probe("memory"); +} +#[test] +#[ignore = "requires independent VM with externally verified whole-VM caps"] +fn cpu_controller_actually_throttles_fixed_work() { + limit_probe("cpu"); +} diff --git a/judge/crates/sandbox/tests/fixtures/cgroup_limits.c b/judge/crates/sandbox/tests/fixtures/cgroup_limits.c new file mode 100644 index 0000000..ddcc293 --- /dev/null +++ b/judge/crates/sandbox/tests/fixtures/cgroup_limits.c @@ -0,0 +1,35 @@ +/* Fixed VM-only resource probes, never submitted programs or host test commands. */ +#include +#include +#include +#include +#include +#include +#include +#include + +int main(void) { + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0 || getppid() <= 1) return 40; + raise(SIGSTOP); +#if KF_FIXTURE == 1 + pid_t pid = fork(); + if (pid == -1) return errno == EAGAIN ? 0 : 41; + if (pid == 0) _exit(42); + waitpid(pid, NULL, 0); + return 42; +#elif KF_FIXTURE == 2 + size_t size = 128u * 1024u * 1024u; + volatile unsigned char *data = malloc(size); + if (data == NULL) return 43; + for (size_t i = 0; i < size; i += 4096) data[i] = 1; + free((void *) data); + return 44; +#elif KF_FIXTURE == 3 + clock_t start = clock(); + volatile unsigned long count = 0; + while (clock() - start < CLOCKS_PER_SEC * 3 / 20) ++count; + return count == 0 ? 45 : 0; +#else +#error KF_FIXTURE must be 1, 2, or 3 +#endif +}