feat(sandbox): gate target startup with verified task cgroups

This commit is contained in:
2026-10-07 01:40:32 +08:00
parent 605785edee
commit 1a91c2e0f6
9 changed files with 1102 additions and 1 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ kf-contracts.workspace = true
clap.workspace = true
serde.workspace = true
serde_json.workspace = true
rustix = { version = "1.1.5", features = ["net", "process"] }
rustix = { version = "1.1.5", features = ["net", "process", "fs"] }
[lints]
workspace = true
+412
View File
@@ -0,0 +1,412 @@
//! Descriptor-relative management of one operator-delegated cgroup v2 subtree.
//! No request controls host paths; this module never changes the system root.
use crate::launcher::blocked::BlockedTarget;
use crate::runtime::ValidatedRuntimeConfig;
use crate::spec::{SpecPolicy, ValidatedSpec};
use rustix::fd::OwnedFd;
use rustix::fs::{AtFlags, Mode, OFlags};
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::{Component, Path};
use std::time::{Duration, Instant};
const CGROUP2_MAGIC: u64 = 0x6367_7270;
const MAX_INTERFACE_BYTES: u64 = 4096;
const DIR_FLAGS: OFlags = OFlags::RDONLY
.union(OFlags::DIRECTORY)
.union(OFlags::NOFOLLOW)
.union(OFlags::CLOEXEC);
#[derive(Debug)]
pub struct CgroupError {
pub source: io::Error,
pub cleanup_error: Option<io::Error>,
}
impl From<io::Error> for CgroupError {
fn from(source: io::Error) -> Self {
Self {
source,
cleanup_error: None,
}
}
}
impl std::fmt::Display for CgroupError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "cgroup operation failed: {:?}", self.source.kind())
}
}
impl std::error::Error for CgroupError {}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AppliedLimits {
pub memory_bytes: u64,
pub swap_bytes: u64,
pub processes: u32,
pub cpu_quota_us: u64,
pub cpu_period_us: u64,
pub oom_group: bool,
}
pub struct DelegatedCgroup {
fd: OwnedFd,
policy: SpecPolicy,
quota: u64,
period: u64,
}
impl DelegatedCgroup {
/// Config comes from the operator. Structural validation is not proof of
/// config-file ownership, service delegation or an exec permit.
/// The root must be empty, owned by this helper and not group/other writable.
pub fn open(config: &ValidatedRuntimeConfig) -> Result<Self, CgroupError> {
let config_data = config.config();
let fd = open_directory(Path::new(&config_data.delegated_cgroup_root))?;
let stat = rustix::fs::fstat(&fd).map_err(io::Error::from)?;
if stat.st_uid != rustix::process::geteuid().as_raw() || stat.st_mode & 0o022 != 0 {
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"unsafe delegated owner/mode",
)
.into());
}
if rustix::fs::fstatfs(&fd).map_err(io::Error::from)?.f_type as u64 != CGROUP2_MAGIC {
return Err(invalid("not a cgroup v2 directory").into());
}
if read(&fd, "cgroup.type")?.trim() != "domain"
|| !read(&fd, "cgroup.procs")?.trim().is_empty()
{
return Err(invalid("delegated root must be an empty domain").into());
}
let controllers = read(&fd, "cgroup.controllers")?;
if !["cpu", "memory", "pids"]
.iter()
.all(|name| controllers.split_whitespace().any(|value| value == *name))
{
return Err(invalid("required controllers unavailable").into());
}
write(&fd, "cgroup.subtree_control", "+cpu +memory +pids")?;
let enabled = read(&fd, "cgroup.subtree_control")?;
if !["cpu", "memory", "pids"]
.iter()
.all(|name| enabled.split_whitespace().any(|value| value == *name))
{
return Err(invalid("required controllers not enabled").into());
}
// Bound concurrent task directories and prevent deeper cgroup creation.
write_verify(&fd, "cgroup.max.depth", "1")?;
write_verify(
&fd,
"cgroup.max.descendants",
&config_data.max_concurrent_runs.to_string(),
)?;
let budget = &config_data.aggregate;
let aggregate = AppliedLimits {
memory_bytes: budget.memory_bytes,
swap_bytes: 0,
processes: budget.processes,
cpu_quota_us: budget.cpu_quota_us,
cpu_period_us: budget.cpu_period_us,
oom_group: true,
};
apply_limits(&fd, &aggregate)?;
Ok(Self {
fd,
policy: config.spec_policy(),
quota: budget.task_cpu_quota_us,
period: budget.cpu_period_us,
})
}
/// Always mkdir exclusive: never adopt, overwrite or reuse an existing task.
pub fn create_task(&self, spec: &ValidatedSpec) -> Result<TaskCgroup, CgroupError> {
spec.spec()
.clone()
.validate(&self.policy)
.map_err(|_| invalid("request exceeds delegated policy"))?;
let name = format!("kf-{}", spec.spec().run_id);
rustix::fs::mkdirat(&self.fd, name.as_str(), Mode::from_raw_mode(0o700))
.map_err(io::Error::from)?;
let opened = rustix::fs::openat(&self.fd, name.as_str(), DIR_FLAGS, Mode::empty());
let fd = match opened {
Ok(fd) => fd,
Err(error) => {
return Err(CgroupError {
source: error.into(),
cleanup_error: rustix::fs::unlinkat(
&self.fd,
name.as_str(),
AtFlags::REMOVEDIR,
)
.err()
.map(Into::into),
});
}
};
let parent = match rustix::io::fcntl_dupfd_cloexec(&self.fd, 3) {
Ok(fd) => fd,
Err(error) => {
return Err(CgroupError {
source: error.into(),
cleanup_error: rustix::fs::unlinkat(
&self.fd,
name.as_str(),
AtFlags::REMOVEDIR,
)
.err()
.map(Into::into),
});
}
};
let limits = AppliedLimits {
memory_bytes: spec.spec().limits.memory_bytes,
swap_bytes: 0,
processes: spec.spec().limits.processes,
cpu_quota_us: self.quota,
cpu_period_us: self.period,
oom_group: true,
};
let mut task = TaskCgroup {
parent,
fd,
name,
limits,
removed: false,
};
let setup = (|| {
if read(&task.fd, "cgroup.type")?.trim() != "domain" {
return Err(invalid("invalid task domain"));
}
apply_limits(&task.fd, &task.limits)?;
write_verify(&task.fd, "cgroup.max.depth", "0")?;
write_verify(&task.fd, "cgroup.max.descendants", "0")?;
// Make fatal termination cover the entire task and retain a kernel
// freeze gate even if somebody sends the bootstrap SIGCONT.
write_verify(&task.fd, "cgroup.freeze", "1")?;
task.wait_event("frozen", "1", Duration::from_secs(1))?;
Ok::<_, io::Error>(())
})();
if let Err(source) = setup {
let cleanup_error = task.kill_and_remove(Duration::from_secs(1)).err();
return Err(CgroupError {
source,
cleanup_error,
});
}
Ok(task)
}
}
pub struct TaskCgroup {
parent: OwnedFd,
fd: OwnedFd,
name: String,
limits: AppliedLimits,
removed: bool,
}
impl TaskCgroup {
pub fn name(&self) -> &str {
&self.name
}
pub fn applied_limits(&self) -> io::Result<AppliedLimits> {
self.live()?;
verify_limits(&self.fd, &self.limits)?;
Ok(self.limits.clone())
}
/// PID comes from the unreaped direct Child; callers cannot enroll arbitrary PIDs.
pub fn enroll(&mut self, target: &mut BlockedTarget) -> Result<(), CgroupError> {
self.live()?;
target.verify_stopped()?;
self.applied_limits()?;
self.wait_event("frozen", "1", Duration::from_secs(1))?;
if !read(&self.fd, "cgroup.procs")?.trim().is_empty() {
return Err(invalid("task already populated").into());
}
write(&self.fd, "cgroup.procs", &target.pid().to_string())?;
let members = read(&self.fd, "cgroup.procs")?;
if members.trim() != target.pid().to_string() {
return Err(invalid("task enrollment mismatch").into());
}
target.verify_stopped()?;
self.wait_event("frozen", "1", Duration::from_secs(1))?;
Ok(())
}
/// Entire group must be empty before descriptor-relative removal. Repeated
/// cleanup is harmless; explicit calls report errors independently.
pub fn kill_and_remove(&mut self, timeout: Duration) -> io::Result<()> {
if self.removed {
return Ok(());
}
if timeout.is_zero() || timeout > Duration::from_secs(5) {
return Err(invalid("invalid cleanup deadline"));
}
write(&self.fd, "cgroup.kill", "1")?;
self.wait_event("populated", "0", timeout)?;
rustix::fs::unlinkat(&self.parent, self.name.as_str(), AtFlags::REMOVEDIR)?;
self.removed = true;
Ok(())
}
fn live(&self) -> io::Result<()> {
if self.removed {
Err(io::Error::new(
io::ErrorKind::NotFound,
"task cgroup removed",
))
} else {
Ok(())
}
}
fn wait_event(&self, key: &str, expected: &str, timeout: Duration) -> io::Result<()> {
let deadline = Instant::now() + timeout;
loop {
let events = read(&self.fd, "cgroup.events")?;
if events
.lines()
.any(|line| line.split_once(' ') == Some((key, expected)))
{
return Ok(());
}
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"cgroup event deadline",
));
}
std::thread::sleep(Duration::from_millis(2));
}
}
}
impl Drop for TaskCgroup {
fn drop(&mut self) {
if !self.removed {
let _ = self.kill_and_remove(Duration::from_millis(250));
}
}
}
fn open_directory(path: &Path) -> io::Result<OwnedFd> {
// The validated root is absolute and strictly below /sys/fs/cgroup. Walk
// every component with NOFOLLOW; a symlink ancestor cannot redirect us.
if !path.starts_with("/sys/fs/cgroup") || path == Path::new("/sys/fs/cgroup") {
return Err(invalid("not a delegated cgroup subtree"));
}
let mut fd = rustix::fs::open("/", DIR_FLAGS, Mode::empty())?;
for component in path.components() {
match component {
Component::RootDir => {}
Component::Normal(name) => {
fd = rustix::fs::openat(&fd, name, DIR_FLAGS, Mode::empty())?
}
_ => return Err(invalid("noncanonical cgroup root")),
}
}
Ok(fd)
}
fn read(fd: &OwnedFd, name: &str) -> io::Result<String> {
let file = rustix::fs::openat(
fd,
name,
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)?;
let mut text = String::new();
File::from(file)
.take(MAX_INTERFACE_BYTES + 1)
.read_to_string(&mut text)?;
if text.len() as u64 > MAX_INTERFACE_BYTES {
return Err(invalid("cgroup interface too large"));
}
Ok(text)
}
fn write(fd: &OwnedFd, name: &str, value: &str) -> io::Result<()> {
let file = rustix::fs::openat(
fd,
name,
OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)?;
// Kernfs values (not streams): one complete write, never retry a short suffix.
if File::from(file).write(value.as_bytes())? != value.len() {
return Err(io::Error::new(
io::ErrorKind::WriteZero,
"short cgroup write",
));
}
Ok(())
}
fn write_verify(fd: &OwnedFd, name: &str, value: &str) -> io::Result<()> {
write(fd, name, value)?;
if read(fd, name)?.trim() != value {
return Err(invalid("cgroup readback mismatch"));
}
Ok(())
}
fn apply_limits(fd: &OwnedFd, limits: &AppliedLimits) -> io::Result<()> {
write_verify(fd, "memory.max", &limits.memory_bytes.to_string())?;
write_verify(fd, "memory.swap.max", &limits.swap_bytes.to_string())?;
write_verify(
fd,
"memory.oom.group",
if limits.oom_group { "1" } else { "0" },
)?;
write_verify(fd, "pids.max", &limits.processes.to_string())?;
write_verify(
fd,
"cpu.max",
&format!("{} {}", limits.cpu_quota_us, limits.cpu_period_us),
)
}
fn verify_limits(fd: &OwnedFd, limits: &AppliedLimits) -> io::Result<()> {
for (name, expected) in [
("memory.max", limits.memory_bytes.to_string()),
("memory.swap.max", limits.swap_bytes.to_string()),
("pids.max", limits.processes.to_string()),
(
"memory.oom.group",
if limits.oom_group { "1" } else { "0" }.into(),
),
(
"cpu.max",
format!("{} {}", limits.cpu_quota_us, limits.cpu_period_us),
),
] {
if read(fd, name)?.trim() != expected {
return Err(invalid("configured limits changed"));
}
}
Ok(())
}
fn invalid(message: &'static str) -> io::Error {
io::Error::new(io::ErrorKind::InvalidInput, message)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn system_root_and_host_paths_are_rejected_before_open() {
for path in [
"/sys/fs/cgroup",
"/tmp/cgroup",
"/sys/fs/cgroup/../cgroup",
"/sys/fs/cgroup/./task",
] {
assert!(open_directory(Path::new(path)).is_err());
}
}
}
@@ -0,0 +1,285 @@
//! Real kernel stop before task enrollment. There is deliberately no exec/release API.
use crate::cgroup::{DelegatedCgroup, TaskCgroup};
use crate::outcome::{ErrorCode, ExecutionError, ExecutionStage};
use crate::runtime::ValidatedRuntimeConfig;
use crate::spec::ValidatedSpec;
use rustix::fd::{AsFd, OwnedFd};
use rustix::process::{Pid, PidfdFlags, Signal, WaitId, WaitIdOptions};
use std::io;
use std::path::Path;
use std::process::{Child, Command, ExitStatus, Stdio};
use std::time::{Duration, Instant};
const STARTUP: Duration = Duration::from_secs(5);
const CLEANUP: Duration = Duration::from_secs(2);
/// Runs only after a fresh exec, without custom pre_exec/fork callbacks. A parent
/// dying before/during prctl is caught by the subsequent kernel parent check.
/// PDEATHSIG refers to the spawning thread: the supervisor must keep it alive.
pub fn bootstrap(expected_parent: u32) -> io::Result<()> {
super::entry::ensure_single_threaded()?;
rustix::process::set_parent_process_death_signal(Some(Signal::KILL))?;
let parent = rustix::process::getppid();
if expected_parent <= 1
|| parent.map(|pid| pid.as_raw_pid() as u32) != Some(expected_parent)
|| rustix::process::parent_process_death_signal()? != Some(Signal::KILL)
{
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"bootstrap parent mismatch",
));
}
rustix::process::kill_process(rustix::process::getpid(), Signal::STOP)?;
// An unexpected SIGCONT cannot run initialization or a user command. Future
// isolation initialization and the separate final exec gate are still absent.
Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"bootstrap has no execution permit",
))
}
/// Sole child owner; callers must not ignore SIGCHLD or reap this child elsewhere.
/// pidfd is obtained before any wait; NOWAIT leaves reaping to the Child handle.
pub struct BlockedTarget {
child: Child,
pidfd: OwnedFd,
reaped: bool,
}
impl BlockedTarget {
/// `executable` is the operator's trusted sandbox binary, never a request argv.
pub fn spawn(executable: &Path) -> io::Result<Self> {
// A ignored/caught SIGCHLD could reap/recycle a PID before pidfd_open.
// Fresh helpers retain the default disposition; reject inherited changes.
let status = std::fs::read_to_string("/proc/self/status")?;
for key in ["SigIgn:", "SigCgt:"] {
let bits = status
.lines()
.find_map(|line| line.strip_prefix(key))
.and_then(|value| u64::from_str_radix(value.trim(), 16).ok())
.ok_or_else(|| io::Error::other("cannot verify SIGCHLD policy"))?;
if bits & (1 << (rustix::process::Signal::CHILD.as_raw() - 1)) != 0 {
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"SIGCHLD must have default disposition",
));
}
}
let mut child = Command::new(executable)
.arg("bootstrap")
.arg("--parent-pid")
.arg(std::process::id().to_string())
.env_clear()
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()?;
let Some(pid) = Pid::from_raw(child.id() as i32) else {
let _ = child.kill();
let _ = child.wait();
return Err(io::Error::other("invalid child PID"));
};
let pidfd = match rustix::process::pidfd_open(pid, PidfdFlags::empty()) {
Ok(fd) => fd,
Err(error) => {
// The unreaped direct child cannot have its PID recycled.
let _ = child.kill();
let _ = child.wait();
return Err(error.into());
}
};
let mut target = Self {
child,
pidfd,
reaped: false,
};
let deadline = Instant::now() + STARTUP;
loop {
let state = rustix::process::waitid(
WaitId::PidFd(target.pidfd.as_fd()),
WaitIdOptions::STOPPED
| WaitIdOptions::EXITED
| WaitIdOptions::NOHANG
| WaitIdOptions::NOWAIT,
)?;
if let Some(state) = state {
if state.stopping_signal() == Some(Signal::STOP.as_raw()) {
target.verify_stopped()?;
return Ok(target);
}
return Err(io::Error::other("bootstrap exited before stop"));
}
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"bootstrap stop deadline",
));
}
std::thread::sleep(Duration::from_millis(2));
}
}
pub fn pid(&self) -> u32 {
self.child.id()
}
pub fn poll_exit(&mut self) -> io::Result<Option<ExitStatus>> {
let status = self.child.try_wait()?;
if status.is_some() {
self.reaped = true;
}
Ok(status)
}
pub fn verify_stopped(&mut self) -> io::Result<()> {
if self.reaped || self.child.try_wait()?.is_some() {
self.reaped = true;
return Err(io::Error::other("bootstrap is not alive"));
}
let status = std::fs::read_to_string(format!("/proc/{}/status", self.pid()))?;
if !status.lines().any(|line| line == "State:\tT (stopped)") {
return Err(io::Error::other("bootstrap is not kernel stopped"));
}
Ok(())
}
pub fn terminate(&mut self) -> io::Result<ExitStatus> {
if self.reaped {
return self.child.wait();
}
match rustix::process::pidfd_send_signal(&self.pidfd, Signal::KILL) {
Ok(()) | Err(rustix::io::Errno::SRCH) => {}
Err(error) => return Err(error.into()),
}
let deadline = Instant::now() + CLEANUP;
loop {
if let Some(status) = self.child.try_wait()? {
self.reaped = true;
return Ok(status);
}
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"bootstrap reap deadline",
));
}
std::thread::sleep(Duration::from_millis(2));
}
}
}
impl Drop for BlockedTarget {
fn drop(&mut self) {
if !self.reaped {
let _ = self.terminate();
}
}
}
#[derive(Debug)]
pub struct InitializationFailure {
pub failure: ExecutionError,
pub cleanup_failure: Option<ExecutionError>,
}
fn failure(stage: ExecutionStage, code: ErrorCode, error: Option<&io::Error>) -> ExecutionError {
ExecutionError {
stage,
code,
errno: error
.and_then(|e| e.raw_os_error())
.and_then(|n| u16::try_from(n).ok()),
}
}
/// Owns the real stopped child and its frozen task cgroup. No lifecycle event,
/// JSON field or public method can turn this into a user-code execution permit.
pub struct PreparedTarget {
target: BlockedTarget,
cgroup: TaskCgroup,
}
impl PreparedTarget {
pub fn prepare(
config: &ValidatedRuntimeConfig,
spec: &ValidatedSpec,
executable: &Path,
) -> Result<Self, InitializationFailure> {
let mut target =
BlockedTarget::spawn(executable).map_err(|error| InitializationFailure {
failure: failure(
ExecutionStage::Preparation,
ErrorCode::OperationFailed,
Some(&error),
),
cleanup_failure: None,
})?;
let setup = (|| {
let root = DelegatedCgroup::open(config)?;
let mut group = root.create_task(spec)?;
if let Err(mut error) = group.enroll(&mut target) {
error.cleanup_error = group.kill_and_remove(CLEANUP).err();
return Err(error);
}
Ok::<_, crate::cgroup::CgroupError>(group)
})();
match setup {
Ok(cgroup) => Ok(Self { target, cgroup }),
Err(error) => {
let reap_error = target.terminate().err();
Err(InitializationFailure {
failure: failure(
ExecutionStage::Cgroup,
ErrorCode::OperationFailed,
Some(&error.source),
),
cleanup_failure: error.cleanup_error.as_ref().or(reap_error.as_ref()).map(
|e| {
failure(
ExecutionStage::Cleanup,
ErrorCode::CleanupIncomplete,
Some(e),
)
},
),
})
}
}
}
pub fn pid(&self) -> u32 {
self.target.pid()
}
pub fn cgroup(&self) -> &TaskCgroup {
&self.cgroup
}
/// Missing isolation is a real initialization failure, never readiness.
/// Kill/reap happens before a refusal is returned to the caller.
pub fn refuse_unimplemented_isolation(mut self) -> InitializationFailure {
let group_error = self.cgroup.kill_and_remove(CLEANUP).err();
let reap_error = self.target.terminate().err();
InitializationFailure {
failure: failure(
ExecutionStage::Namespaces,
ErrorCode::CapabilityUnavailable,
None,
),
cleanup_failure: group_error.as_ref().or(reap_error.as_ref()).map(|e| {
failure(
ExecutionStage::Cleanup,
ErrorCode::CleanupIncomplete,
Some(e),
)
}),
}
}
}
impl Drop for PreparedTarget {
fn drop(&mut self) {
let _ = self.cgroup.kill_and_remove(CLEANUP);
let _ = self.target.terminate();
}
}
+2
View File
@@ -6,6 +6,8 @@
pub mod lifecycle;
#[cfg(target_os = "linux")]
pub mod blocked;
#[cfg(target_os = "linux")]
pub mod entry;
#[cfg(target_os = "linux")]
+3
View File
@@ -8,3 +8,6 @@ pub mod outcome;
pub mod probe;
pub mod runtime;
pub mod spec;
#[cfg(target_os = "linux")]
pub mod cgroup;
+17
View File
@@ -35,6 +35,13 @@ enum Command {
#[arg(long)]
control_socket: PathBuf,
},
/// Internal bootstrap: stops itself and cannot execute a user command.
#[cfg(target_os = "linux")]
#[command(hide = true)]
Bootstrap {
#[arg(long)]
parent_pid: u32,
},
}
fn main() -> ExitCode {
@@ -69,6 +76,16 @@ fn main() -> ExitCode {
),
}
}
#[cfg(target_os = "linux")]
Command::Bootstrap { parent_pid } => {
match kf_sandbox::launcher::blocked::bootstrap(parent_pid) {
Ok(()) => ExitCode::from(2),
Err(error) => spec_error(
"bootstrap_refused",
Some(serde_json::json!({"io_kind": format!("{:?}", error.kind())})),
),
}
}
}
}
+316
View File
@@ -0,0 +1,316 @@
#![cfg(target_os = "linux")]
//! These tests are skipped by ordinary cargo test. Only an independently capped
//! VM runner may opt in; absence of that environment must fail, never pass.
use kf_sandbox::cgroup::DelegatedCgroup;
use kf_sandbox::launcher::blocked::{BlockedTarget, PreparedTarget};
use kf_sandbox::outcome::{ErrorCode, ExecutionStage};
use kf_sandbox::runtime::{RuntimeConfig, ValidatedRuntimeConfig};
use kf_sandbox::spec::{ExecutionSpec, SpecPolicy, ValidatedSpec};
use std::fs;
use std::os::unix::fs::PermissionsExt;
use std::os::unix::process::ExitStatusExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
static ATTEMPT: AtomicU64 = AtomicU64::new(0);
fn require_vm() {
assert_eq!(
std::env::var("KF_SANDBOX_INDEPENDENT_VM").as_deref(),
Ok("1")
);
assert_eq!(
fs::read_to_string("/kf/INDEPENDENT_VM").unwrap().trim(),
"isolated-test-guest"
);
assert!(
fs::read_to_string("/sys/class/dmi/id/sys_vendor")
.unwrap()
.contains("QEMU")
);
for path in ["/mnt/hgfs", "/home/kronecker", "/root/.ssh"] {
assert!(!Path::new(path).exists());
}
let mounts = fs::read_to_string("/proc/self/mountinfo").unwrap();
for forbidden in ["hgfs", "9p", "virtiofs"] {
assert!(!mounts.contains(forbidden));
}
let net: Vec<_> = fs::read_dir("/sys/class/net")
.unwrap()
.map(|entry| entry.unwrap().file_name())
.collect();
assert_eq!(net, ["lo"]);
}
struct Root {
path: PathBuf,
config: RuntimeConfig,
}
impl Root {
fn new() -> Self {
require_vm();
let path = PathBuf::from(format!(
"/sys/fs/cgroup/kf-vm-{}-{}",
std::process::id(),
ATTEMPT.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir(&path).unwrap();
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
let mut config: RuntimeConfig = serde_json::from_str(include_str!(
"../../../../deploy/sandbox.runtime.example.json"
))
.unwrap();
config.delegated_cgroup_root = path.to_str().unwrap().into();
config.aggregate.task_cpu_quota_us = 10_000;
Self { path, config }
}
fn validated(&self) -> ValidatedRuntimeConfig {
self.config.clone().validate().unwrap()
}
}
impl Drop for Root {
fn drop(&mut self) {
if self.path.exists() {
let _ = fs::write(self.path.join("cgroup.kill"), "1");
if let Ok(entries) = fs::read_dir(&self.path) {
for entry in entries.flatten() {
if entry.file_type().is_ok_and(|kind| kind.is_dir()) {
let _ = fs::remove_dir(entry.path());
}
}
}
let _ = fs::remove_dir(&self.path);
}
}
}
fn spec(id: &str, memory: u64, processes: u32) -> ValidatedSpec {
let mut spec: ExecutionSpec =
serde_json::from_str(include_str!("fixtures/spec.valid.json")).unwrap();
spec.run_id = id.into();
spec.limits.memory_bytes = memory;
spec.limits.processes = processes;
spec.validate(&SpecPolicy::default()).unwrap()
}
fn binary() -> PathBuf {
std::env::var_os("KF_SANDBOX_TEST_BIN")
.expect("VM sandbox path")
.into()
}
fn children() -> Vec<u32> {
let mut children = Vec::new();
for task in fs::read_dir("/proc/self/task").unwrap() {
children.extend(
fs::read_to_string(task.unwrap().path().join("children"))
.unwrap()
.split_whitespace()
.map(|pid| pid.parse::<u32>().unwrap()),
);
}
children.sort_unstable();
children
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn stopped_target_is_enrolled_frozen_and_reaped_before_namespace_refusal() {
let root = Root::new();
let target =
PreparedTarget::prepare(&root.validated(), &spec("gated", 64 << 20, 8), &binary()).unwrap();
let pid = target.pid();
let path = root.path.join(target.cgroup().name());
let limits = target.cgroup().applied_limits().unwrap();
assert_eq!(limits.memory_bytes, 64 << 20);
assert_eq!(limits.swap_bytes, 0);
assert_eq!(limits.processes, 8);
assert_eq!(
(limits.cpu_quota_us, limits.cpu_period_us),
(10_000, 100_000)
);
assert!(limits.oom_group);
for _ in 0..3 {
assert!(
fs::read_to_string(format!("/proc/{pid}/status"))
.unwrap()
.contains("State:\tT (stopped)")
);
assert_eq!(
fs::read_to_string(path.join("cgroup.freeze"))
.unwrap()
.trim(),
"1"
);
assert_eq!(
fs::read_to_string(path.join("cgroup.procs"))
.unwrap()
.trim(),
pid.to_string()
);
std::thread::sleep(Duration::from_millis(10));
}
let failure = target.refuse_unimplemented_isolation();
assert_eq!(failure.failure.stage, ExecutionStage::Namespaces);
assert_eq!(failure.failure.code, ErrorCode::CapabilityUnavailable);
assert!(failure.cleanup_failure.is_none());
assert!(!Path::new(&format!("/proc/{pid}")).exists());
assert!(!path.exists());
println!(
"KF_GATED_PROOF pid={pid} memory={} swap=0 pids=8 cpu=10000/100000 frozen=1 refusal=namespaces reaped=1 removed=1",
limits.memory_bytes
);
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn unsafe_delegation_fails_at_cgroup_and_does_not_leave_bootstrap() {
let root = Root::new();
let before = children();
fs::set_permissions(&root.path, fs::Permissions::from_mode(0o777)).unwrap();
let result =
PreparedTarget::prepare(&root.validated(), &spec("unsafe", 64 << 20, 8), &binary());
let Err(failure) = result else {
panic!("unsafe root must fail");
};
assert_eq!(failure.failure.stage, ExecutionStage::Cgroup);
assert!(failure.cleanup_failure.is_none());
assert_eq!(children(), before);
assert!(!root.path.join("kf-unsafe").exists());
println!("KF_CGROUP_FAILURE_PROOF stage=cgroup bootstrap_reaped=1");
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn exclusive_tasks_and_kernel_directory_count_prevent_adoption_and_overbooking() {
let root = Root::new();
let config = root.validated();
let delegate = DelegatedCgroup::open(&config).unwrap();
let mut one = delegate.create_task(&spec("one", 64 << 20, 8)).unwrap();
assert!(delegate.create_task(&spec("one", 64 << 20, 8)).is_err());
let mut two = delegate.create_task(&spec("two", 64 << 20, 8)).unwrap();
assert!(delegate.create_task(&spec("three", 64 << 20, 8)).is_err());
assert_eq!(one.applied_limits().unwrap().memory_bytes, 64 << 20);
one.kill_and_remove(Duration::from_secs(1)).unwrap();
one.kill_and_remove(Duration::from_secs(1)).unwrap();
two.kill_and_remove(Duration::from_secs(1)).unwrap();
assert!(!root.path.join("kf-one").exists());
assert!(!root.path.join("kf-two").exists());
println!("KF_EXCLUSIVE_PROOF collision_rejected=1 max_tasks=2 idempotent_cleanup=1");
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn changed_limits_are_rejected_while_child_is_still_stopped() {
let root = Root::new();
let delegate = DelegatedCgroup::open(&root.validated()).unwrap();
let mut group = delegate.create_task(&spec("changed", 64 << 20, 8)).unwrap();
let path = root.path.join(group.name());
fs::write(path.join("memory.max"), (32 << 20).to_string()).unwrap();
let mut child = BlockedTarget::spawn(&binary()).unwrap();
assert!(group.enroll(&mut child).is_err());
child.verify_stopped().unwrap();
assert!(
fs::read_to_string(path.join("cgroup.procs"))
.unwrap()
.trim()
.is_empty()
);
group.kill_and_remove(Duration::from_secs(1)).unwrap();
assert_eq!(child.terminate().unwrap().signal(), Some(9));
println!("KF_CHANGED_LIMIT_PROOF enrollment_rejected=1 blocked=1 terminated=1");
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn request_cannot_exceed_operator_ceiling() {
let root = Root::new();
let delegate = DelegatedCgroup::open(&root.validated()).unwrap();
assert!(
delegate
.create_task(&spec("excess", 1024 << 20, 8))
.is_err()
);
assert!(!root.path.join("kf-excess").exists());
println!("KF_CEILING_PROOF request_excess_rejected=1");
}
fn counter(raw: &str, key: &str) -> u64 {
raw.lines()
.find_map(|line| {
line.split_once(' ')
.filter(|(name, _)| *name == key)
.map(|(_, value)| value.parse().unwrap())
})
.unwrap()
}
fn limit_probe(kind: &str) {
let root = Root::new();
let delegate = DelegatedCgroup::open(&root.validated()).unwrap();
let processes = if kind == "pids" { 1 } else { 8 };
let mut task = delegate
.create_task(&spec(kind, 32 << 20, processes))
.unwrap();
let path = root.path.join(task.name());
let mut child = BlockedTarget::spawn(Path::new(&format!("/kf/bin/limit-{kind}"))).unwrap();
task.enroll(&mut child).unwrap();
// VM-only fixed probes: deliberately exercise controllers after admission.
// Production APIs expose no thaw/continue or user execution permit.
fs::write(path.join("cgroup.freeze"), "0").unwrap();
let pid = rustix::process::Pid::from_raw(child.pid() as i32).unwrap();
let pidfd = rustix::process::pidfd_open(pid, rustix::process::PidfdFlags::empty()).unwrap();
rustix::process::pidfd_send_signal(&pidfd, rustix::process::Signal::CONT).unwrap();
let deadline = Instant::now() + Duration::from_secs(10);
let status = loop {
if let Some(status) = child.poll_exit().unwrap() {
break status;
}
assert!(Instant::now() < deadline, "bounded fixed resource probe");
std::thread::sleep(Duration::from_millis(5));
};
let cpu = fs::read_to_string(path.join("cpu.stat")).unwrap();
let memory = fs::read_to_string(path.join("memory.events")).unwrap();
let pids = fs::read_to_string(path.join("pids.events")).unwrap();
match kind {
"pids" => {
assert!(status.success());
assert!(counter(&pids, "max") >= 1);
}
"memory" => {
assert_eq!(status.signal(), Some(9));
assert!(counter(&memory, "oom_kill") >= 1);
}
"cpu" => {
assert!(status.success());
assert!(counter(&cpu, "usage_usec") >= 150_000);
assert!(counter(&cpu, "nr_throttled") >= 1);
}
_ => unreachable!(),
}
println!(
"KF_LIMIT_PROOF kind={kind} status={status} cpu={cpu:?} memory={memory:?} pids={pids:?}"
);
task.kill_and_remove(Duration::from_secs(1)).unwrap();
assert!(!path.exists());
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn pids_controller_actually_denies_fork() {
limit_probe("pids");
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn memory_controller_actually_kills_at_limit() {
limit_probe("memory");
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn cpu_controller_actually_throttles_fixed_work() {
limit_probe("cpu");
}
+35
View File
@@ -0,0 +1,35 @@
/* Fixed VM-only resource probes, never submitted programs or host test commands. */
#include <errno.h>
#include <signal.h>
#include <stdlib.h>
#include <sys/prctl.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>
int main(void) {
if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0 || getppid() <= 1) return 40;
raise(SIGSTOP);
#if KF_FIXTURE == 1
pid_t pid = fork();
if (pid == -1) return errno == EAGAIN ? 0 : 41;
if (pid == 0) _exit(42);
waitpid(pid, NULL, 0);
return 42;
#elif KF_FIXTURE == 2
size_t size = 128u * 1024u * 1024u;
volatile unsigned char *data = malloc(size);
if (data == NULL) return 43;
for (size_t i = 0; i < size; i += 4096) data[i] = 1;
free((void *) data);
return 44;
#elif KF_FIXTURE == 3
clock_t start = clock();
volatile unsigned long count = 0;
while (clock() - start < CLOCKS_PER_SEC * 3 / 20) ++count;
return count == 0 ? 45 : 0;
#else
#error KF_FIXTURE must be 1, 2, or 3
#endif
}