feat(sandbox): build private read-only roots with pivot_root

This commit is contained in:
2026-10-07 10:02:21 +08:00
parent f22308080f
commit 0f68eb5fb0
13 changed files with 888 additions and 32 deletions
+39 -5
View File
@@ -5,7 +5,7 @@ Cargo workspace 分为五个 crate:
| Crate | 职责 | 当前能力 |
| --- | --- | --- |
| `kf-contracts` | 跨服务通知和执行契约 | 最小 v1 通知、共享 JSON 样例检查 |
| `kf-sandbox` | Linux 隔离、限制、计量与清理 | 规格/配置检查、内部 cgroup/namespace 初始化;`run` 拒绝执行 |
| `kf-sandbox` | Linux 隔离、限制、计量与清理 | 规格/配置检查、内部 cgroup/namespace/运行根初始化;`run` 拒绝执行 |
| `kf-engine` | 语言适配、测试点编排、比较与确定性汇总 | 预留库边界,无模拟判定 |
| `kf-worker` | 拉取、租约、工件准备与本地编排 | `--check` 入口,无消费循环 |
| `kf-coordinator` | 节点登记、租约、状态与结果写入权 | HTTP 存活检查,无任务端点 |
@@ -165,13 +165,14 @@ memory.max、交换空间 0、pids.max、cpu.max 和 memory.oom.group,限制
负责进程与控制编排。它的 `NamespacedTarget` 只推进固定的可信初始化,不接收用户命令。
它沿用停止目标及 cgroup 纳入流程;确认限额、成员和冻结状态后,通过私有 socketpair
发送初始化命令,再仅在此阶段解除冻结和继续 bootstrap。通道仍使用 1 KiB 帧及绝对截止时间,
协议只含 initialize/cancel 和初始化结果,没有用户 argv、路径或 exec 命令。
协议只含 initialize/cancel、固定运行根预算和初始化结果,没有用户 argv、宿主路径或 exec 命令。
socketpair 的 SO_PEERCRED 标识创建者,不能当作子进程 PID 认证;安全依据是可信 Command
独占继承的端点。bootstrap 在 unshare 前核对创建者。PID 1 的对端位于祖先 PID namespace,
内核会把该对端 PID 映射为 0,rustix UCred 的非零 Pid 类型不能表示它,因此 PID 1 只使用
已经认证并独占继承的端点,不在该角色再次调用 socket_peercred。控制 socket 暂占内部入口的
FD 0,stdout/stderr 为 null;这里没有用户 stdio,最终 FD 白名单和 stdio 收集仍待实现。
FD 0,通常 stdout/stderr 为 null;运行根初始化暂借 FD 1/2 传递可信目录能力,切根后恢复为 null。
这里没有用户 stdio,最终 FD 白名单和 stdio 收集仍待实现。
初始化要求原 user namespace 中的 root helper,建立 mount/PID/IPC/UTS/network/cgroup
namespace,并立即把挂载传播递归设为 private。P0 选定的身份模型保留原 user namespace;
@@ -186,7 +187,7 @@ namespace,负责等待这个 init;用户进程树的 PID-1 回收循环尚
cgroup namespace 在实际任务成员关系建立后创建,PID 1 核对自己的 cgroup 视图为 `0::/`。
它只虚拟化 cgroup 路径;限额和终止继续由外部监督器持有的 cgroup FD 执行。完成此阶段后,
最小运行根、受限 proc/dev、身份及 seccomp 均未就绪;`refuse_unimplemented_root` 返回
namespace 单独初始化时运行根尚未就绪;`refuse_unimplemented_root` 返回
root_filesystem 阶段拒绝并清理,CLI run 继续拒绝。
`namespace_vm.rs` 与 `blocked_vm.rs` 使用同一个 VM 条件守卫。独立 VM 中的固定 C 探针验证
@@ -198,6 +199,39 @@ PID namespace 的后续子进程。探针由验收运行者在 VM 中进入目
参见 [PID namespace](https://man7.org/linux/man-pages/man7/pid_namespaces.7.html)、
[cgroup namespace](https://man7.org/linux/man-pages/man7/cgroup_namespaces.7.html)。
## 私有运行根与有界临时空间
`isolation/root.rs` 负责可信目录能力、固定挂载操作和内核读回;`launcher/root.rs` 的
`RootedTarget` 负责租用、阶段控制、失败回滚以及 namespace 终止后的目录清理。
它仅推进可信初始化;受限 proc/dev、任务身份、capabilities、seccomp 和用户进程树监督
仍待实现,成功建立运行根不能放行用户代码,CLI `run` 继续拒绝。
镜像和任务父目录只来自可信运行配置。逐级 NOFOLLOW 打开目录,要求 root 所有且不可被
组或其他用户写入;镜像的 work/tmp/proc/dev 挂载锚点必须是实际目录。每任务目录排他创建,
不接管同名旧目录,不递归删除未经租用的内容。部署方须保证镜像发布、内容及其底层存储
不可被其他进程修改;当前检查不替代可信镜像清单、输入工件的符号链接/硬链接/TOCTOU 校验。
监督器只传递已打开的镜像/任务目录 FD 和工作空间预算,协议不能选择宿主挂载路径。
unshare 后继承 FD 仍指向旧 mount,因此初始化器通过内核报告的可信路径在新 namespace
中重新逐级打开,并核对 dev/inode。镜像采用非递归 bind,排除源目录内已有子挂载,
再设为 read-only/nosuid/nodev;每任务另外挂载独立 work/tmp tmpfs。
最小根使用 `pivot_root(".", ".")` 的堆叠方式,detach 旧根并切到 `/work`,
释放所有初始化目录 FD 后才报告完成。初始 ramfs/rootfs 等不能 pivot 的环境返回阶段错误。
`workdir_bytes` 同时限制 work 和 tmp:按 4 KiB 向下取整后约 3/4 分给 work、1/4 分给 tmp,
总量不超过申请值,每个挂载也限制 inode 数。过小或不支持的预算在创建任务目录前拒绝。
work 为 nosuid/nodev,保留后续编译和执行能力;tmp 另加 noexec。
监督器从祖先 procfs 核对实际根的镜像 dev/inode、只读标志、scratch 文件系统/容量/标志,
拒绝 init 仍持有目录 FD 的情况,再保持 STOP/freeze。proc/dev 目前为空,必要设备尚未挂载。
`root_vm.rs` 只在规定的独立 VM 中验收真实 pivot、只读写入 EROFS、tmp 写满 ENOSPC、
跨任务 scratch 隔离、宿主挂载不变、排他租用和失败清理。固定探针由 VM 验收运行者进入
目标挂载 namespace;这不证明用户 exec、身份降权、完整根逃逸防护或进程树回收已完成。
普通工程检查将这些特权用例明确标记 ignored。
参见 [pivot_root](https://man7.org/linux/man-pages/man2/pivot_root.2.html) 和
[mount](https://man7.org/linux/man-pages/man2/mount.2.html)。
## 可信运行配置与启动前检查
```bash
@@ -209,7 +243,7 @@ cargo run --locked -p kf-sandbox -- check-runtime --config ../deploy/sandbox.run
所需接口包括 user/mount/PID/IPC/UTS/network/cgroup namespace、pidfd 等待、身份降权、seccomp filter,
以及实际委派子树中的 CPU/memory/pids 控制器、`cgroup.kill`、`memory.peak`、内存/进程事件和交换空间限制。
初始镜像类型固定为 `linux_gnu_cxx_x86_64_v1`:受控只读目录镜像、GNU C/C++ 编译器、动态加载器和
可信 `manifest.json`;镜像内容、权限和只读挂载验证仍未实现。
可信 `manifest.json`;内部运行根阶段已核对目录权限与实际只读挂载,镜像内容清单校验仍未实现。
配置由可信部署方提供,最多 64 KiB;不能由执行请求覆盖。`runtime.rs` 校验受控 state/tasks/image
目录布局、独立 cgroup 子树、普通 Worker 身份和独立任务 UID/GID 槽位;任务身份从 100000 起,
+16
View File
@@ -240,6 +240,22 @@ impl TaskCgroup {
self.verify_members(members)
}
pub(crate) fn begin_root_initialization(
&mut self,
members: &[u32],
init_pidfd: &OwnedFd,
) -> io::Result<()> {
self.verify_members(members)?;
self.applied_limits()?;
self.wait_event("frozen", "1", Duration::from_secs(1))?;
write_verify(&self.fd, "cgroup.freeze", "0")?;
self.wait_event("frozen", "0", Duration::from_secs(1))?;
Ok(rustix::process::pidfd_send_signal(
init_pidfd,
rustix::process::Signal::CONT,
)?)
}
fn verify_members(&self, expected: &[u32]) -> io::Result<()> {
let mut members = read(&self.fd, "cgroup.procs")?
.split_whitespace()
@@ -1,3 +1,4 @@
//! Fixed Linux isolation operations; launcher owns process/control orchestration.
pub mod namespaces;
pub(crate) mod root;
+338
View File
@@ -0,0 +1,338 @@
//! Trusted image/task directory capabilities and fixed private root operations.
//! Request argv/paths never select a host mount source or target.
use crate::runtime::ValidatedRuntimeConfig;
use crate::spec::ValidatedSpec;
use rustix::fd::OwnedFd;
use rustix::fs::{AtFlags, Mode, OFlags, StatVfsMountFlags};
use rustix::mount::{MountFlags, UnmountFlags};
use std::ffi::CString;
use std::io;
use std::os::unix::fs::MetadataExt;
use std::path::{Component, Path};
const DIR: OFlags = OFlags::RDONLY
.union(OFlags::DIRECTORY)
.union(OFlags::NOFOLLOW)
.union(OFlags::CLOEXEC);
const MAX_STORAGE: u64 = 256 * 1024 * 1024;
#[derive(Clone, Copy, Debug)]
pub struct RootBudget {
pub work_bytes: u64,
pub tmp_bytes: u64,
}
impl RootBudget {
pub(crate) fn for_spec(spec: &ValidatedSpec) -> io::Result<Self> {
let total = spec.spec().limits.workdir_bytes / 4096 * 4096;
let tmp_bytes = total / 4 / 4096 * 4096;
let budget = Self {
work_bytes: total - tmp_bytes,
tmp_bytes,
};
if !budget.valid() {
return Err(invalid("unsupported root storage budget"));
}
Ok(budget)
}
pub(crate) fn valid(&self) -> bool {
self.work_bytes >= 4096
&& self.tmp_bytes >= 4096
&& self
.work_bytes
.checked_add(self.tmp_bytes)
.is_some_and(|n| n <= MAX_STORAGE)
&& self.work_bytes.is_multiple_of(4096)
&& self.tmp_bytes.is_multiple_of(4096)
}
}
pub(crate) struct RootDescriptors {
pub image: OwnedFd,
pub task: OwnedFd,
}
#[derive(Debug)]
pub(crate) struct RootError {
pub source: io::Error,
pub cleanup_error: Option<io::Error>,
}
impl From<io::Error> for RootError {
fn from(source: io::Error) -> Self {
Self {
source,
cleanup_error: None,
}
}
}
impl From<rustix::io::Errno> for RootError {
fn from(source: rustix::io::Errno) -> Self {
io::Error::from(source).into()
}
}
pub(crate) struct RootLease {
parent: OwnedFd,
task: OwnedFd,
image: OwnedFd,
name: String,
removed: bool,
root_created: bool,
pub budget: RootBudget,
}
fn invalid(message: &'static str) -> io::Error {
io::Error::new(io::ErrorKind::InvalidInput, message)
}
fn safe_directory(fd: &OwnedFd) -> io::Result<()> {
let stat = rustix::fs::fstat(fd)?;
if rustix::fs::FileType::from_raw_mode(stat.st_mode) != rustix::fs::FileType::Directory
|| stat.st_uid != 0
|| stat.st_mode & 0o022 != 0
{
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"unsafe image/task directory owner or mode",
));
}
Ok(())
}
fn open_directory(path: &Path) -> io::Result<OwnedFd> {
if !path.is_absolute() {
return Err(invalid("root capability must be absolute"));
}
let mut fd = rustix::fs::open("/", DIR, Mode::empty())?;
safe_directory(&fd)?;
for component in path.components() {
match component {
Component::RootDir => {}
Component::Normal(name) => {
fd = rustix::fs::openat(&fd, name, DIR, Mode::empty())?;
safe_directory(&fd)?;
}
_ => return Err(invalid("noncanonical root capability")),
}
}
Ok(fd)
}
fn verify_anchors(image: &OwnedFd) -> io::Result<()> {
// Sources contain directories rather than symlink mount targets. Image
// publication/content immutability is an operator responsibility; artifact
// TOCTOU/export validation is a separate, still-unimplemented stage.
for name in ["work", "tmp", "proc", "dev"] {
let fd = rustix::fs::openat(image, name, DIR, Mode::empty())?;
safe_directory(&fd)?;
}
Ok(())
}
fn rebase_directory(fd: &OwnedFd) -> io::Result<OwnedFd> {
use rustix::fd::AsRawFd;
// Inherited FDs retain their original mount references across unshare.
// Resolve the kernel-reported trusted path in the new namespace, walk it
// without symlinks, and require the same inode before using it as a mount
// target. No request controls this path or the root-owned ancestors.
let path = std::fs::read_link(format!("/proc/self/fd/{}", fd.as_raw_fd()))?;
let rebased = open_directory(&path)?;
let a = rustix::fs::fstat(fd)?;
let b = rustix::fs::fstat(&rebased)?;
if a.st_dev != b.st_dev || a.st_ino != b.st_ino {
return Err(invalid("directory capability identity changed"));
}
Ok(rebased)
}
impl RootLease {
pub(crate) fn prepare(
config: &ValidatedRuntimeConfig,
spec: &ValidatedSpec,
) -> Result<Self, RootError> {
spec.spec()
.clone()
.validate(&config.spec_policy())
.map_err(|_| invalid("root request exceeds operator ceiling"))?;
let budget = RootBudget::for_spec(spec)?;
let image = open_directory(Path::new(&config.config().image_root))?;
verify_anchors(&image)?;
let parent = open_directory(Path::new(&config.config().tasks_root))?;
let name = format!("kf-{}", spec.spec().run_id);
rustix::fs::mkdirat(&parent, name.as_str(), Mode::from_raw_mode(0o700))?;
let task = match rustix::fs::openat(&parent, name.as_str(), DIR, Mode::empty()) {
Ok(fd) => fd,
Err(source) => {
return Err(RootError {
source: source.into(),
cleanup_error: rustix::fs::unlinkat(&parent, name.as_str(), AtFlags::REMOVEDIR)
.err()
.map(Into::into),
});
}
};
let mut lease = Self {
parent,
task,
image,
name,
removed: false,
root_created: false,
budget,
};
if let Err(source) = (|| {
safe_directory(&lease.task)?;
rustix::fs::mkdirat(&lease.task, "root", Mode::from_raw_mode(0o700))?;
lease.root_created = true;
Ok::<_, io::Error>(())
})() {
return Err(RootError {
source,
cleanup_error: lease.cleanup().err(),
});
}
Ok(lease)
}
pub(crate) fn descriptors(&self) -> io::Result<RootDescriptors> {
Ok(RootDescriptors {
image: rustix::io::fcntl_dupfd_cloexec(&self.image, 3)?,
task: rustix::io::fcntl_dupfd_cloexec(&self.task, 3)?,
})
}
pub(crate) fn verify_root(&self, pid: u32) -> io::Result<()> {
let image = rustix::fs::fstat(&self.image)?;
let actual = std::fs::metadata(format!("/proc/{pid}/root"))?;
if actual.dev() != image.st_dev || actual.ino() != image.st_ino {
return Err(invalid("actual root does not match leased image"));
}
let root = rustix::fs::open(
format!("/proc/{pid}/root"),
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC,
Mode::empty(),
)?;
if !rustix::fs::fstatvfs(&root)?
.f_flag
.contains(StatVfsMountFlags::RDONLY)
{
return Err(invalid("image root is not read-only"));
}
for (name, bytes, tmp) in [
("work", self.budget.work_bytes, false),
("tmp", self.budget.tmp_bytes, true),
] {
let fd = rustix::fs::openat(&root, name, DIR, Mode::empty())?;
let fs = rustix::fs::fstatvfs(&fd)?;
let required = StatVfsMountFlags::NOSUID
| StatVfsMountFlags::NODEV
| if tmp {
StatVfsMountFlags::NOEXEC
} else {
StatVfsMountFlags::empty()
};
if rustix::fs::fstatfs(&fd)?.f_type as u64 != 0x0102_1994
|| fs.f_blocks.checked_mul(fs.f_frsize) != Some(bytes)
|| !fs.f_flag.contains(required)
|| fs.f_flag.contains(StatVfsMountFlags::RDONLY)
{
return Err(invalid("scratch mount verification failed"));
}
}
for entry in std::fs::read_dir(format!("/proc/{pid}/fd"))? {
if std::fs::metadata(entry?.path())?.is_dir() {
return Err(invalid("initialization directory FD leaked"));
}
}
Ok(())
}
pub(crate) fn cleanup(&mut self) -> io::Result<()> {
if self.removed {
return Ok(());
}
// No recursive deletion or adoption. All mounts/data exist only in the
// helper's private namespace, which must be terminated before this.
if self.root_created {
rustix::fs::unlinkat(&self.task, "root", AtFlags::REMOVEDIR)?;
self.root_created = false;
}
rustix::fs::unlinkat(&self.parent, self.name.as_str(), AtFlags::REMOVEDIR)?;
self.removed = true;
Ok(())
}
}
impl Drop for RootLease {
fn drop(&mut self) {
let _ = self.cleanup();
}
}
fn scratch(path: &str, bytes: u64, tmp: bool) -> io::Result<()> {
let flags = MountFlags::NOSUID
| MountFlags::NODEV
| if tmp {
MountFlags::NOEXEC
} else {
MountFlags::empty()
};
let data = CString::new(format!(
"size={bytes},nr_inodes={},mode={}",
(bytes / 4096).clamp(64, 65536),
if tmp { "1777" } else { "0700" }
))
.map_err(|_| invalid("scratch mount options"))?;
rustix::mount::mount("tmpfs", path, "tmpfs", flags, Some(data.as_c_str()))?;
Ok(())
}
/// Called only in the fresh PID-1 role after actual namespace verification.
pub(crate) fn install(budget: RootBudget) -> io::Result<()> {
crate::launcher::entry::ensure_single_threaded()?;
if !budget.valid() {
return Err(invalid("invalid root storage budget"));
}
let image = rustix::io::fcntl_dupfd_cloexec(rustix::stdio::stdout(), 3)?;
let task = rustix::io::fcntl_dupfd_cloexec(rustix::stdio::stderr(), 3)?;
safe_directory(&image)?;
safe_directory(&task)?;
verify_anchors(&image)?;
let image = rebase_directory(&image)?;
let task = rebase_directory(&task)?;
let null = std::fs::File::options()
.read(true)
.write(true)
.open("/dev/null")?;
use rustix::fd::AsRawFd;
let target = format!("/proc/self/fd/{}/root", task.as_raw_fd());
// Nonrecursive bind deliberately excludes any source submounts. One image
// bind is made read-only before adding our separate scratch mounts.
rustix::mount::mount_bind(
format!("/proc/self/fd/{}", image.as_raw_fd()),
target.as_str(),
)?;
rustix::mount::mount_remount(
target.as_str(),
MountFlags::BIND | MountFlags::RDONLY | MountFlags::NOSUID | MountFlags::NODEV,
"",
)?;
scratch(&format!("{target}/work"), budget.work_bytes, false)?;
scratch(&format!("{target}/tmp"), budget.tmp_bytes, true)?;
let root = rustix::fs::openat(&task, "root", DIR, Mode::empty())?;
rustix::process::fchdir(&root)?;
// Documented stacking form avoids modifying the immutable image to create
// an old-root directory. Initial ramfs/rootfs remains unsupported/refused.
rustix::process::pivot_root(".", ".")?;
rustix::mount::unmount(".", UnmountFlags::DETACH)?;
rustix::process::chdir("/")?;
if !rustix::fs::statvfs("/")?
.f_flag
.contains(StatVfsMountFlags::RDONLY)
{
return Err(invalid("root read-only verification failed"));
}
// Release all image/task/old-root directory capabilities before reporting.
rustix::stdio::dup2_stdout(&null)?;
rustix::stdio::dup2_stderr(&null)?;
drop(root);
drop(task);
drop(image);
drop(null);
rustix::process::chdir("/work")?;
Ok(())
}
+17 -4
View File
@@ -51,11 +51,12 @@ pub struct BlockedTarget {
impl BlockedTarget {
/// `executable` is the operator's trusted sandbox binary, never a request argv.
pub fn spawn(executable: &Path) -> io::Result<Self> {
Self::spawn_inner(executable, None)
Self::spawn_inner(executable, None, None)
}
pub(crate) fn spawn_initializing(
executable: &Path,
roots: Option<crate::isolation::root::RootDescriptors>,
) -> io::Result<(Self, super::protocol::ControlChannel)> {
let (parent, child) = UnixStream::pair()?;
// socketpair credentials identify its creator, not the eventual child.
@@ -70,11 +71,15 @@ impl BlockedTarget {
},
Instant::now() + super::protocol::MAX_CONTROL_TIMEOUT,
)?;
let target = Self::spawn_inner(executable, Some(child))?;
let target = Self::spawn_inner(executable, Some(child), roots)?;
Ok((target, channel))
}
fn spawn_inner(executable: &Path, initialization: Option<UnixStream>) -> io::Result<Self> {
fn spawn_inner(
executable: &Path,
initialization: Option<UnixStream>,
roots: Option<crate::isolation::root::RootDescriptors>,
) -> io::Result<Self> {
// A ignored/caught SIGCHLD could reap/recycle a PID before pidfd_open.
// Fresh helpers retain the default disposition; reject inherited changes.
let status = std::fs::read_to_string("/proc/self/status")?;
@@ -106,6 +111,13 @@ impl BlockedTarget {
.arg("--initialize-namespaces")
.stdin(Stdio::from(fd));
}
if let Some(roots) = roots {
command
.arg("--initialize-root")
.stdout(Stdio::from(roots.image))
.stderr(Stdio::from(roots.task))
.current_dir("/");
}
let mut child = command.spawn()?;
let Some(pid) = Pid::from_raw(child.id() as i32) else {
let _ = child.kill();
@@ -266,9 +278,10 @@ impl PreparedTarget {
config: &ValidatedRuntimeConfig,
spec: &ValidatedSpec,
executable: &Path,
roots: Option<crate::isolation::root::RootDescriptors>,
) -> Result<(Self, super::protocol::ControlChannel), InitializationFailure> {
let (mut target, channel) =
BlockedTarget::spawn_initializing(executable).map_err(|error| {
BlockedTarget::spawn_initializing(executable, roots).map_err(|error| {
InitializationFailure {
failure: failure(
ExecutionStage::Preparation,
+2
View File
@@ -15,3 +15,5 @@ pub mod entry;
pub mod namespaces;
#[cfg(target_os = "linux")]
pub mod protocol;
#[cfg(target_os = "linux")]
pub mod root;
+128 -11
View File
@@ -8,6 +8,7 @@ use super::protocol::{
use crate::outcome::ExecutionStage;
use crate::runtime::ValidatedRuntimeConfig;
use crate::spec::ValidatedSpec;
use rustix::fd::OwnedFd;
use rustix::process::Signal;
use std::io;
use std::os::unix::net::UnixStream;
@@ -36,7 +37,7 @@ fn inherited_channel(parent_pid: u32) -> io::Result<ControlChannel> {
/// Fixed namespace bootstrap. The inherited socket is authenticated before
/// unshare; changing PID namespaces changes SO_PEERCRED's PID translation.
pub fn bootstrap(expected_parent: u32) -> io::Result<()> {
pub fn bootstrap(expected_parent: u32, initialize_root: bool) -> io::Result<()> {
super::entry::ensure_single_threaded()?;
rustix::process::set_parent_process_death_signal(Some(Signal::KILL))?;
if expected_parent <= 1
@@ -52,21 +53,38 @@ pub fn bootstrap(expected_parent: u32) -> io::Result<()> {
return Ok(());
}
NamespaceCommand::Initialize {} => {}
NamespaceCommand::PrepareRoot { .. } => {
return Err(denied("root command before namespaces"));
}
}
let setup = (|| {
crate::isolation::namespaces::create()?;
super::entry::ensure_single_threaded()?;
// NEWPID applies to children, so a second fresh exec becomes PID 1.
// No Rust initialization occurs in a custom fork/pre_exec callback.
let child = Command::new(std::env::current_exe()?)
let mut command = Command::new(std::env::current_exe()?);
command
.arg("namespace-init")
.arg("--parent-pid")
.arg(std::process::id().to_string())
.env_clear()
.stdin(Stdio::from(channel.duplicate_stream()?))
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()?;
.stderr(Stdio::null());
if initialize_root {
command
.arg("--initialize-root")
.stdout(Stdio::from(rustix::io::fcntl_dupfd_cloexec(
rustix::stdio::stdout(),
3,
)?))
.stderr(Stdio::from(rustix::io::fcntl_dupfd_cloexec(
rustix::stdio::stderr(),
3,
)?))
.current_dir("/");
}
let child = command.spawn()?;
Ok::<_, io::Error>(child)
})();
let mut child = match setup {
@@ -81,7 +99,16 @@ pub fn bootstrap(expected_parent: u32) -> io::Result<()> {
// PID 1 is the sole control writer now. Release the bootstrap's copies so
// control EOF has its actual meaning. The bootstrap owns/reaps PID 1 only;
// user descendants and the final init reaping loop are a separate stage.
rustix::stdio::dup2_stdin(std::fs::File::open("/dev/null")?)?;
let null = std::fs::File::options()
.read(true)
.write(true)
.open("/dev/null")?;
rustix::stdio::dup2_stdin(&null)?;
if initialize_root {
rustix::stdio::dup2_stdout(&null)?;
rustix::stdio::dup2_stderr(&null)?;
}
drop(null);
drop(channel);
let status = child.wait()?;
if !status.success() {
@@ -101,7 +128,7 @@ fn status_value(status: &str, name: &str) -> io::Result<u32> {
/// This helper runs before a new proc mount: the inherited procfs still reports
/// outer PIDs, allowing the post-prctl parent check even when getppid() is zero
/// in the new PID namespace. Later root setup must replace this inherited proc.
pub fn namespace_init(expected_outer_parent: u32) -> io::Result<()> {
pub fn namespace_init(expected_outer_parent: u32, initialize_root: bool) -> io::Result<()> {
super::entry::ensure_single_threaded()?;
if rustix::process::getpid().as_raw_pid() != 1 || rustix::process::getppid().is_some() {
return Err(denied("not the namespace init"));
@@ -127,14 +154,38 @@ pub fn namespace_init(expected_outer_parent: u32) -> io::Result<()> {
// Only the supervisor in the ancestor PID namespace stops PID 1. Signals
// sent within the namespace have special PID-1 restrictions. While it
// verifies/stops us, this bounded receive can only cancel/refuse.
let _ = channel.receive::<NamespaceCommand>()?;
match channel.receive::<NamespaceCommand>()? {
NamespaceCommand::PrepareRoot {
work_bytes,
tmp_bytes,
} if initialize_root => {
if let Err(error) =
crate::isolation::root::install(crate::isolation::root::RootBudget {
work_bytes,
tmp_bytes,
})
{
channel.send(&NamespaceEvent::Failed {
errno: error.raw_os_error().and_then(|n| u16::try_from(n).ok()),
})?;
return Err(error);
}
channel.send(&NamespaceEvent::RootInitialized {})?;
let _ = channel.receive::<NamespaceCommand>()?;
}
NamespaceCommand::Cancel {} => return Ok(()),
_ => {}
}
Err(denied("namespace init has no execution permit"))
}
/// Actual namespaces only, not complete isolation or an execution permit.
/// Both helper processes remain in the same bounded, frozen task cgroup.
pub struct NamespacedTarget {
prepared: PreparedTarget,
pub(crate) prepared: PreparedTarget,
control: ControlChannel,
init_pidfd: OwnedFd,
allow_root: bool,
init_pid: u32,
ids: NamespaceIds,
}
@@ -145,8 +196,26 @@ impl NamespacedTarget {
spec: &ValidatedSpec,
executable: &Path,
) -> Result<Self, InitializationFailure> {
Self::prepare_inner(config, spec, executable, None)
}
pub(crate) fn prepare_for_root(
config: &ValidatedRuntimeConfig,
spec: &ValidatedSpec,
executable: &Path,
roots: crate::isolation::root::RootDescriptors,
) -> Result<Self, InitializationFailure> {
Self::prepare_inner(config, spec, executable, Some(roots))
}
fn prepare_inner(
config: &ValidatedRuntimeConfig,
spec: &ValidatedSpec,
executable: &Path,
roots: Option<crate::isolation::root::RootDescriptors>,
) -> Result<Self, InitializationFailure> {
let allow_root = roots.is_some();
let (mut prepared, mut control) =
PreparedTarget::prepare_initializing(config, spec, executable)?;
PreparedTarget::prepare_initializing(config, spec, executable, roots)?;
let initialize = (|| {
let parent_ids = NamespaceIds::observe(std::process::id())?;
control.send(&NamespaceCommand::Initialize {})?;
@@ -197,18 +266,66 @@ impl NamespacedTarget {
prepared
.cgroup
.finish_namespace_initialization(&[prepared.pid(), init_pid])?;
Ok::<_, io::Error>((init_pid, ids))
Ok::<_, io::Error>((init_pid, ids, pidfd))
})();
match initialize {
Ok((init_pid, ids)) => Ok(Self {
Ok((init_pid, ids, init_pidfd)) => Ok(Self {
prepared,
init_pid,
ids,
control,
init_pidfd,
allow_root,
}),
Err(error) => Err(prepared.refuse_at(ExecutionStage::Namespaces, Some(&error))),
}
}
pub(crate) fn initialize_root(
&mut self,
budget: crate::isolation::root::RootBudget,
) -> std::io::Result<()> {
if !self.allow_root {
return Err(denied("root descriptors not inherited"));
}
self.control.send(&NamespaceCommand::PrepareRoot {
work_bytes: budget.work_bytes,
tmp_bytes: budget.tmp_bytes,
})?;
self.prepared
.cgroup
.begin_root_initialization(&[self.prepared.pid(), self.init_pid], &self.init_pidfd)?;
match self.control.receive::<NamespaceEvent>()? {
NamespaceEvent::RootInitialized {} => {}
NamespaceEvent::Failed { errno: Some(errno) } => {
return Err(io::Error::from_raw_os_error(i32::from(errno)));
}
_ => return Err(denied("root initialization failed")),
}
rustix::process::pidfd_send_signal(&self.init_pidfd, Signal::STOP)?;
let deadline = Instant::now() + Duration::from_secs(2);
loop {
let status = std::fs::read_to_string(format!("/proc/{}/status", self.init_pid))?;
if status_value(&status, "PPid:")? != self.prepared.pid() {
return Err(denied("root init parent mismatch"));
}
if status.lines().any(|l| l == "State:\tT (stopped)") {
break;
}
if Instant::now() >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"root init stop deadline",
));
}
std::thread::sleep(Duration::from_millis(2));
}
self.prepared
.cgroup
.finish_namespace_initialization(&[self.prepared.pid(), self.init_pid])?;
Ok(())
}
pub fn init_pid(&self) -> u32 {
self.init_pid
}
+13 -1
View File
@@ -40,6 +40,7 @@ pub enum ChildMessage {
#[serde(tag = "command", rename_all = "snake_case", deny_unknown_fields)]
pub enum NamespaceCommand {
Initialize {},
PrepareRoot { work_bytes: u64, tmp_bytes: u64 },
Cancel {},
}
@@ -47,6 +48,7 @@ pub enum NamespaceCommand {
#[serde(tag = "event", rename_all = "snake_case", deny_unknown_fields)]
pub enum NamespaceEvent {
Initialized { init_pid: u32 },
RootInitialized {},
Failed { errno: Option<u16> },
Cancelled {},
}
@@ -64,7 +66,17 @@ impl sealed::Sealed for ParentMessage {}
impl sealed::Sealed for NamespaceCommand {}
impl WireMessage for NamespaceCommand {
fn valid(&self) -> bool {
true
match self {
Self::PrepareRoot {
work_bytes,
tmp_bytes,
} => crate::isolation::root::RootBudget {
work_bytes: *work_bytes,
tmp_bytes: *tmp_bytes,
}
.valid(),
_ => true,
}
}
}
impl sealed::Sealed for NamespaceEvent {}
+107
View File
@@ -0,0 +1,107 @@
//! Supervisor ownership of a leased root and a still-gated namespace target.
use super::blocked::{InitializationFailure, failure};
use super::namespaces::NamespacedTarget;
use crate::isolation::root::RootLease;
use crate::outcome::{ErrorCode, ExecutionStage};
use crate::runtime::ValidatedRuntimeConfig;
use crate::spec::ValidatedSpec;
use std::io;
use std::path::Path;
pub use crate::isolation::root::RootBudget;
pub struct RootedTarget {
// Terminate the private namespace before deleting its outside directory.
target: NamespacedTarget,
lease: RootLease,
}
fn attach_cleanup(error: &mut InitializationFailure, cleanup: Option<io::Error>) {
if error.cleanup_failure.is_none() {
error.cleanup_failure = cleanup.as_ref().map(|e| {
failure(
ExecutionStage::Cleanup,
ErrorCode::CleanupIncomplete,
Some(e),
)
});
}
}
impl RootedTarget {
pub fn prepare(
config: &ValidatedRuntimeConfig,
spec: &ValidatedSpec,
executable: &Path,
) -> Result<Self, InitializationFailure> {
let mut lease =
RootLease::prepare(config, spec).map_err(|error| InitializationFailure {
failure: failure(
ExecutionStage::RootFilesystem,
ErrorCode::OperationFailed,
Some(&error.source),
),
cleanup_failure: error.cleanup_error.as_ref().map(|e| {
failure(
ExecutionStage::Cleanup,
ErrorCode::CleanupIncomplete,
Some(e),
)
}),
})?;
let descriptors = lease.descriptors().map_err(|error| {
let mut failure = InitializationFailure {
failure: failure(
ExecutionStage::RootFilesystem,
ErrorCode::OperationFailed,
Some(&error),
),
cleanup_failure: None,
};
attach_cleanup(&mut failure, lease.cleanup().err());
failure
})?;
let mut target =
match NamespacedTarget::prepare_for_root(config, spec, executable, descriptors) {
Ok(target) => target,
Err(mut failure) => {
attach_cleanup(&mut failure, lease.cleanup().err());
return Err(failure);
}
};
if let Err(error) = (|| {
target.initialize_root(lease.budget)?;
lease.verify_root(target.init_pid())?;
Ok::<_, io::Error>(())
})() {
let mut failure = target
.prepared
.refuse_at(ExecutionStage::RootFilesystem, Some(&error));
attach_cleanup(&mut failure, lease.cleanup().err());
return Err(failure);
}
Ok(Self { target, lease })
}
pub fn init_pid(&self) -> u32 {
self.target.init_pid()
}
pub fn bootstrap_pid(&self) -> u32 {
self.target.bootstrap_pid()
}
pub fn cgroup(&self) -> &crate::cgroup::TaskCgroup {
self.target.cgroup()
}
pub fn budget(&self) -> RootBudget {
self.lease.budget
}
/// Restricted proc/dev, identity, capabilities and seccomp are still absent.
pub fn refuse_remaining_isolation(mut self) -> InitializationFailure {
let mut failure = self
.target
.prepared
.refuse_at(ExecutionStage::RootFilesystem, None);
attach_cleanup(&mut failure, self.lease.cleanup().err());
failure
}
}
+16 -10
View File
@@ -43,6 +43,8 @@ enum Command {
parent_pid: u32,
#[arg(long)]
initialize_namespaces: bool,
#[arg(long, requires = "initialize_namespaces")]
initialize_root: bool,
},
/// Internal fresh PID-1 helper; refuses arbitrary commands and user exec.
#[cfg(target_os = "linux")]
@@ -50,6 +52,8 @@ enum Command {
NamespaceInit {
#[arg(long)]
parent_pid: u32,
#[arg(long)]
initialize_root: bool,
},
}
@@ -89,9 +93,10 @@ fn main() -> ExitCode {
Command::Bootstrap {
parent_pid,
initialize_namespaces,
initialize_root,
} => {
let result = if initialize_namespaces {
kf_sandbox::launcher::namespaces::bootstrap(parent_pid)
kf_sandbox::launcher::namespaces::bootstrap(parent_pid, initialize_root)
} else {
kf_sandbox::launcher::blocked::bootstrap(parent_pid)
};
@@ -104,15 +109,16 @@ fn main() -> ExitCode {
}
}
#[cfg(target_os = "linux")]
Command::NamespaceInit { parent_pid } => {
match kf_sandbox::launcher::namespaces::namespace_init(parent_pid) {
Ok(()) => ExitCode::from(2),
Err(error) => spec_error(
"namespace_init_refused",
Some(serde_json::json!({"io_kind":format!("{:?}",error.kind())})),
),
}
}
Command::NamespaceInit {
parent_pid,
initialize_root,
} => match kf_sandbox::launcher::namespaces::namespace_init(parent_pid, initialize_root) {
Ok(()) => ExitCode::from(2),
Err(error) => spec_error(
"namespace_init_refused",
Some(serde_json::json!({"io_kind":format!("{:?}",error.kind())})),
),
},
}
}
+37
View File
@@ -0,0 +1,37 @@
/* Fixed, bounded probe: compile/run only in the independently capped VM. */
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <sched.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/statfs.h>
#include <sys/wait.h>
#include <unistd.h>
static int probe(int ns,int root){
if(setns(ns,CLONE_NEWNS)||fchdir(root)||chroot(".")||chdir("/"))return 10;
close(ns);close(root);
int fd=open("/image-sentinel",O_WRONLY|O_TRUNC|O_CLOEXEC);if(fd!=-1||errno!=EROFS)return 11;
fd=open("/image-sentinel",O_RDONLY|O_CLOEXEC);char value[32]={0};if(fd<0||read(fd,value,31)!=15||strcmp(value,"immutable-image"))return 12;close(fd);
if(access("/kf",F_OK)==0||access("/sys",F_OK)==0||access("/var/lib/kf-judge",F_OK)==0)return 13;
struct statfs work,tmp;if(statfs("/work",&work)||statfs("/tmp",&tmp)||work.f_type!=0x01021994||tmp.f_type!=0x01021994||!memcmp(&work.f_fsid,&tmp.f_fsid,sizeof work.f_fsid))return 14;
fd=open("/work/isolated-sentinel",O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC,0600);if(fd<0||write(fd,"task",4)!=4)return 15;close(fd);
fd=open("/tmp/bounded",O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC,0600);if(fd<0)return 16;
char bytes[65536]={0};unsigned long total=0;int exhausted=0;
while(total<=20UL*1024*1024){ssize_t n=write(fd,bytes,sizeof bytes);if(n<0){if(errno==ENOSPC)exhausted=1;break;}total+=(unsigned long)n;}
close(fd);unlink("/tmp/bounded");if(!exhausted||total!=16UL*1024*1024)return 17;
return 0;
}
int main(int argc,char **argv){
alarm(8);if(argc!=2)return 2;for(char *p=argv[1];*p;p++)if(*p<'0'||*p>'9')return 2;
char path[128];if(snprintf(path,sizeof path,"/proc/%s/ns/mnt",argv[1])>=(int)sizeof path)return 2;
int ns=open(path,O_RDONLY|O_CLOEXEC);
if(snprintf(path,sizeof path,"/proc/%s/root",argv[1])>=(int)sizeof path)return 2;
int root=open(path,O_RDONLY|O_DIRECTORY|O_CLOEXEC);if(ns<0||root<0)return 3;
pid_t child=fork();if(child==0)_exit(probe(ns,root));close(ns);close(root);int status=0;
if(child<0||waitpid(child,&status,0)!=child||!WIFEXITED(status)||WEXITSTATUS(status)){fprintf(stderr,"root probe status=%d\n",status);return 4;}
puts("KF_ROOT_PROBE readonly=1 old_root_absent=1 scratch=1 tmp_enospc=1");return 0;
}
+173
View File
@@ -0,0 +1,173 @@
#![cfg(target_os = "linux")]
//! Real root/pivot acceptance requires the externally capped independent VM.
mod support;
use kf_sandbox::launcher::root::RootedTarget;
use kf_sandbox::outcome::ExecutionStage;
use std::fs;
use std::path::Path;
use std::process::Command;
use std::time::{Duration, Instant};
use support::{Root, binary, children, spec};
fn gone(pid: u32) {
let deadline = Instant::now() + Duration::from_secs(2);
while Path::new(&format!("/proc/{pid}")).exists() {
assert!(
Instant::now() < deadline,
"VM init must reap orphaned helper"
);
std::thread::sleep(Duration::from_millis(5));
}
}
fn task_path(id: &str) -> String {
format!("/var/lib/kf-judge/tasks/kf-{id}")
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn real_pivot_readonly_image_and_independent_bounded_scratch_are_verified() {
let root = Root::new();
let before = fs::read_to_string("/proc/self/mountinfo").unwrap();
let one = RootedTarget::prepare(&root.validated(), &spec("root-one", 64 << 20, 8), &binary())
.unwrap();
let two = RootedTarget::prepare(&root.validated(), &spec("root-two", 64 << 20, 8), &binary())
.unwrap();
let output = Command::new("/kf/bin/root-probe")
.arg(one.init_pid().to_string())
.env_clear()
.output()
.unwrap();
assert!(
output.status.success(),
"fixed root probe failed: {:?} {:?}",
output.stdout,
output.stderr
);
assert_eq!(
String::from_utf8(output.stdout).unwrap().trim(),
"KF_ROOT_PROBE readonly=1 old_root_absent=1 scratch=1 tmp_enospc=1"
);
let a = format!("/proc/{}/root", one.init_pid());
let b = format!("/proc/{}/root", two.init_pid());
assert!(Path::new(&format!("{a}/work/isolated-sentinel")).exists());
assert!(!Path::new(&format!("{b}/work/isolated-sentinel")).exists());
assert!(
!Path::new("/var/lib/kf-judge/images/gnu-cxx-x86_64-v1/work/isolated-sentinel").exists()
);
for target in [&one, &two] {
assert_eq!(
target.budget().work_bytes + target.budget().tmp_bytes,
64 << 20
);
assert!(
fs::read_to_string(format!("/proc/{}/status", target.init_pid()))
.unwrap()
.contains("State:\tT (stopped)")
);
let mountinfo =
fs::read_to_string(format!("/proc/{}/mountinfo", target.init_pid())).unwrap();
assert_eq!(
mountinfo.lines().count(),
3,
"root/work/tmp only: {mountinfo}"
);
assert!(mountinfo.lines().any(|l| {
l.split_whitespace().nth(4) == Some("/")
&& l.split_whitespace()
.nth(5)
.unwrap()
.split(',')
.any(|f| f == "ro")
}));
assert!(!mountinfo.contains(" shared:") && !mountinfo.contains(" master:"));
for entry in fs::read_dir(format!("/proc/{}/fd", target.init_pid())).unwrap() {
assert!(!fs::metadata(entry.unwrap().path()).unwrap().is_dir());
}
assert_eq!(
fs::read_to_string(root.path.join(target.cgroup().name()).join("cgroup.freeze"))
.unwrap()
.trim(),
"1"
);
}
assert_eq!(fs::read_to_string("/proc/self/mountinfo").unwrap(), before);
let ids = (one.init_pid(), two.init_pid());
for (target, id) in [(one, "root-one"), (two, "root-two")] {
let bootstrap = target.bootstrap_pid();
let cgroup = root.path.join(target.cgroup().name());
let failure = target.refuse_remaining_isolation();
assert_eq!(failure.failure.stage, ExecutionStage::RootFilesystem);
assert!(failure.cleanup_failure.is_none());
assert!(!cgroup.exists());
assert!(!Path::new(&task_path(id)).exists());
gone(bootstrap);
}
gone(ids.0);
gone(ids.1);
println!(
"KF_ROOT_PROOF init_pids={ids:?} pivot_root=1 ro_image=1 work_tmp_independent=1 bounded_tmp=1 no_directory_fds=1 only_three_mounts=1 host_mounts_unchanged=1 refusal=1"
);
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn namespace_failure_rolls_back_the_exclusive_root_directory() {
let root = Root::new();
let before = children();
let result = RootedTarget::prepare(
&root.validated(),
&spec("root-ns-fail", 64 << 20, 1),
&binary(),
);
let Err(failure) = result else {
panic!("PID 1 creation must fail at pids.max=1");
};
assert_eq!(failure.failure.stage, ExecutionStage::Namespaces);
assert_eq!(failure.failure.errno, Some(11));
assert!(failure.cleanup_failure.is_none());
assert!(!Path::new(&task_path("root-ns-fail")).exists());
assert!(!root.path.join("kf-root-ns-fail").exists());
assert_eq!(children(), before);
println!(
"KF_ROOT_ROLLBACK_PROOF namespace_failure=11 root_directory_removed=1 bootstrap_reaped=1"
);
}
#[test]
#[ignore = "requires independent VM with externally verified whole-VM caps"]
fn unsafe_image_is_rejected_before_spawning_and_existing_tasks_are_not_adopted() {
let mut root = Root::new();
let before = children();
root.config.image_root = "/var/lib/kf-judge/images/unsafe".into();
let result = RootedTarget::prepare(
&root.validated(),
&spec("root-unsafe", 64 << 20, 8),
&binary(),
);
let Err(failure) = result else {
panic!("unsafe image must fail");
};
assert_eq!(failure.failure.stage, ExecutionStage::RootFilesystem);
assert!(failure.cleanup_failure.is_none());
assert_eq!(children(), before);
assert!(!Path::new(&task_path("root-unsafe")).exists());
root.config.image_root = "/var/lib/kf-judge/images/gnu-cxx-x86_64-v1".into();
let target = RootedTarget::prepare(
&root.validated(),
&spec("root-collision", 64 << 20, 8),
&binary(),
)
.unwrap();
assert!(
RootedTarget::prepare(
&root.validated(),
&spec("root-collision", 64 << 20, 8),
&binary()
)
.is_err()
);
assert!(Path::new(&format!("/proc/{}/root", target.init_pid())).exists());
drop(target);
assert!(!Path::new(&task_path("root-collision")).exists());
println!("KF_ROOT_POLICY_PROOF unsafe_image_rejected=1 exclusive_lease=1 drop_cleanup=1");
}
+1 -1
View File
@@ -37,7 +37,7 @@ pub fn require_vm() {
pub struct Root {
pub path: PathBuf,
config: RuntimeConfig,
pub config: RuntimeConfig,
}
impl Root {