feat(sandbox): bind readonly inputs through authenticated capabilities

This commit is contained in:
2026-10-07 17:57:10 +08:00
parent 9a2b1c071f
commit 329c9222a3
13 changed files with 895 additions and 38 deletions
+19 -1
View File
@@ -125,7 +125,7 @@ FD、栈和其他信号仍按原退出状态记录,缺少专门证据时不猜
双方用内核 `SO_PEERCRED` 核对预期 PID/UID/GID;父端 PID 来源于仍持有的 `Child`,
子端父 PID 来源于内核。内部入口不提供用户命令参数,握手消息不能转换成生命周期确认事件。
`launcher/protocol.rs` 使用带版本的 8 字节帧头和最多 1 KiB JSON。先拒绝未知版本、标志及
`launcher/protocol.rs` 使用带版本的 8 字节帧头和有界 JSON(普通消息最多 1 KiB,输入路径阶段最多 8 KiB)。先拒绝未知版本、标志及
非法长度,再读取固定大小缓冲;发送序列化也使用固定缓冲。父子消息类型分开,拒绝多余字段、
非法 run_id 和未知命令。所有 I/O 共用一个绝对截止时间(入口 5 秒,通道最大 10 秒),
部分帧不能续期;EOF、解析、超时或写入错误关闭并锁定通道,不能重新同步后继续初始化。
@@ -311,3 +311,21 @@ CPU 带宽和 CPU 总时间分别配置;预算目前只完成政策校验,
接口语义参见 [cgroup v2](https://docs.kernel.org/admin-guide/cgroup-v2.html)、
[pidfd_open](https://man7.org/linux/man-pages/man2/pidfd_open.2.html) 和
[seccomp filter](https://docs.kernel.org/userspace-api/seccomp_filter.html)。
## 只读输入初始化
`launcher/inputs.rs` 的 `InputBoundTarget` 在受限 proc/dev 就绪后继续固定初始化,
经已认证的控制 socket 逐个发送封印、只读、CLOEXEC 的文件 FD。接收端只接受预定的
消息次序、一个 FD、有限数量与总字节;多余/截断 ancillary 数据使通道永久失败。
stdin 只保留在监督器的独立快照中,不挂载到用户工作区,也尚未接入执行 stdio。
匿名 memfd 不能直接 bind 到任务 namespace。初始化器用固定 64 KiB 缓冲将快照
复制到有界 work tmpfs,再将该文件 self-bind 为 readonly/nosuid/nodev;数据另加 noexec,
仅规格指定的 native 主程序保留执行位。复制占用原工作区预算,ENOSPC 立即拒绝并回滚。
监督器逐字节核对封印来源与绑定内容、文件类型/权限/属主、挂载标志及 FD 关闭情况,
随后保持目标冻结。root 拥有的 sticky 工作区及输入上层目录允许旁边生成编译输出,
保护输入和其祖先不被未来任务身份替换。该固定阶段尚无用户执行入口。
`inputs_vm.rs` 在独立受限 VM 验收真实只读写入 EROFS、执行/noexec 区分、输入祖先
保护、stdin 不可见、不同任务隔离,以及链接/已有文件/空间不足导致的部分挂载回滚。
身份降权、capabilities/seccomp、用户进程树与 stdio 仍须完成后才能启用 `run`。
+25 -8
View File
@@ -17,6 +17,7 @@ pub const MAX_SNAPSHOT_BYTES: u64 = 64 * 1024 * 1024;
const SEALS: SealFlags = SealFlags::WRITE
.union(SealFlags::SHRINK)
.union(SealFlags::GROW)
.union(SealFlags::EXEC)
.union(SealFlags::SEAL);
const RESOLVE: ResolveFlags = ResolveFlags::BENEATH
.union(ResolveFlags::NO_SYMLINKS)
@@ -98,22 +99,28 @@ impl ArtifactRoot {
for input in &request.inputs {
// Validated IDs are a single component, so no untrusted ancestor
// directory can substitute a differently owned publication tree.
let file = self.snapshot_id(&input.artifact_id, &mut budget)?;
let executable = request.argv[0] == format!("/work/{}", input.path);
let file = self.snapshot_id(&input.artifact_id, &mut budget, executable)?;
files.push(PreparedInput {
path: input.path.clone(),
executable: request.argv[0] == format!("/work/{}", input.path),
executable,
file,
});
}
let stdin = request
.stdin_artifact
.as_ref()
.map(|id| self.snapshot_id(id, &mut budget))
.map(|id| self.snapshot_id(id, &mut budget, false))
.transpose()?;
Ok(PreparedInputs { files, stdin })
}
fn snapshot_id(&self, id: &str, budget: &mut SnapshotBudget) -> io::Result<SnapshotFile> {
fn snapshot_id(
&self,
id: &str,
budget: &mut SnapshotBudget,
executable: bool,
) -> io::Result<SnapshotFile> {
if id.is_empty()
|| id.len() > 64
|| !id
@@ -122,7 +129,7 @@ impl ArtifactRoot {
{
return Err(refused("invalid artifact identifier"));
}
snapshot(&self.fd, id, budget, Some(self.owner))
snapshot(&self.fd, id, budget, Some(self.owner), executable)
}
}
@@ -150,7 +157,7 @@ impl OutputPlan {
for path in &self.paths {
files.push(OutputSnapshot {
path: path.clone(),
file: snapshot(work, path, &mut budget, None)?,
file: snapshot(work, path, &mut budget, None, false)?,
});
}
// Errors drop every partial FD; there is no partially published batch.
@@ -221,6 +228,7 @@ fn snapshot(
relative: &str,
budget: &mut SnapshotBudget,
owner: Option<u32>,
executable: bool,
) -> io::Result<SnapshotFile> {
path(relative, "artifact").map_err(|_| refused("noncanonical artifact path"))?;
// Atomic containment; reject unsupported openat2 or races, no fallback or
@@ -243,7 +251,13 @@ fn snapshot(
}
let mut target = File::from(rustix::fs::memfd_create(
"kf-artifact",
MemfdFlags::CLOEXEC | MemfdFlags::ALLOW_SEALING,
MemfdFlags::CLOEXEC
| MemfdFlags::ALLOW_SEALING
| if executable {
MemfdFlags::EXEC
} else {
MemfdFlags::NOEXEC_SEAL
},
)?);
let mut remaining = bytes;
let mut buffer = [0_u8; 64 * 1024];
@@ -257,7 +271,10 @@ fn snapshot(
if source.read(&mut buffer[..1])? != 0 || !unchanged(&before, &rustix::fs::fstat(&source)?) {
return Err(refused("artifact changed during copy"));
}
rustix::fs::fchmod(&target, Mode::from_raw_mode(0o400))?;
rustix::fs::fchmod(
&target,
Mode::from_raw_mode(if executable { 0o555 } else { 0o444 }),
)?;
rustix::fs::fcntl_add_seals(&target, SEALS)?;
if !rustix::fs::fcntl_get_seals(&target)?.contains(SEALS) {
return Err(refused("artifact seals not applied"));
@@ -0,0 +1,199 @@
//! Fixed pre-exec read-only bindings. No task code exists during this stage.
use crate::spec::path;
use rustix::fd::OwnedFd;
use rustix::fs::{FileType, Mode, OFlags, ResolveFlags, SealFlags, StatVfsMountFlags};
use rustix::mount::MountFlags;
use std::io;
fn denied(message: &'static str) -> io::Error {
io::Error::new(io::ErrorKind::PermissionDenied, message)
}
const RESOLVE: ResolveFlags = ResolveFlags::BENEATH
.union(ResolveFlags::NO_SYMLINKS)
.union(ResolveFlags::NO_MAGICLINKS)
.union(ResolveFlags::NO_XDEV);
const DIR: OFlags = OFlags::RDONLY
.union(OFlags::DIRECTORY)
.union(OFlags::CLOEXEC);
fn directory(parent: &OwnedFd, name: &str) -> io::Result<OwnedFd> {
match rustix::fs::mkdirat(parent, name, Mode::from_raw_mode(0o1777)) {
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
Err(error) => return Err(error.into()),
}
let next = rustix::fs::openat2(parent, name, DIR, Mode::empty(), RESOLVE)?;
let stat = rustix::fs::fstat(&next)?;
if stat.st_uid != 0 || stat.st_gid != 0 {
return Err(denied("input parent owner"));
}
// Root-owned sticky directories protect input ancestors while permitting
// the future task identity to create compiler output beside input files.
rustix::fs::fchmod(&next, Mode::from_raw_mode(0o1777))?;
Ok(next)
}
pub(crate) fn bind(relative: &str, fd: &OwnedFd, bytes: u64, executable: bool) -> io::Result<()> {
super::super::launcher::entry::ensure_single_threaded()?;
if rustix::process::getpid().as_raw_pid() != 1 || !rustix::process::geteuid().is_root() {
return Err(denied("input binding requires privileged namespace init"));
}
path(relative, "input").map_err(|_| denied("input binding path"))?;
let source = rustix::fs::fstat(fd)?;
let seals =
SealFlags::WRITE | SealFlags::GROW | SealFlags::SHRINK | SealFlags::EXEC | SealFlags::SEAL;
if FileType::from_raw_mode(source.st_mode) != FileType::RegularFile
|| source.st_nlink != 0
|| u64::try_from(source.st_size).ok() != Some(bytes)
|| source.st_mode & 0o7777 != if executable { 0o555 } else { 0o444 }
|| !rustix::fs::fcntl_get_seals(fd)?.contains(seals)
|| !rustix::io::fcntl_getfd(fd)?.contains(rustix::io::FdFlags::CLOEXEC)
|| rustix::fs::fcntl_getfl(fd)? & OFlags::ACCMODE != OFlags::RDONLY
{
return Err(denied("unsealed or mismatched input descriptor"));
}
let mut parent = rustix::fs::open("/work", DIR | OFlags::NOFOLLOW, Mode::empty())?;
for component in relative
.split('/')
.collect::<Vec<_>>()
.split_last()
.ok_or_else(|| denied("empty input path"))?
.1
{
parent = directory(&parent, component)?;
}
let name = relative
.rsplit('/')
.next()
.ok_or_else(|| denied("input basename"))?;
let placeholder = rustix::fs::openat2(
&parent,
name,
OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC,
Mode::from_raw_mode(0o600),
RESOLVE,
)?;
// Anonymous memfd mounts are internal to the kernel and cannot be bound
// into this mount namespace (EINVAL). Copy the sealed source into the
// already-bounded work tmpfs before making a read-only self-bind. No task
// code runs and no writable handle survives the acknowledgement.
let mut buffer = [0_u8; 65536];
let mut offset = 0_u64;
while offset < bytes {
let length = usize::try_from((bytes - offset).min(buffer.len() as u64)).unwrap();
let count = rustix::io::pread(fd, &mut buffer[..length], offset)?;
if count == 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"sealed input EOF",
));
}
let mut written = 0;
while written < count {
let n = rustix::io::pwrite(
&placeholder,
&buffer[written..count],
offset + written as u64,
)?;
if n == 0 {
return Err(io::Error::new(io::ErrorKind::WriteZero, "input copy"));
}
written += n;
}
offset += count as u64;
}
rustix::fs::fchmod(
&placeholder,
Mode::from_raw_mode(if executable { 0o555 } else { 0o444 }),
)?;
drop(placeholder);
// Target components were opened without links and are held in root-owned
// directories. Only the single-threaded trusted helper/bootstrap exist;
// the parent verified task membership before releasing this fixed stage.
let target = format!("/work/{relative}");
rustix::mount::mount_bind(&target, &target)?;
let flags = MountFlags::BIND
| MountFlags::RDONLY
| MountFlags::NOSUID
| MountFlags::NODEV
| if executable {
MountFlags::empty()
} else {
MountFlags::NOEXEC
};
rustix::mount::mount_remount(&target, flags, c"")?;
Ok(())
}
pub(crate) fn finish() -> io::Result<()> {
let work = rustix::fs::open("/work", DIR | OFlags::NOFOLLOW, Mode::empty())?;
let stat = rustix::fs::fstat(&work)?;
if stat.st_uid != 0 || stat.st_gid != 0 {
return Err(denied("work owner"));
}
rustix::fs::fchmod(&work, Mode::from_raw_mode(0o1777))?;
Ok(())
}
pub(crate) fn verify(pid: u32, inputs: &[crate::artifacts::PreparedInput]) -> io::Result<()> {
let work = rustix::fs::open(format!("/proc/{pid}/root/work"), DIR, Mode::empty())?;
let stat = rustix::fs::fstat(&work)?;
if stat.st_uid != 0 || stat.st_gid != 0 || stat.st_mode & 0o7777 != 0o1777 {
return Err(denied("work mode"));
}
for input in inputs {
let target = rustix::fs::openat2(
&work,
input.path.as_str(),
OFlags::RDONLY | OFlags::CLOEXEC,
Mode::empty(),
RESOLVE & !ResolveFlags::NO_XDEV,
)?;
let actual = rustix::fs::fstat(&target)?;
let expected = rustix::fs::fstat(&input.file)?;
let flags = rustix::fs::fstatvfs(&target)?.f_flag;
if FileType::from_raw_mode(actual.st_mode) != FileType::RegularFile
|| actual.st_nlink != 1
|| actual.st_size != expected.st_size
|| actual.st_mode != expected.st_mode
|| actual.st_uid != expected.st_uid
|| actual.st_gid != expected.st_gid
|| !flags.contains(
StatVfsMountFlags::RDONLY | StatVfsMountFlags::NOSUID | StatVfsMountFlags::NODEV,
)
|| flags.contains(StatVfsMountFlags::NOEXEC) == input.executable
{
return Err(denied("input binding readback"));
}
// Compare fixed-sized chunks through the retained sealed source FD.
// Neither a hash collision nor a pathname re-open can validate input.
let mut source = [0_u8; 65536];
let mut bound = [0_u8; 65536];
let mut offset = 0_u64;
while offset < input.file.bytes() {
let length =
usize::try_from((input.file.bytes() - offset).min(source.len() as u64)).unwrap();
let a = rustix::io::pread(&input.file, &mut source[..length], offset)?;
let b = rustix::io::pread(&target, &mut bound[..length], offset)?;
if a == 0 || a != b || source[..a] != bound[..b] {
return Err(denied("input content readback"));
}
offset += a as u64;
}
}
// No input/directory capability is retained by PID 1 after acknowledgement.
let mut fds = std::fs::read_dir(format!("/proc/{pid}/fd"))?
.map(|entry| {
entry.and_then(|e| {
e.file_name()
.to_str()
.and_then(|s| s.parse::<i32>().ok())
.ok_or_else(|| denied("fd name"))
})
})
.collect::<io::Result<Vec<_>>>()?;
fds.sort_unstable();
if fds != [0, 1, 2, 3] {
return Err(denied("input descriptor leaked"));
}
Ok(())
}
@@ -1,5 +1,6 @@
//! Fixed Linux isolation operations; launcher owns process/control orchestration.
pub(crate) mod inputs;
pub mod namespaces;
pub(crate) mod proc_dev;
pub(crate) mod root;
+125
View File
@@ -0,0 +1,125 @@
//! Authenticated descriptor transfer and fixed pre-exec input initialization.
use super::blocked::{InitializationFailure, failure};
use super::proc_dev::RestrictedRootTarget;
use super::protocol::{ControlChannel, NamespaceCommand, NamespaceEvent};
use crate::artifacts::{ArtifactRoot, SnapshotFile};
use crate::outcome::{ErrorCode, ExecutionStage};
use crate::runtime::ValidatedRuntimeConfig;
use crate::spec::ValidatedSpec;
use std::{io, path::Path};
pub struct InputBoundTarget {
pub(crate) target: RestrictedRootTarget,
stdin: Option<SnapshotFile>,
}
impl InputBoundTarget {
pub fn prepare(
config: &ValidatedRuntimeConfig,
spec: &ValidatedSpec,
executable: &Path,
artifacts: &ArtifactRoot,
) -> Result<Self, InitializationFailure> {
// Resolve/copy before starting the bounded helper handshake.
let inputs = artifacts
.snapshot_inputs(spec)
.map_err(|error| InitializationFailure {
failure: failure(
ExecutionStage::Preparation,
ErrorCode::OperationFailed,
Some(&error),
),
cleanup_failure: None,
})?;
let target = RestrictedRootTarget::prepare(config, spec, executable)?;
Self::bind_prepared(target, inputs)
}
/// Resume an existing verified restricted root. Artifacts are always
/// resolved against the immutable request saved with that same target.
pub fn bind(
target: RestrictedRootTarget,
artifacts: &ArtifactRoot,
) -> Result<Self, InitializationFailure> {
let inputs = match artifacts.snapshot_inputs(&target.root.request) {
Ok(inputs) => inputs,
Err(error) => return Err(target.root.refuse_root_error(&error)),
};
Self::bind_prepared(target, inputs)
}
fn bind_prepared(
mut target: RestrictedRootTarget,
inputs: crate::artifacts::PreparedInputs,
) -> Result<Self, InitializationFailure> {
if let Err(error) = target.root.initialize_inputs(&inputs) {
return Err(target.root.refuse_root_error(&error));
}
// Only the independent stdin handle remains in the supervisor. Work
// input FDs are dropped after verifying the actual bound content and mounts.
Ok(Self {
target,
stdin: inputs.stdin,
})
}
pub fn root(&self) -> &super::root::RootedTarget {
self.target.root()
}
pub fn stdin(&self) -> Option<&SnapshotFile> {
self.stdin.as_ref()
}
pub fn refuse_remaining_isolation(self) -> InitializationFailure {
self.target.refuse_remaining_isolation()
}
}
pub(crate) fn initialize(channel: &mut ControlChannel, count: u32, bytes: u64) -> io::Result<()> {
let setup = (|| {
channel.send(&NamespaceEvent::InputsStarted {})?;
let mut total = 0_u64;
let mut paths = std::collections::BTreeSet::new();
for _ in 0..count {
let NamespaceCommand::AttachInput {
path,
bytes: size,
executable,
} = channel.receive()?
else {
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"input stage order",
));
};
total = total
.checked_add(size)
.filter(|total| *total <= bytes)
.ok_or_else(|| io::Error::new(io::ErrorKind::PermissionDenied, "input total"))?;
if !paths.insert(path.clone()) {
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"duplicate input",
));
}
let fd = channel.receive_fd()?;
crate::isolation::inputs::bind(&path, &fd, size, executable)?;
drop(fd);
channel.send(&NamespaceEvent::InputAttached {})?;
}
if total != bytes
|| channel.receive::<NamespaceCommand>()? != (NamespaceCommand::FinishInputs {})
{
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"unfinished input batch",
));
}
crate::isolation::inputs::finish()?;
channel.send(&NamespaceEvent::InputsInitialized {})
})();
if let Err(error) = &setup {
let _ = channel.send(&NamespaceEvent::Failed {
errno: error.raw_os_error().and_then(|n| u16::try_from(n).ok()),
});
}
setup
}
+2
View File
@@ -12,6 +12,8 @@ pub mod blocked;
#[cfg(target_os = "linux")]
pub mod entry;
#[cfg(target_os = "linux")]
pub mod inputs;
#[cfg(target_os = "linux")]
pub mod namespaces;
#[cfg(target_os = "linux")]
pub mod proc_dev;
@@ -54,7 +54,7 @@ pub fn bootstrap(expected_parent: u32, initialize_root: bool) -> io::Result<()>
return Ok(());
}
NamespaceCommand::Initialize {} => {}
NamespaceCommand::PrepareRoot { .. } | NamespaceCommand::PrepareProcDev {} => {
_ => {
return Err(denied("root command before namespaces"));
}
}
@@ -182,6 +182,13 @@ pub fn namespace_init(expected_outer_parent: u32, initialize_root: bool) -> io::
return Err(error);
}
channel.send(&NamespaceEvent::ProcDevInitialized {})?;
match channel.receive::<NamespaceCommand>()? {
NamespaceCommand::PrepareInputs { count, bytes } => {
super::inputs::initialize(&mut channel, count, bytes)?;
}
NamespaceCommand::Cancel {} => return Ok(()),
_ => return Err(denied("input command before preparation")),
}
let _ = channel.receive::<NamespaceCommand>()?;
}
NamespaceCommand::Cancel {} => return Ok(()),
@@ -203,6 +210,7 @@ pub struct NamespacedTarget {
allow_root: bool,
root_ready: bool,
proc_dev_ready: bool,
inputs_ready: bool,
init_pid: u32,
ids: NamespaceIds,
}
@@ -295,6 +303,7 @@ impl NamespacedTarget {
allow_root,
root_ready: false,
proc_dev_ready: false,
inputs_ready: false,
}),
Err(error) => Err(prepared.refuse_at(ExecutionStage::Namespaces, Some(&error))),
}
@@ -330,6 +339,55 @@ impl NamespacedTarget {
Ok(())
}
pub(crate) fn initialize_inputs(
&mut self,
inputs: &[crate::artifacts::PreparedInput],
) -> io::Result<()> {
if !self.proc_dev_ready || self.inputs_ready || inputs.len() > 64 {
return Err(denied("input initialization order"));
}
let bytes = inputs
.iter()
.try_fold(0_u64, |total, input| {
total
.checked_add(input.file.bytes())
.filter(|n| *n <= crate::artifacts::MAX_SNAPSHOT_BYTES)
})
.ok_or_else(|| denied("input bytes"))?;
self.control.send(&NamespaceCommand::PrepareInputs {
count: inputs.len() as u32,
bytes,
})?;
self.prepared
.cgroup
.begin_root_initialization(&[self.prepared.pid(), self.init_pid], &self.init_pidfd)?;
self.expect_event(NamespaceEvent::InputsStarted {})?;
for input in inputs {
self.control.send(&NamespaceCommand::AttachInput {
path: input.path.clone(),
bytes: input.file.bytes(),
executable: input.executable,
})?;
self.control.send_fd(&input.file)?;
self.expect_event(NamespaceEvent::InputAttached {})?;
}
self.control.send(&NamespaceCommand::FinishInputs {})?;
self.expect_event(NamespaceEvent::InputsInitialized {})?;
self.stop_fixed_stage()?;
self.inputs_ready = true;
Ok(())
}
fn expect_event(&mut self, expected: NamespaceEvent) -> io::Result<()> {
match self.control.receive::<NamespaceEvent>()? {
event if event == expected => Ok(()),
NamespaceEvent::Failed { errno: Some(errno) } => {
Err(io::Error::from_raw_os_error(i32::from(errno)))
}
_ => Err(denied("fixed initialization acknowledgement")),
}
}
fn initialize_filesystem(
&mut self,
command: NamespaceCommand,
@@ -339,13 +397,11 @@ impl NamespacedTarget {
self.prepared
.cgroup
.begin_root_initialization(&[self.prepared.pid(), self.init_pid], &self.init_pidfd)?;
match self.control.receive::<NamespaceEvent>()? {
event if event == expected => {}
NamespaceEvent::Failed { errno: Some(errno) } => {
return Err(io::Error::from_raw_os_error(i32::from(errno)));
}
_ => return Err(denied("root initialization failed")),
}
self.expect_event(expected)?;
self.stop_fixed_stage()
}
fn stop_fixed_stage(&mut self) -> io::Result<()> {
rustix::process::pidfd_send_signal(&self.init_pidfd, Signal::STOP)?;
let deadline = Instant::now() + Duration::from_secs(2);
loop {
@@ -6,7 +6,7 @@ use crate::spec::ValidatedSpec;
use std::path::Path;
pub struct RestrictedRootTarget {
root: RootedTarget,
pub(crate) root: RootedTarget,
}
impl RestrictedRootTarget {
+194 -19
View File
@@ -2,13 +2,21 @@
//! Messages cannot produce Lifecycle events or prove completed OS operations.
use crate::outcome::{ErrorCode, ExecutionStage};
use rustix::fd::{AsFd, OwnedFd};
use rustix::net::{
RecvAncillaryBuffer, RecvAncillaryMessage, RecvFlags, ReturnFlags, SendAncillaryBuffer,
SendAncillaryMessage, SendFlags,
};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use std::io::{self, Read, Write};
use std::io::{self, IoSlice, IoSliceMut, Write};
use std::mem::MaybeUninit;
use std::net::Shutdown;
use std::os::unix::net::UnixStream;
use std::time::{Duration, Instant};
pub const MAX_MESSAGE_BYTES: usize = 1024;
const MAX_NAMESPACE_MESSAGE_BYTES: usize = 8192;
const FD_MARKER: u8 = 0xa5;
pub const MAX_CONTROL_TIMEOUT: Duration = Duration::from_secs(10);
const PREFIX: [u8; 4] = [b'K', b'F', 1, 0];
@@ -40,8 +48,21 @@ pub enum ChildMessage {
#[serde(tag = "command", rename_all = "snake_case", deny_unknown_fields)]
pub enum NamespaceCommand {
Initialize {},
PrepareRoot { work_bytes: u64, tmp_bytes: u64 },
PrepareRoot {
work_bytes: u64,
tmp_bytes: u64,
},
PrepareProcDev {},
PrepareInputs {
count: u32,
bytes: u64,
},
AttachInput {
path: String,
bytes: u64,
executable: bool,
},
FinishInputs {},
Cancel {},
}
@@ -51,6 +72,9 @@ pub enum NamespaceEvent {
Initialized { init_pid: u32 },
RootInitialized {},
ProcDevInitialized {},
InputsStarted {},
InputAttached {},
InputsInitialized {},
Failed { errno: Option<u16> },
Cancelled {},
}
@@ -61,12 +85,14 @@ mod sealed {
/// Sealed to keep the control protocol separate from arbitrary execution data.
pub trait WireMessage: sealed::Sealed + Serialize + DeserializeOwned {
const MAX_BYTES: usize = MAX_MESSAGE_BYTES;
fn valid(&self) -> bool;
}
impl sealed::Sealed for ParentMessage {}
impl sealed::Sealed for NamespaceCommand {}
impl WireMessage for NamespaceCommand {
const MAX_BYTES: usize = MAX_NAMESPACE_MESSAGE_BYTES;
fn valid(&self) -> bool {
match self {
Self::PrepareRoot {
@@ -77,6 +103,13 @@ impl WireMessage for NamespaceCommand {
tmp_bytes: *tmp_bytes,
}
.valid(),
Self::PrepareInputs { count, bytes } => {
*count <= 64 && *bytes <= crate::artifacts::MAX_SNAPSHOT_BYTES
}
Self::AttachInput { path, bytes, .. } => {
crate::spec::path(path, "input").is_ok()
&& *bytes <= crate::artifacts::MAX_SNAPSHOT_BYTES
}
_ => true,
}
}
@@ -185,12 +218,55 @@ impl ControlChannel {
self.finish(result)
}
/// Exactly one descriptor and one marker, immediately after AttachInput.
/// SO_PEERCRED was authenticated before any capability can be received.
pub(crate) fn send_fd(&mut self, fd: impl AsFd) -> io::Result<()> {
let result = (|| {
let mut space = [MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(1))];
let borrowed = [fd.as_fd()];
let mut ancillary = SendAncillaryBuffer::new(&mut space);
if !ancillary.push(SendAncillaryMessage::ScmRights(&borrowed)) {
return Err(invalid("descriptor control buffer"));
}
loop {
self.stream.set_write_timeout(Some(self.remaining()?))?;
match rustix::net::sendmsg(
&self.stream,
&[IoSlice::new(&[FD_MARKER])],
&mut ancillary,
SendFlags::NOSIGNAL,
) {
Ok(1) => return Ok(()),
Err(error) if error == rustix::io::Errno::INTR => continue,
Err(error) => return Err(error.into()),
_ => return Err(invalid("descriptor marker write")),
}
}
})();
self.finish(result)
}
pub(crate) fn receive_fd(&mut self) -> io::Result<OwnedFd> {
let result = (|| {
let mut marker = [0_u8; 1];
let (bytes, fd) = self.read_message(&mut marker, true)?;
if bytes != 1 || marker[0] != FD_MARKER {
return Err(invalid("descriptor marker read"));
}
fd.ok_or_else(|| invalid("missing input descriptor"))
})();
self.finish(result)
}
fn send_inner<T: WireMessage>(&mut self, message: &T) -> io::Result<()> {
self.remaining()?;
if !message.valid() {
return Err(invalid("invalid control message"));
}
let mut body = FixedBuffer::default();
let mut body = FixedBuffer {
limit: T::MAX_BYTES,
..FixedBuffer::default()
};
serde_json::to_writer(&mut body, message)
.map_err(|_| invalid("control message too large"))?;
let mut header = [0u8; 8];
@@ -203,8 +279,8 @@ impl ControlChannel {
fn receive_inner<T: WireMessage>(&mut self) -> io::Result<T> {
let mut header = [0u8; 8];
self.read_bytes(&mut header)?;
let length = decode_header(header)?;
let mut bytes = [0u8; MAX_MESSAGE_BYTES];
let length = decode_header(header, T::MAX_BYTES)?;
let mut bytes = [0u8; MAX_NAMESPACE_MESSAGE_BYTES];
self.read_bytes(&mut bytes[..length])?;
let message: T = serde_json::from_slice(&bytes[..length])
.map_err(|_| invalid("invalid control JSON"))?;
@@ -229,17 +305,57 @@ impl ControlChannel {
fn read_bytes(&mut self, mut bytes: &mut [u8]) -> io::Result<()> {
while !bytes.is_empty() {
self.stream.set_read_timeout(Some(self.remaining()?))?;
match self.stream.read(bytes) {
Ok(0) => return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "control EOF")),
Ok(count) => bytes = &mut bytes[count..],
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
Err(error) => return Err(error),
}
let (count, _) = self.read_message(bytes, false)?;
bytes = &mut bytes[count..];
}
Ok(())
}
fn read_message(
&mut self,
bytes: &mut [u8],
allow_fd: bool,
) -> io::Result<(usize, Option<OwnedFd>)> {
loop {
self.stream.set_read_timeout(Some(self.remaining()?))?;
let mut space = [MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(2))];
let mut ancillary = RecvAncillaryBuffer::new(&mut space);
let received = match rustix::net::recvmsg(
&self.stream,
&mut [IoSliceMut::new(bytes)],
&mut ancillary,
RecvFlags::CMSG_CLOEXEC,
) {
Err(error) if error == rustix::io::Errno::INTR => continue,
other => other?,
};
let mut fd = None;
let mut invalid_control = received.flags.contains(ReturnFlags::CTRUNC);
for message in ancillary.drain() {
match message {
RecvAncillaryMessage::ScmRights(fds) => {
for next in fds {
if !allow_fd || fd.is_some() {
invalid_control = true;
}
if fd.is_none() {
fd = Some(next);
}
}
}
_ => invalid_control = true,
}
}
if invalid_control {
return Err(invalid("unexpected or truncated descriptors"));
}
if received.bytes == 0 {
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "control EOF"));
}
return Ok((received.bytes, fd));
}
}
fn write_bytes(&mut self, mut bytes: &[u8]) -> io::Result<()> {
while !bytes.is_empty() {
self.stream.set_write_timeout(Some(self.remaining()?))?;
@@ -271,35 +387,37 @@ fn invalid(message: &'static str) -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, message)
}
fn decode_header(header: [u8; 8]) -> io::Result<usize> {
fn decode_header(header: [u8; 8], limit: usize) -> io::Result<usize> {
if header[..4] != PREFIX {
return Err(invalid("unsupported control framing"));
}
let length =
u32::from_be_bytes(header[4..].try_into().expect("four-byte header length")) as usize;
if length == 0 || length > MAX_MESSAGE_BYTES {
if length == 0 || length > limit {
return Err(invalid("invalid control frame length"));
}
Ok(length)
}
struct FixedBuffer {
bytes: [u8; MAX_MESSAGE_BYTES],
bytes: [u8; MAX_NAMESPACE_MESSAGE_BYTES],
len: usize,
limit: usize,
}
impl Default for FixedBuffer {
fn default() -> Self {
Self {
bytes: [0; MAX_MESSAGE_BYTES],
bytes: [0; MAX_NAMESPACE_MESSAGE_BYTES],
len: 0,
limit: MAX_MESSAGE_BYTES,
}
}
}
impl Write for FixedBuffer {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
if bytes.len() > MAX_MESSAGE_BYTES - self.len {
if bytes.len() > self.limit - self.len {
return Err(invalid("control message too large"));
}
self.bytes[self.len..self.len + bytes.len()].copy_from_slice(bytes);
@@ -315,13 +433,70 @@ impl Write for FixedBuffer {
mod tests {
use super::*;
fn pair() -> (ControlChannel, ControlChannel) {
let (a, b) = UnixStream::pair().unwrap();
let peer = ExpectedPeer {
pid: std::process::id(),
uid: rustix::process::geteuid().as_raw(),
gid: rustix::process::getegid().as_raw(),
};
(
ControlChannel::new(a, peer, Instant::now() + MAX_CONTROL_TIMEOUT).unwrap(),
ControlChannel::new(b, peer, Instant::now() + MAX_CONTROL_TIMEOUT).unwrap(),
)
}
#[test]
fn descriptor_transport_preserves_identity_and_cloexec() {
let (mut a, mut b) = pair();
let file = std::fs::File::open("/dev/null").unwrap();
a.send_fd(&file).unwrap();
let received = b.receive_fd().unwrap();
let x = rustix::fs::fstat(&file).unwrap();
let y = rustix::fs::fstat(&received).unwrap();
assert_eq!((x.st_dev, x.st_ino), (y.st_dev, y.st_ino));
assert!(
rustix::io::fcntl_getfd(&received)
.unwrap()
.contains(rustix::io::FdFlags::CLOEXEC)
);
}
#[test]
fn descriptor_in_a_json_frame_poison_connection() {
let (mut a, mut b) = pair();
let file = std::fs::File::open("/dev/null").unwrap();
a.send_fd(&file).unwrap();
assert!(b.receive::<NamespaceCommand>().is_err());
assert!(b.receive_fd().is_err());
assert!(a.send(&NamespaceCommand::Cancel {}).is_err());
}
#[test]
fn namespace_paths_fit_without_expanding_ordinary_control_frames() {
let (mut a, mut b) = pair();
let path = (0..10)
.map(|_| "x".repeat(200))
.collect::<Vec<_>>()
.join("/");
let command = NamespaceCommand::AttachInput {
path,
bytes: 0,
executable: false,
};
a.send(&command).unwrap();
assert_eq!(b.receive::<NamespaceCommand>().unwrap(), command);
assert_eq!(ParentMessage::MAX_BYTES, 1024);
assert_eq!(NamespaceCommand::MAX_BYTES, 8192);
}
#[test]
fn header_rejects_oversized_length_before_body_read() {
for length in [0, 1025, u32::MAX] {
let mut header = [0u8; 8];
header[..4].copy_from_slice(&PREFIX);
header[4..].copy_from_slice(&length.to_be_bytes());
assert!(decode_header(header).is_err());
assert!(decode_header(header, MAX_MESSAGE_BYTES).is_err());
}
}
@@ -331,7 +506,7 @@ mod tests {
let mut header = [0u8; 8];
header[..4].copy_from_slice(&prefix);
header[4..].copy_from_slice(&1u32.to_be_bytes());
assert!(decode_header(header).is_err());
assert!(decode_header(header, MAX_MESSAGE_BYTES).is_err());
}
}
+25
View File
@@ -15,6 +15,7 @@ pub struct RootedTarget {
target: NamespacedTarget,
lease: RootLease,
outputs: crate::artifacts::OutputPlan,
pub(crate) request: ValidatedSpec,
}
fn attach_cleanup(error: &mut InitializationFailure, cleanup: Option<io::Error>) {
@@ -86,6 +87,7 @@ impl RootedTarget {
target,
lease,
outputs: crate::artifacts::OutputPlan::for_spec(spec),
request: spec.clone(),
})
}
pub fn init_pid(&self) -> u32 {
@@ -123,6 +125,29 @@ impl RootedTarget {
crate::isolation::proc_dev::verify(self.init_pid())
}
pub(crate) fn initialize_inputs(
&mut self,
inputs: &crate::artifacts::PreparedInputs,
) -> io::Result<()> {
let expected = &self.request.spec().inputs;
if inputs.files.len() != expected.len()
|| inputs.files.iter().zip(expected).any(|(actual, expected)| {
actual.path != expected.path
|| actual.executable
!= (self.request.spec().argv[0] == format!("/work/{}", expected.path))
})
{
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"input binding plan mismatch",
));
}
self.target.initialize_inputs(&inputs.files)?;
self.lease.verify_root(self.init_pid())?;
self.lease.verify_root(self.bootstrap_pid())?;
crate::isolation::inputs::verify(self.init_pid(), &inputs.files)
}
pub(crate) fn refuse_root_error(mut self, error: &io::Error) -> InitializationFailure {
let mut failure = self
.target
+1 -1
View File
@@ -139,7 +139,7 @@ impl SpecPolicy {
}
/// An immutable validated request, not proof that the environment is safe.
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct ValidatedSpec(ExecutionSpec);
impl ValidatedSpec {
+34
View File
@@ -0,0 +1,34 @@
#define _GNU_SOURCE
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <grp.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
static int absent(const char *p) { int fd=open(p,O_RDONLY|O_CLOEXEC); if(fd>=0){close(fd);return 0;} return errno==ENOENT; }
int main(int argc,char **argv) {
if(argc!=2)return 2;
char marker[64]={0};FILE *f=fopen("/kf/INDEPENDENT_VM","r");
if(!f||!fgets(marker,sizeof marker,f)||strcmp(marker,"isolated-test-guest\n"))return 3;
fclose(f);
char *end=NULL;long pid=strtol(argv[1],&end,10);if(!end||*end||pid<=1)return 4;
char ns[64],root[64];snprintf(ns,sizeof ns,"/proc/%ld/ns/mnt",pid);snprintf(root,sizeof root,"/proc/%ld/root",pid);
int n=open(ns,O_RDONLY|O_CLOEXEC),d=open(root,O_RDONLY|O_DIRECTORY|O_CLOEXEC);
if(n<0||d<0||setns(n,CLONE_NEWNS)||fchdir(d)||chroot(".")||chdir("/"))return 5;
close(n);close(d);
if(setgroups(0,NULL)||setresgid(100000,100000,100000)||setresuid(100000,100000,100000))return 6;
char bytes[32]={0};int fd=open("/work/source/input",O_RDONLY|O_CLOEXEC);if(fd<0||read(fd,bytes,sizeof bytes)!=15||memcmp(bytes,"protected-input",15))return 7;close(fd);
fd=open("/work/source/input",O_WRONLY|O_TRUNC|O_CLOEXEC);if(fd>=0||(errno!=EROFS&&errno!=EACCES))return 8;
if(!rename("/work/source/input","/work/source/stolen"))return 9;
if(!rename("/work/source","/work/stolen-dir"))return 10;
struct stat s;struct statvfs v;
if(stat("/work/main",&s)||(s.st_mode&07777)!=0555||statvfs("/work/main",&v)||(v.f_flag&ST_NOEXEC))return 11;
if(statvfs("/work/source/input",&v)||!(v.f_flag&ST_RDONLY)||!(v.f_flag&ST_NOEXEC))return 12;
if(!absent("/work/input-001")||!absent("/image-outside-sentinel")||!absent("/kf/INDEPENDENT_VM"))return 13;
fd=open("/work/source/result",O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC,0600);if(fd<0||write(fd,"output",6)!=6)return 14;close(fd);
puts("KF_INPUT_BIND_PROBE data_ro=1 program_exec=1 data_noexec=1 ancestor_sticky=1 sibling_output=1 stdin_hidden=1 outside_absent=1");return 0;
}
+205
View File
@@ -0,0 +1,205 @@
#![cfg(target_os = "linux")]
mod support;
use kf_sandbox::artifacts::ArtifactRoot;
use kf_sandbox::launcher::inputs::InputBoundTarget;
use kf_sandbox::launcher::proc_dev::RestrictedRootTarget;
use kf_sandbox::outcome::ExecutionStage;
use kf_sandbox::spec::{InputArtifact, SpecPolicy, ValidatedSpec};
use std::os::unix::fs::{PermissionsExt, symlink};
use std::{
fs,
path::{Path, PathBuf},
process::Command,
};
use support::{Root, binary, children};
fn spec(id: &str) -> ValidatedSpec {
let mut request = support::spec(id, 64 << 20, 8).spec().clone();
request.inputs.push(InputArtifact {
artifact_id: "data-001".into(),
path: "source/input".into(),
});
request.validate(&SpecPolicy::default()).unwrap()
}
struct Publications(PathBuf);
impl Publications {
fn new() -> Self {
support::require_vm();
let path = PathBuf::from(format!("/kf/input-publications-{}", std::process::id()));
fs::create_dir(&path).unwrap();
fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap();
for (id, value) in [
("program-001", b"program-payload".as_slice()),
("data-001", b"protected-input".as_slice()),
("input-001", b"stdin-private".as_slice()),
] {
fs::write(path.join(id), value).unwrap();
}
Self(path)
}
fn capability(&self) -> ArtifactRoot {
ArtifactRoot::from_directory(
rustix::fs::open(
&self.0,
rustix::fs::OFlags::PATH
| rustix::fs::OFlags::DIRECTORY
| rustix::fs::OFlags::CLOEXEC,
rustix::fs::Mode::empty(),
)
.unwrap(),
)
.unwrap()
}
}
impl Drop for Publications {
fn drop(&mut self) {
fs::remove_dir_all(&self.0).unwrap();
}
}
#[test]
#[ignore = "requires independent capped VM and fixed read-only binding probe"]
fn inputs_are_bound_readonly_with_separate_stdin_and_protected_ancestors() {
let root = Root::new();
let publications = Publications::new();
let artifacts = publications.capability();
let before = children();
let mounts = fs::read_to_string("/proc/self/mountinfo").unwrap();
let one = InputBoundTarget::prepare(
&root.validated(),
&spec("input-bind-one"),
&binary(),
&artifacts,
)
.unwrap();
let two = InputBoundTarget::prepare(
&root.validated(),
&spec("input-bind-two"),
&binary(),
&artifacts,
)
.unwrap();
let a = format!("/proc/{}/root", one.root().init_pid());
let b = format!("/proc/{}/root", two.root().init_pid());
fs::write(format!("{b}/work/peer-only"), b"peer").unwrap();
assert!(!Path::new(&format!("{a}/work/peer-only")).exists());
assert_eq!(
fs::read(format!("{a}/work/source/input")).unwrap(),
b"protected-input"
);
fs::write(publications.0.join("data-001"), b"later source mutation").unwrap();
assert_eq!(
fs::write(format!("{a}/work/source/input"), b"tamper")
.unwrap_err()
.raw_os_error(),
Some(30)
);
assert_eq!(
fs::read(format!("{a}/work/source/input")).unwrap(),
b"protected-input"
);
let mut stdin = Vec::new();
std::io::Read::read_to_end(&mut one.stdin().unwrap().reader().unwrap(), &mut stdin).unwrap();
assert_eq!(stdin, b"stdin-private");
assert!(!Path::new(&format!("{a}/work/input-001")).exists());
let output = Command::new("/kf/bin/input-bind-probe")
.arg(one.root().init_pid().to_string())
.env_clear()
.output()
.unwrap();
assert!(
output.status.success(),
"{:?} {:?}",
output.stdout,
output.stderr
);
assert_eq!(
String::from_utf8(output.stdout).unwrap().trim(),
"KF_INPUT_BIND_PROBE data_ro=1 program_exec=1 data_noexec=1 ancestor_sticky=1 sibling_output=1 stdin_hidden=1 outside_absent=1"
);
assert_eq!(fs::read_to_string("/proc/self/mountinfo").unwrap(), mounts);
for (target, id) in [(one, "input-bind-one"), (two, "input-bind-two")] {
let info =
fs::read_to_string(format!("/proc/{}/mountinfo", target.root().init_pid())).unwrap();
assert_eq!(
info.lines().count(),
7,
"five base mounts and two input files"
);
let error = target.refuse_remaining_isolation();
assert!(error.cleanup_failure.is_none());
assert!(!Path::new(&format!("/var/lib/kf-judge/tasks/kf-{id}")).exists());
}
assert_eq!(children(), before);
println!(
"KF_INPUT_BIND_PROOF immutable_bindings=1 exec_role_sealed=1 data_noexec=1 separate_stdin=1 ancestors_protected=1 sibling_outputs=1 peer_isolated=1 input_fds_closed=1 host_mounts_unchanged=1 task_cleanup=1 refusal=1"
);
}
#[test]
#[ignore = "requires independent capped VM"]
fn input_destination_links_and_existing_files_fail_closed_and_roll_back() {
let root = Root::new();
let publications = Publications::new();
let artifacts = publications.capability();
let before = children();
let mounts = fs::read_to_string("/proc/self/mountinfo").unwrap();
for (id, symlink_parent) in [("input-parent-link", true), ("input-existing", false)] {
let target =
RestrictedRootTarget::prepare(&root.validated(), &spec(id), &binary()).unwrap();
let work = PathBuf::from(format!("/proc/{}/root/work", target.root().init_pid()));
if symlink_parent {
symlink("/tmp", work.join("source")).unwrap();
} else {
fs::write(work.join("main"), b"must not replace").unwrap();
}
let Err(error) = InputBoundTarget::bind(target, &artifacts) else {
panic!("unsafe destination accepted");
};
assert_eq!(error.failure.stage, ExecutionStage::RootFilesystem);
assert_eq!(
error.failure.errno,
Some(if symlink_parent { 40 } else { 17 })
);
assert!(error.cleanup_failure.is_none());
assert!(!root.path.join(format!("kf-{id}")).exists());
assert!(!Path::new(&format!("/var/lib/kf-judge/tasks/kf-{id}")).exists());
}
assert_eq!(children(), before);
assert_eq!(fs::read_to_string("/proc/self/mountinfo").unwrap(), mounts);
println!(
"KF_INPUT_BIND_FAILURE_PROOF ancestor_symlink=40 existing_file=17 partial_bindings_cleanup=1 task_cleanup=1 child_reaped=1 host_mounts_unchanged=1"
);
}
#[test]
#[ignore = "requires independent capped VM; bounded tmpfs exhaustion"]
fn input_copy_exhaustion_refuses_and_removes_partial_bindings() {
let root = Root::new();
let publications = Publications::new();
fs::write(publications.0.join("data-001"), vec![0x5a; 12_288]).unwrap();
let mut request = spec("input-no-space").spec().clone();
request.limits.file_bytes = 16_384;
request.limits.workdir_bytes = 16_384;
let request = request.validate(&SpecPolicy::default()).unwrap();
let before = children();
let mounts = fs::read_to_string("/proc/self/mountinfo").unwrap();
let Err(error) = InputBoundTarget::prepare(
&root.validated(),
&request,
&binary(),
&publications.capability(),
) else {
panic!("input copy exceeded bounded tmpfs");
};
assert_eq!(error.failure.stage, ExecutionStage::RootFilesystem);
assert_eq!(error.failure.errno, Some(28));
assert!(error.cleanup_failure.is_none());
assert!(!root.path.join("kf-input-no-space").exists());
assert!(!Path::new("/var/lib/kf-judge/tasks/kf-input-no-space").exists());
assert_eq!(children(), before);
assert_eq!(fs::read_to_string("/proc/self/mountinfo").unwrap(), mounts);
println!(
"KF_INPUT_SPACE_PROOF enospc=28 partial_bindings_cleanup=1 no_input_exemption=1 child_reaped=1 host_mounts_unchanged=1"
);
}