diff --git a/judge/crates/sandbox/src/acceptance.rs b/judge/crates/sandbox/src/acceptance.rs index 74a7112..3e45715 100644 --- a/judge/crates/sandbox/src/acceptance.rs +++ b/judge/crates/sandbox/src/acceptance.rs @@ -8,6 +8,7 @@ use crate::{ }; use std::{fs, io, io::Read, os::unix::fs::PermissionsExt, path::Path}; mod normal; +mod resources; pub(crate) fn verify_guest(require_env: bool) -> io::Result<()> { if (require_env && std::env::var("KF_SANDBOX_INDEPENDENT_VM").as_deref() != Ok("1")) || fs::read_to_string("/kf/INDEPENDENT_VM")?.trim() != "isolated-test-guest" @@ -117,6 +118,9 @@ pub fn run_case(case: &str) -> io::Result { if case.starts_with("normal-") { return normal::run_case(case); } + if case.starts_with("resource-") { + return resources::run_case(case); + } let rollback = case.strip_prefix("cancel-").filter(|s| { matches!( *s, diff --git a/judge/crates/sandbox/src/acceptance/resources.rs b/judge/crates/sandbox/src/acceptance/resources.rs new file mode 100644 index 0000000..d6841a6 --- /dev/null +++ b/judge/crates/sandbox/src/acceptance/resources.rs @@ -0,0 +1,43 @@ +//! Finite resource probes, executable only in the verified independent VM. +use super::{Fixture, describe}; +use crate::{ + launcher::run::{self, MonitorMode}, + spec::{ExecutionSpec, SpecPolicy}, +}; +use std::io; + +pub(super) fn run_case(case: &str) -> io::Result { + let mode = match case { + "resource-cpu" => "cpu", + "resource-oom" => "oom", + "resource-fork" => "fork", + "resource-threads" => "threads", + "resource-fds" => "fds", + _ => return Err(io::Error::other("unknown resource case")), + }; + let fixture = Fixture::new("/kf/bin/resource-fixture")?; + let config = fixture.config()?; + let root = fixture.root()?; + let mut request: ExecutionSpec = + serde_json::from_str(include_str!("../../tests/fixtures/spec.valid.json"))?; + request.run_id = case.into(); + request.argv.push(mode.into()); + request.stdin_artifact = None; + request.limits.cpu_time_us = if mode == "cpu" { 500000 } else { 2_000_000 }; + request.limits.wall_time_us = 8_000_000; + if mode == "oom" { + request.limits.memory_bytes = 32 << 20; + } + let spec = request + .validate(&SpecPolicy::default()) + .map_err(io::Error::other)?; + let completed = run::execute( + &config, + &spec, + &std::env::current_exe()?, + &root, + MonitorMode::Normal, + ) + .map_err(|e| io::Error::other(format!("resource initialization: {:?}", e.failure)))?; + describe(case, &spec, &completed, false) +} diff --git a/judge/crates/sandbox/src/accounting.rs b/judge/crates/sandbox/src/accounting.rs index 23bd73f..e5d9d2a 100644 --- a/judge/crates/sandbox/src/accounting.rs +++ b/judge/crates/sandbox/src/accounting.rs @@ -7,6 +7,7 @@ pub struct AccountingSnapshot { pub(crate) cpu_us: u64, pub(crate) peak: u64, pub(crate) events: MemoryEvents, + pids_max: u64, } impl AccountingSnapshot { pub fn cpu_time_us(&self) -> u64 { @@ -18,7 +19,7 @@ impl AccountingSnapshot { pub fn memory_events(&self) -> &MemoryEvents { &self.events } - pub(crate) fn parse(cpu: &str, peak: &str, events: &str) -> io::Result { + pub(crate) fn parse(cpu: &str, peak: &str, events: &str, pids: &str) -> io::Result { Ok(Self { cpu_us: metric(cpu, "usage_usec")?, peak: decimal(peak.trim())?, @@ -28,6 +29,7 @@ impl AccountingSnapshot { oom_kill: metric(events, "oom_kill")?, oom_group_kill: metric(events, "oom_group_kill")?, }, + pids_max: metric(pids, "max")?, }) } pub(crate) fn monotonic_since(&self, prior: &Self) -> io::Result<()> { @@ -35,8 +37,16 @@ impl AccountingSnapshot { return Err(invalid("accounting counter reset")); } self.event_delta(prior)?; + self.process_events_since(prior)?; Ok(()) } + /// Trusted fresh task group's failed creation count. Do not infer a process + /// limit from EAGAIN alone: it can have unrelated kernel causes. + pub(crate) fn process_events_since(&self, base: &Self) -> io::Result { + self.pids_max + .checked_sub(base.pids_max) + .ok_or_else(|| invalid("process event counter reset")) + } fn event_delta(&self, base: &Self) -> io::Result { let sub = |a: u64, b: u64| { a.checked_sub(b) @@ -114,6 +124,7 @@ mod tests { oom_kill: n, oom_group_kill: n, }, + pids_max: n, } } #[test] @@ -123,6 +134,7 @@ mod tests { "user_usec 20\nusage_usec 30\nsystem_usec 10\n", "4096\n", events, + "max 4\n", ) .unwrap(); assert_eq!(a.cpu_us, 30); @@ -135,12 +147,22 @@ mod tests { "usage_usec 1 extra", "usage_usec -1", ] { - assert!(AccountingSnapshot::parse(cpu, "4096", events).is_err()); + assert!(AccountingSnapshot::parse(cpu, "4096", events, "max 0").is_err()); } - assert!(AccountingSnapshot::parse("usage_usec 1", "max", events).is_err()); + assert!(AccountingSnapshot::parse("usage_usec 1", "max", events, "max 0").is_err()); assert!( - AccountingSnapshot::parse("usage_usec 1", "0", "max 1\noom 1\noom_kill 1").is_err() + AccountingSnapshot::parse("usage_usec 1", "0", "max 1\noom 1\noom_kill 1", "max 0") + .is_err() ); + for pids in [ + "", + "other 1", + "max -1", + "max 18446744073709551616", + "max 1\nmax 2", + ] { + assert!(AccountingSnapshot::parse("usage_usec 1", "0", events, pids).is_err()); + } } #[test] fn resets_fail_and_usage_keeps_absolute_fresh_group_peak_and_enrollment_cpu() { @@ -166,4 +188,12 @@ mod tests { assert!(sample(40, 199, 4).monotonic_since(&last).is_err()); assert!(sample(40, 300, 2).monotonic_since(&last).is_err()); } + #[test] + fn process_events_use_enrollment_delta_and_reject_reset_independently() { + let base = sample(10, 100, 2); + let mut current = sample(40, 300, 4); + assert_eq!(current.process_events_since(&base).unwrap(), 2); + current.pids_max = 1; + assert!(current.monotonic_since(&base).is_err()); + } } diff --git a/judge/crates/sandbox/src/cgroup.rs b/judge/crates/sandbox/src/cgroup.rs index 750046b..a44fe82 100644 --- a/judge/crates/sandbox/src/cgroup.rs +++ b/judge/crates/sandbox/src/cgroup.rs @@ -232,6 +232,7 @@ impl TaskCgroup { &read(&self.fd, "cpu.stat")?, &read(&self.fd, "memory.peak")?, &read(&self.fd, "memory.events")?, + &read(&self.fd, "pids.events")?, ) } pub(crate) fn accounting_baseline(&self) -> io::Result<&crate::accounting::AccountingSnapshot> { diff --git a/judge/crates/sandbox/src/supervisor.rs b/judge/crates/sandbox/src/supervisor.rs index efe7211..69e65d6 100644 --- a/judge/crates/sandbox/src/supervisor.rs +++ b/judge/crates/sandbox/src/supervisor.rs @@ -67,7 +67,7 @@ impl<'a> BudgetSupervisor<'a> { wall_limit: spec.spec().limits.wall_time_us, started: None, finished_at: None, - observations: Vec::with_capacity(3), + observations: Vec::with_capacity(4), ended: false, }; if supervisor.last.cpu_time_us() - supervisor.base.cpu_time_us() @@ -137,7 +137,7 @@ impl<'a> BudgetSupervisor<'a> { let sample = self.group.accounting_snapshot()?; sample.monotonic_since(&self.last)?; let usage = sample.usage(&self.base, self.wall_us()?)?; - self.observe(&usage); + self.observe(&usage, sample.process_events_since(&self.base)?); self.last = sample; if !self.observations.is_empty() { self.group.terminate_tree(TERMINATE)?; @@ -167,7 +167,7 @@ impl<'a> BudgetSupervisor<'a> { let sample = self.group.accounting_snapshot()?; sample.monotonic_since(&self.last)?; let usage = sample.usage(&self.base, self.wall_us()?)?; - self.observe(&usage); + self.observe(&usage, sample.process_events_since(&self.base)?); self.last = sample; self.ended = true; Ok(BudgetEvent::Finished { @@ -188,11 +188,11 @@ impl<'a> BudgetSupervisor<'a> { }) .transpose() } - fn observe(&mut self, usage: &ResourceUsage) { + fn observe(&mut self, usage: &ResourceUsage, process_events: u64) { let Measurement::Measured { value: wall } = usage.wall_time_us else { return; }; - let mut found = Vec::with_capacity(3); + let mut found = Vec::with_capacity(4); if matches!(&usage.memory_events,Measurement::Measured {value:e} if e.max>0&&e.oom>0&&e.oom_kill>0&&e.oom_group_kill>0) { found.push(LimitEvidence::Memory); @@ -205,6 +205,11 @@ impl<'a> BudgetSupervisor<'a> { if wall >= self.wall_limit { found.push(LimitEvidence::WallTime { observed_us: wall }); } + if process_events > 0 { + found.push(LimitEvidence::Processes { + max_events_delta: process_events, + }); + } for evidence in found { if !self .observations diff --git a/judge/crates/sandbox/tests/fixtures/resource_fixture.c b/judge/crates/sandbox/tests/fixtures/resource_fixture.c new file mode 100644 index 0000000..2a86401 --- /dev/null +++ b/judge/crates/sandbox/tests/fixtures/resource_fixture.c @@ -0,0 +1,76 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +static void sleep_forever(void) { for (;;) usleep(10000); } +static void *busy(void *unused) { + (void)unused; + volatile unsigned long value = 1; + for (;;) value = value * 1664525 + 1013904223; + return NULL; +} +static void *sleeper(void *unused) { (void)unused; sleep_forever(); return NULL; } +int main(int argc, char **argv) { + if (argc != 2) return 100; + if (!strcmp(argv[1], "cpu")) { + pid_t p = fork(); + if (p < 0) return 101; + if (p == 0) { + pthread_t threads[2]; + for (int i = 0; i < 2; ++i) if (pthread_create(&threads[i], NULL, busy, NULL)) _exit(102); + if (write(1, "CPU_CHILD_THREADS_READY\n", 24) != 24) _exit(103); + busy(NULL); + } + sleep_forever(); + } + if (!strcmp(argv[1], "oom")) { + pid_t p = fork(); + if (p < 0) return 104; + if (p == 0) { + /* Finite 128 MiB request in a 32 MiB task; no host allocator. */ + for (int i = 0; i < 128; ++i) { + volatile unsigned char *block = malloc(1 << 20); + if (!block) _exit(105); + for (int j = 0; j < (1 << 20); j += 4096) block[j] = (unsigned char)i; + } + _exit(106); + } + sleep_forever(); + } + if (!strcmp(argv[1], "fork") || !strcmp(argv[1], "threads")) { + int thread_mode = !strcmp(argv[1], "threads"); + pthread_t threads[32]; + for (int i = 0; i < 32; ++i) { + int error; + if (thread_mode) error = pthread_create(&threads[i], NULL, sleeper, NULL); + else { + pid_t p = fork(); + if (p == 0) sleep_forever(); + error = p < 0 ? errno : 0; + } + if (error) { + if (error != EAGAIN) return 107; + sleep_forever(); + } + } + return 108; /* Kernel cap must reject well before this finite ceiling. */ + } + if (!strcmp(argv[1], "fds")) { + struct rlimit limit; + if (getrlimit(RLIMIT_NOFILE, &limit) || limit.rlim_cur != 64 || limit.rlim_max != 64) return 109; + int fds[128], n = 0, saved = 0; + for (; n < 128; ++n) { + fds[n] = open("/dev/null", O_RDONLY | O_CLOEXEC); + if (fds[n] < 0) { saved = errno; break; } + } + for (int i = 0; i < n; ++i) close(fds[i]); + printf("FD_PROOF soft=64 hard=64 opened=%d errno=%d\n", n, saved); + return n == 61 && saved == EMFILE ? 0 : 110; + } + return 111; +} diff --git a/judge/crates/sandbox/tests/resource_execution_vm.rs b/judge/crates/sandbox/tests/resource_execution_vm.rs new file mode 100644 index 0000000..2e58781 --- /dev/null +++ b/judge/crates/sandbox/tests/resource_execution_vm.rs @@ -0,0 +1,137 @@ +#![cfg(all(target_os = "linux", feature = "vm-acceptance"))] +#[allow(dead_code)] +mod support; +use std::{fs, process::Command}; +fn run(case: &str) -> serde_json::Value { + support::require_vm(); + let output = Command::new(support::binary()) + .args(["vm-case", case]) + .env_clear() + .env("KF_SANDBOX_INDEPENDENT_VM", "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{case}: {} {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let v: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(v["report"]["cleanup"], "complete", "{v}"); + assert_eq!(v["report"]["errors"], serde_json::json!([]), "{v}"); + assert_eq!(v["immutable_snapshots_verified"], true); + assert_eq!( + v["proof"]["order"], + serde_json::json!([ + "verified_isolation", + "exec_gate", + "cgroup_empty", + "owned_reaped", + "final_accounting", + "final_export", + "resource_removal" + ]) + ); + for metric in [ + "cpu_time_us", + "wall_time_us", + "memory_peak_bytes", + "memory_events", + ] { + assert_eq!(v["report"]["usage"][metric]["state"], "measured", "{v}"); + } + for dir in ["tasks", "identities", "captures"] { + assert_eq!( + fs::read_dir(format!("/var/lib/kf-judge/{dir}")) + .unwrap() + .count(), + 0 + ); + } + println!("KF_RESOURCE_EXECUTION_PROOF {v}"); + v +} +fn limited(v: &serde_json::Value, kind: &str) { + assert_eq!( + v["outcome"], + serde_json::json!({"kind":"limit_exceeded", "primary":kind}), + "{v}" + ); + assert!( + v["report"]["limits"] + .as_array() + .unwrap() + .iter() + .any(|l| l["evidence"]["kind"] == kind) + ); +} +#[test] +#[ignore = "independent capped VM: aggregate busy child threads, monotonic sleep and actual group OOM"] +fn cpu_wall_and_group_oom() { + let cpu = run("resource-cpu"); + limited(&cpu, "cpu_time"); + assert!( + cpu["report"]["usage"]["cpu_time_us"]["value"] + .as_u64() + .unwrap() + >= 500000 + ); + assert!(cpu["proof"]["reaped_orphans"].as_u64().unwrap() > 0); + assert!( + cpu["stdout_prefix_hex"] + .as_str() + .unwrap() + .starts_with("4350555f4348494c445f544852454144535f5245414459") + ); + limited(&run("wall"), "wall_time"); + let oom = run("resource-oom"); + limited(&oom, "memory"); + let events = &oom["report"]["usage"]["memory_events"]["value"]; + for key in ["max", "oom", "oom_kill", "oom_group_kill"] { + assert!(events[key].as_u64().unwrap() > 0, "{oom}"); + } + assert!(events["oom_kill"].as_u64().unwrap() >= 2, "{oom}"); +} +#[test] +#[ignore = "independent capped VM: finite fork/pthread growth reaches task pids.max and full tree stop"] +fn process_and_thread_limits_use_trusted_kernel_counters() { + for case in ["resource-fork", "resource-threads"] { + let v = run(case); + limited(&v, "processes"); + assert!(v["report"]["limits"].as_array().unwrap().iter().any(|l| { + l["evidence"]["max_events_delta"] + .as_u64() + .is_some_and(|n| n > 0) + })); + } +} +#[test] +#[ignore = "independent capped VM: stream floods, real EFBIG/ENOSPC/EMFILE and finite retained data"] +fn output_file_space_and_descriptor_limits() { + for case in ["stdout", "stderr"] { + let v = run(case); + limited(&v, case); + assert_eq!(v["report"][case]["retained_bytes"], 4096); + assert_eq!(v["report"][case]["observed_bytes"]["value"], 4097); + assert_eq!(v["report"][case]["truncated"], true); + assert!(v["proof"]["reaped_orphans"].as_u64().unwrap() > 0); + } + for (case, expected) in [ + ("disk", "KF_DISK_PROOF file=1 work_enospc=1 tmp_enospc=1\n"), + ( + "resource-fds", + "FD_PROOF soft=64 hard=64 opened=61 errno=24\n", + ), + ] { + let v = run(case); + assert_eq!( + v["outcome"], + serde_json::json!({"kind":"exited","code":0}), + "{v}" + ); + assert_eq!(v["report"]["limits"], serde_json::json!([])); + let hex: String = expected.bytes().map(|b| format!("{b:02x}")).collect(); + assert_eq!(v["stdout_prefix_hex"], hex); + assert_eq!(v["report"]["stdout"]["retained_bytes"], expected.len()); + } +}