Kronecker/KF-OnlineJudge
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

merge into: Kronecker/KF-OnlineJudge:feat/single-node-delivery
Branches Tags
Kronecker/KF-OnlineJudge:main Kronecker/KF-OnlineJudge:feat/single-node-delivery Kronecker/KF-OnlineJudge:feat/admin-problem-management Kronecker/KF-OnlineJudge:feat/python-java-judging Kronecker/KF-OnlineJudge:feat/frontend-pages Kronecker/KF-OnlineJudge:feat/judging-pipeline Kronecker/KF-OnlineJudge:feat/linux-sandbox Kronecker/KF-OnlineJudge:chore/workspace-bootstrap
..
pull from: Kronecker/KF-OnlineJudge:feat/frontend-pages
Branches Tags
Kronecker/KF-OnlineJudge:feat/single-node-delivery Kronecker/KF-OnlineJudge:main Kronecker/KF-OnlineJudge:feat/admin-problem-management Kronecker/KF-OnlineJudge:feat/python-java-judging Kronecker/KF-OnlineJudge:feat/frontend-pages Kronecker/KF-OnlineJudge:feat/judging-pipeline Kronecker/KF-OnlineJudge:feat/linux-sandbox Kronecker/KF-OnlineJudge:chore/workspace-bootstrap
2 Commits
feat/single-node-delivery .. feat/frontend-pages
Author SHA1 Message Date
Kronecker 2ad35865c0 feat(frontend): show submission performance in problem workbench 2026-10-09 14:45:53 +08:00
Kronecker 62132324da feat(frontend): add submission performance distribution previews 2026-10-09 14:45:53 +08:00
579 changed files with 1474 additions and 105356 deletions
Expand all files Collapse all files
.github/workflows/engineering.yml
-203
View File
@@ -1,203 +0,0 @@
name: Trusted engineering checks
on:
push:
branches: [feat/single-node-delivery]
permissions:
contents: read
concurrency:
group: kf-onlinejudge-trusted-engineering
cancel-in-progress: false
jobs:
check:
if: github.server_url == 'https://github.com' && github.repository == 'KiriAky107/KF-OnlineJudge'
runs-on: ubuntu-24.04
timeout-minutes: 30
defaults:
run:
shell: bash --noprofile --norc -euo pipefail {0}
env:
CI: "true"
RUSTUP_TOOLCHAIN: 1.99.0
CARGO_BUILD_JOBS: "2"
CARGO_NET_RETRY: "2"
UV_PYTHON: 3.13.16
UV_NO_PROGRESS: "1"
UV_HTTP_TIMEOUT: "30"
UV_HTTP_RETRIES: "2"
steps:
- name: Check out the pushed commit
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
timeout-minutes: 2
with:
persist-credentials: false
fetch-depth: 1
fetch-tags: false
submodules: false
lfs: false
- name: Set up Node.js
uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
timeout-minutes: 3
with:
node-version: 26.11.1
check-latest: false
package-manager-cache: false
- name: Set up uv
uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
timeout-minutes: 3
with:
version: 0.12.24
python-version: 3.13.16
enable-cache: false
- name: Install locked toolchains and dependencies
timeout-minutes: 10
run: |
test "$GITHUB_SERVER_URL" = https://github.com
test "$GITHUB_REPOSITORY" = KiriAky107/KF-OnlineJudge
test "$GITHUB_EVENT_NAME" = push
test "$GITHUB_REF" = refs/heads/feat/single-node-delivery
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
uv python install 3.13.16
rustup toolchain install 1.99.0 --profile minimal --component rustfmt --component clippy
npm install --global --ignore-scripts pnpm@11.28.4
make install
- name: Check toolchain versions and CI inputs
timeout-minutes: 1
run: |
test "$(node --version)" = v26.11.1
test "$(pnpm --version)" = 11.28.4
test "$(rustc --version | cut -d ' ' -f 2)" = 1.99.0
test "$(uv --version | cut -d ' ' -f 2)" = 0.12.24
backend/.venv/bin/python - <<'PYCODE'
import os
import subprocess
import sys
assert sys.version_info[:3] == (3, 13, 16)
assert not any(name.startswith(('KF_', 'PGPASSWORD', 'PGSERVICE', 'DATABASE_URL', 'AMQP_URL'))
for name in os.environ), 'CI must not carry node credentials or VM flags'
assert not subprocess.check_output(['git', 'ls-files', '--', 'docs/', '.localplan/'])
PYCODE
- name: Check architecture and generated contracts
timeout-minutes: 3
run: |
uv run --project backend --locked python scripts/check_architecture.py
make contracts
git diff --exit-code -- contracts frontend/src/api/generated/schema.d.ts
- name: Check Backend and run ordinary tests
timeout-minutes: 5
working-directory: backend
run: |
uv run --locked ruff check . ../scripts
uv run --locked ruff format --check . ../scripts
uv run --locked pytest \
tests/test_contracts.py \
tests/test_judging_contracts.py \
tests/test_judging_v2_contracts.py \
tests/test_submission_performance_contract.py \
tests/test_admin_import_contracts.py \
tests/test_availability.py \
tests/test_dependency_errors.py \
tests/test_production_logging.py \
tests/test_delivery.py \
tests/test_delivery_io.py \
tests/test_delivery_proxy_startup.py \
tests/test_delivery_validation_work.py \
tests/test_delivery_bytecode.py \
tests/test_delivery_backup.py \
tests/test_delivery_restore_io.py \
tests/test_restore_io_contracts.py \
tests/test_restore_native_transport.py \
tests/test_operator_diagnostics.py \
tests/test_operator_imports.py \
tests/test_health.py \
tests/test_identity.py \
tests/test_coordinator_control.py \
--junitxml="$RUNNER_TEMP/kf-engineering-backend.xml"
uv run --locked python -m kf_oj.contracts.export --check
- name: Check Rust and run library and contract tests
timeout-minutes: 8
working-directory: judge
run: |
cargo fmt --all -- --check
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo test --workspace --lib --locked
cargo test --package kf-contracts --locked
cargo test --package kf-sandbox --bin kf-restore-io --locked
- name: Check and build Frontend
timeout-minutes: 5
run: make check-frontend
- name: Parse development Compose configuration
timeout-minutes: 1
run: make infra-config
- name: Collect public contracts and engineering report
timeout-minutes: 1
run: |
backend/.venv/bin/python - <<'PYCODE'
import hashlib
import json
import os
import subprocess
import xml.etree.ElementTree as ET
from pathlib import Path
source = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()
assert source == os.environ['GITHUB_SHA']
destination = Path(os.environ['RUNNER_TEMP']) / 'kf-engineering-artifacts'
destination.mkdir(mode=0o700)
names = subprocess.check_output(['git', 'ls-files', '-z', '--',
'contracts/openapi/platform.v1.json',
'contracts/schemas/*.json']).decode().split('\0')[:-1]
assert 1 <= len(names) <= 64
files, total = [], 0
for name in names:
path = Path(name)
assert not any(part.is_symlink() for part in (path, *path.parents))
assert path.is_file() and path.stat().st_size <= 2 * 1024**2
data = path.read_bytes()
json.loads(data)
total += len(data)
assert total <= 8 * 1024**2
output = destination / name
output.parent.mkdir(parents=True, exist_ok=True)
output.write_bytes(data)
files.append({'path': name, 'bytes': len(data),
'sha256': hashlib.sha256(data).hexdigest()})
junit = ET.parse(Path(os.environ['RUNNER_TEMP']) / 'kf-engineering-backend.xml')
suites = list(junit.getroot().iter('testsuite'))
backend = {name: sum(int(suite.attrib.get(name, 0)) for suite in suites)
for name in ('tests', 'failures', 'errors', 'skipped')}
assert backend['failures'] == backend['errors'] == 0
report = {'schema_version': 1, 'source_sha': source,
'scope': 'Trusted engineering checks only', 'backend': backend,
'toolchains': {'python': '3.13.16', 'rust': '1.99.0',
'node': '26.11.1', 'pnpm': '11.28.4', 'uv': '0.12.24'},
'files': files,
'not_run': ['real PostgreSQL acceptance', 'independent VM cases',
'user source execution', 'adverse archives and filesystem acceptance',
'Linux isolation acceptance', 'deployment']}
(destination / 'engineering-report.json').write_text(json.dumps(report, indent=2)+'\n')
PYCODE
- name: Upload public contracts and engineering report
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
timeout-minutes: 2
with:
name: engineering-checks-${{ github.sha }}
path: ${{ runner.temp }}/kf-engineering-artifacts
if-no-files-found: error
include-hidden-files: false
overwrite: false
retention-days: 1
compression-level: 6
AGENTS.md
-1
View File
@@ -16,7 +16,6 @@
- 沙箱未验证隔离前拒绝执行；不得用直接运行宿主命令作为临时判题实现。
- 只建立当期需要的接口和实现；未实现能力明确报告，不提供模拟成绩或虚假任务成功状态。
- 使用项目锁文件；新增依赖说明职责。凭据、私有代码和隐藏测试数据不进入仓库。
- 测试备份不得积累：测试结束并保存必要验收证据后，及时删除不再使用的备份、虚拟机磁盘/快照及临时工件；当前任务仍依赖的资源须列明用途和清理条件，被替代后立即清理，不能以保留日志为由长期保留大体积副本。
按变更运行 `make check-backend`、`make check-judge`、`make check-frontend` 和 `make architecture`。
跨工程骨架或契约改动运行完整 `make check`，记录实际通过项和未验证的能力。
Makefile
+1 -1
View File
@@ -57,7 +57,7 @@ format:
cd frontend && $(PNPM) format
api:
cd backend && $(UV) run --locked uvicorn kf_oj.api.app:create_app --factory --reload --host 127.0.0.1 --port 8000 --no-proxy-headers --limit-concurrency 32 --backlog 64
cd backend && $(UV) run --locked uvicorn kf_oj.api.app:create_app --factory --reload --host 127.0.0.1 --port 8000
frontend:
cd frontend && $(PNPM) dev
README.md
+7 -29
View File
@@ -3,26 +3,14 @@
以 Rust 自研判题与 Linux 沙箱、以 Python/FastAPI 提供平台服务的在线评测系统。
前端使用 Vue 3、TypeScript、Element Plus 和 Vite。
当前版本为 **0.0.1**：沙箱已有受控隔离、C/C++ 编译及 native 执行；平台已实现真实身份、
不可变题目、工件与幂等提交创建。事务 outbox、持久化 Coordinator、普通 Worker 和受控
helper 工厂已接通 C17/C++20、固定 CPython 3.13 与 OpenJDK 21 的普通题评测；登录、题库、提交和个人成绩页面
使用真实接口。训练、比赛、社区等扩展页面仍为开发预览。
管理控制台使用同源 Cookie 和服务端维护权限，支持有界 ZIP 题包上传、候选预览、
不可变版本发布、题目下架/恢复及维护审计。初次授权与恢复走独立运维身份的本地 CLI；
导入 Worker 仅解析数据，不执行题包内容或学生代码。操作、限额和恢复步骤见
[题目维护运维说明](deploy/admin-problems.md)。
题目工作台支持当前版本的“本题我的提交”、右侧详情与完整个人详情。
性能展示读取受验证最终 AC 的 CPU 总和与内存峰值，只比较同版本、语言、精确工具链、
预算与可信测量环境的最近有效记录。缺测、旧环境无法验证或其他记录不足五条时明确显示
不可比较；不会填入示例成绩。口径、查询边界和权限见[提交模块说明](backend/src/kf_oj/modules/submissions/README.md)。
当前版本为 **0.0.1 工程骨架**：平台 API 和协调器提供存活接口，前端可检查实际 API 连接。
沙箱隔离执行、判题、账号、题库、提交与学习业务尚未实现。
## 工程划分
| 目录 | 职责 |
| --- | --- |
| `backend/` | Python 模块化单体、平台 API、题包导入 Worker、待投递事件发布器及唯一数据库迁移入口 |
| `backend/` | Python 模块化单体、平台 API、学习 Worker、待投递事件发布器及唯一数据库迁移入口 |
| `judge/` | Rust workspace：协议、沙箱、判题编排、判题 Worker 和协调器 |
| `frontend/` | 页面、路由、请求生命周期与生成的 API 类型 |
| `contracts/` | 版本化 JSON Schema 与 OpenAPI 快照 |
@@ -37,14 +25,11 @@ Python 业务域分为 `identity`、`problems`、`submissions`、`community`、`
Rust 分为 `kf-contracts`、`kf-sandbox`、`kf-engine`、`kf-worker`、`kf-coordinator` 五个 crate。
协调器拥有执行状态写入权，判题 Worker 不直接写评测数据库。
Sandbox 内的 `kf-restore-io` 仅在固定离线恢复单元中复制和核验固定数据域，
通过共享有界协议接收文件记录；不启动程序、服务或写业务状态。
沙箱执行入口每次核验隔离与限额，任何未验证配置继续拒绝执行。
用户程序及隔离/超限验收只在独立、整体限额的无业务凭据测试 VM 内运行。
沙箱尚未验证隔离能力，当前执行入口明确拒绝运行程序。
## 安装与检查
工具链基线：Python 3.13.16 + uv、Rust 1.99.0、Node 26.11.1 + pnpm 11.28.4。
工具链基线：Python 3.13.16 + uv、Rust 1.99.0、Node 26.10.0 + pnpm 11.28.4。
Python 和 Rust 版本由工程配置固定，三套工程分别提交依赖锁文件。
开发与构建使用 Linux 原生文件系统。
@@ -77,18 +62,11 @@ make sandbox-probe # 只读观察 Linux 环境，不创建隔离资源
```
本地数据库与队列的启动步骤见 [部署说明](deploy/README.md)。
业务表和权限分工由后端 Alembic 唯一管理，见 [迁移与权限说明](backend/migrations/README.md)。
业务表和数据库 revision 尚未建立；迁移统一由后端 Alembic 管理。
模型服务与第三方插件尚未接入。
开发中的规划文档仅在本地维护，不包含在远端仓库中。
单机提交服务的配置、最小权限、停止恢复与独立 VM 验收要求见
[判题说明](judge/README.md#单机持久化协调与常驻-worker)和[部署说明](deploy/README.md#单机提交服务的启动与停止)。
当前普通题执行支持固定 GNU C17/C++20、CPython 3.13.16 和 OpenJDK 21.0.12.1+1-LTS，
使用精确字节比较。Python 固定单文件 main.py 与标准库；Java 固定默认包 Main.java/Main，
允许有界辅助/内部/匿名类。题目必须显式声明对应语言能力，旧题目不会自动获得新语言。
新版源码与普通 Worker 的沙箱执行已在独立 VM 验收；完整平台/浏览器及故障验收按工程阶段
分别记录，生产部署需另行进行。自助注册、自测、SPJ/交互题、Rust 用户程序、分布式调度
和学习能力仍待后续实现。
后续逐步实现沙箱、单机判题、分布式恢复、并行优化和学习功能。
## 许可证
backend/.env.example
+1 -11
View File
@@ -1,15 +1,5 @@
KF_OJ_ENVIRONMENT=development
KF_OJ_LOG_LEVEL=INFO
# Dedicated platform login inherits only kf_oj_platform; replace development credentials.
# Optional until database-backed use cases are implemented.
KF_OJ_DATABASE_URL=postgresql+asyncpg://kf_oj:local-dev-only@127.0.0.1:5432/kf_oj
KF_OJ_AMQP_URL=amqp://kf_oj:local-dev-only@127.0.0.1:5672/kf_oj
KF_OJ_BROWSER_ORIGINS='["http://localhost:5173","http://localhost:8000","http://127.0.0.1:5173","http://127.0.0.1:8000"]'
KF_OJ_SESSION_LIFETIME_SECONDS=43200
# Required for submissions: private absolute artifact root and trusted JSON admission policy.
# KF_OJ_ARTIFACT_ROOT=/srv/kf-platform/artifacts
KF_OJ_ARTIFACT_MAX_BYTES=536870912
KF_OJ_ARTIFACT_MAX_FILES=4096
# KF_OJ_JUDGE_ADMISSION=<JSON matching the Coordinator and Worker policy>
# Cancellation requires both; token equals Coordinator platform_token, never a Worker token.
# KF_OJ_COORDINATOR_URL=http://127.0.0.1:8100
# KF_OJ_COORDINATOR_TOKEN=<privately generated 64 lowercase hexadecimal characters>
backend/README.md
+4 -26
View File
@@ -15,33 +15,11 @@ make api
make worker-check
```
API 提供存活、真实登录/退出/当前用户、公开题目与有效 Worker 租约的清单/工件读取。
配置数据库后在 lifespan 内组合模块，配置缺失时业务端口明确不可用，存活接口仍可用。
学习 Worker 目前仅支持 `--check`；Rust 判题 Worker 使用单独的常驻消费入口。
提交创建 API 在持久化源码后，同一事务写入初始评测和 outbox；按用户与幂等键
防止重复创建，待评测总量和个人待评测数有固定上限。
待投递事件使用独立进程 `uv run --locked python -m kf_oj.infra.messaging.publisher`；
配置 `KF_OJ_DATABASE_URL` 为专用平台角色，`KF_OJ_AMQP_URL` 为内部 broker。
发布器一次只锁定一个到期事件；使用持久队列、消息及 mandatory publisher confirm，
确认后才提交投递标记。锁过期允许重投，消息不是执行权，Coordinator claim 才是。
队列最多 64 条、32 KiB，拒绝溢出，消息有效期 10 分钟；broker 不可用采用有界退避，
不阻塞提交事务。日志仅固定错误代码，不打印源码、答案或连接口令。
后台每次投递最多 12 秒，关闭连接最多 2 秒，SIGTERM 在当前有界操作结束后退出。
账号/题目/工件命令及限制见对应模块 README；模型能力尚未接入。
个人提交查询 `/api/v1/submissions` 使用 `(created_at,id)` 降序游标，最多100条；
`/{id}`、`/{id}/source` 仅允许提交本人读取，统一拒绝外来/不存在记录。
成绩投影不包含租约、工件摘要、隐藏用例 ID/输入/答案或程序输出；编译诊断至多64KiB，
缺失计量保留 null，未完成报告不展示成绩。身份/提交/内部响应均为 no-store。
`POST /{id}/cancel` 验证本人、Origin、会话和 CSRF，再调用 Coordinator 接受取消；
202仅表示已持久化意图，实际停止与资源回收由执行服务确认。
配置 `KF_OJ_COORDINATOR_URL` 为数值 loopback HTTP origin，
`KF_OJ_COORDINATOR_TOKEN` 与私有 Coordinator 的 `platform_token` 一致；
缺少配置或通信失败返回503，不直接修改执行状态。新增运行依赖 `httpx` 仅负责这个
3秒有界控制调用，禁用代理/重定向，不缓存或打印远端响应及内部凭据。
API 当前只有 `/api/v1/health/live`，不连接数据库或消息队列。
Worker 目前仅支持 `--check`，默认拒绝进入尚未实现的任务循环。
日志、配置和迁移入口已建立；业务表、登录、判题投递和模型接入尚未实现。
Python 虚拟环境位于 `backend/.venv`，可在 `backend/` 直接使用 `uv sync --locked` 和 `uv run --locked`。
数据库由 Alembic 唯一管理；业务模型在 `migrations/env.py` 中显式登记元数据。
数据库由 Alembic 唯一管理；业务模型落地后在 `migrations/env.py` 中显式登记元数据。
不要用 API 启动时的 `create_all` 代替迁移，也不要让 Rust 维护另一套迁移历史。
配置示例在 `.env.example`；执行迁移前复制为 `.env` 并设置实际数据库连接。
backend/migrations/README.md
-39
View File
@@ -1,39 +0,0 @@
# 数据库所有权
Alembic 是唯一的 schema 入口。先对专用 PostgreSQL 18 数据库运行 `uv run --locked alembic
upgrade head`，API 不在启动时创建表。迁移账号拥有该数据库对象并需要 `CREATEROLE`；它不作为
任何运行服务账号。生产配置及登录角色由部署方提供，不在源码里设置默认数据库口令。
初始迁移建立两个受限的 `NOLOGIN` 权限组：`kf_oj_platform` 和 `kf_oj_coordinator`。
平台组创建 submission / queued evaluation / outbox，不能指定或更新执行状态、attempt 或成绩。
协调器组领取和更新评测、插入 attempt / 用例成绩及恢复通知，不能修改账号、题库或提交身份。
Worker 没有数据库角色，也不需要数据库连接配置。
后续迁移另建 `kf_oj_identity_operator` 与 `kf_oj_problem_import` 两个受限 `NOLOGIN`
权限组。前者只通过固定账号授权函数维护管理员能力/启用状态，不能读取口令哈希、
直接修改角色字段或绕过最后一名启用管理员保护；后者仅处理已授权导入、候选/工件与
追加审计，不能写评测状态、attempt、终态或表现指标。判题 Worker 继续无数据库身份。
平台写入权按列收敛；当前会话与授权版本在每次维护事务内重新核对。
可信表现指标由 Coordinator 在有效最终报告事务中派生，平台、导入器和普通 Worker
不得自行写入。候选与发布版本/用例不可变，成功维护回执及审计只追加；清理保留所有
已发布版本、合法旧提交和未结束候选的工件引用。
升级前停止新维护写入、备份专用数据库与私有工件，并用独立迁移账号执行升级和
`uv run --locked alembic check`。既有登录角色继续绑定各自最小权限组，迁移账号不进入
运行服务配置。降级不作为丢弃候选或回滚一次发布的手段；含新语言/已引用数据时的
降级拒绝须先核对原因，不能清空历史来通过检查。账号 bootstrap/恢复步骤见
[维护运维说明](../../deploy/admin-problems.md)。
服务使用各自独立的普通 `LOGIN` 角色，分别加入对应权限组；登录角色不能拥有数据库/表、
不能加入迁移账号或其他高权限组，且必须为 `NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS`。
初始迁移拒绝已有的高权限同名组和继承其他角色的同名组。应用启动时不运行 migration。
降级删除本项目业务表与触发器，保留权限组及操作方配置的登录账号，避免删除集群级身份。
发布是一次单向转换：先在事务内创建版本和连续测试点，再设置 `published_at`。
数据库触发器拒绝再次发布、修改已发布版本和追加/改写已发布测试点；评测通过复合外键
永久绑定 submission 的版本，当前 attempt 和用例成绩也由复合外键绑定同一评测及版本。
工件引用具有数据库生成的 `ready` 状态，引用存在时不能转为 `deleting` 或删除元数据。
真实数据库验收使用显式 `KF_OJ_TEST_POSTGRES` 管理连接。每个测试只创建和删除随机命名的
`kf_oj_test_*` 数据库，不清空已有业务库。例行无数据库测试会明确跳过这些验收；跳过不算通过。
backend/migrations/env.py
+1 -8
View File
@@ -5,21 +5,14 @@ from alembic import context
from sqlalchemy import Connection
from kf_oj.core.settings import Settings
from kf_oj.infra.artifacts import models as artifact_models # noqa: F401
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.session import build_engine
from kf_oj.infra.messaging import models as messaging_models # noqa: F401
from kf_oj.modules.identity import models as identity_models # noqa: F401
from kf_oj.modules.operations import models as operations_models # noqa: F401
from kf_oj.modules.problems import import_models, maintenance_models # noqa: F401
from kf_oj.modules.problems import models as problem_models # noqa: F401
from kf_oj.modules.submissions import models as submission_models # noqa: F401
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
# Explicit composition root. Infra and domains do not import foreign ORM models.
# Register each business module's ORM models here when its schema is implemented.
target_metadata = Base.metadata
backend/migrations/versions/0001_judging_foundation_judging_foundation_and_role_boundaries.py
-588
View File
@@ -1,588 +0,0 @@
"""judging foundation and role boundaries
Revision ID: 0001_judging_foundation
Revises:
Create Date: 2026-10-09 00:57:34.004943
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "0001_judging_foundation"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def install_guards_and_permissions() -> None:
# Serialize case insertion against the one-way publication transition.
op.execute("""
CREATE FUNCTION public.kf_immutable_case() RETURNS trigger LANGUAGE plpgsql
SET search_path = pg_catalog, public AS $$
DECLARE published timestamptz;
BEGIN
IF TG_OP = 'UPDATE' THEN
RAISE EXCEPTION 'test cases are immutable' USING ERRCODE = '23514';
END IF;
SELECT published_at INTO published FROM public.problem_versions
WHERE id = CASE WHEN TG_OP = 'DELETE' THEN OLD.version_id ELSE NEW.version_id END
FOR UPDATE;
IF published IS NOT NULL THEN
RAISE EXCEPTION 'published test cases are immutable' USING ERRCODE = '23514';
END IF;
IF TG_OP = 'DELETE' THEN RETURN OLD; END IF;
RETURN NEW;
END $$
""")
op.execute("""
CREATE TRIGGER kf_immutable_case BEFORE INSERT OR UPDATE OR DELETE ON public.problem_cases
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_case()
""")
op.execute("""
CREATE FUNCTION public.kf_immutable_version() RETURNS trigger LANGUAGE plpgsql
SET search_path = pg_catalog, public AS $$
DECLARE count_cases integer; last_position integer;
BEGIN
IF TG_OP = 'INSERT' THEN
IF NEW.published_at IS NOT NULL THEN
RAISE EXCEPTION 'publish after inserting cases' USING ERRCODE = '23514';
END IF;
RETURN NEW;
END IF;
IF TG_OP = 'DELETE' THEN
IF OLD.published_at IS NOT NULL THEN
RAISE EXCEPTION 'published version is immutable' USING ERRCODE = '23514';
END IF;
RETURN OLD;
END IF;
IF OLD.published_at IS NOT NULL OR NEW.published_at IS NULL
OR (to_jsonb(NEW) - 'published_at') != (to_jsonb(OLD) - 'published_at') THEN
RAISE EXCEPTION 'version may only be published once' USING ERRCODE = '23514';
END IF;
SELECT count(*), max(position) INTO count_cases, last_position
FROM public.problem_cases WHERE version_id = OLD.id;
IF count_cases NOT BETWEEN 1 AND 256 OR last_position != count_cases - 1 THEN
RAISE EXCEPTION 'publication requires contiguous ordered cases'
USING ERRCODE = '23514';
END IF;
RETURN NEW;
END $$
""")
op.execute("""
CREATE TRIGGER kf_immutable_version BEFORE INSERT OR UPDATE OR DELETE
ON public.problem_versions
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_version()
""")
op.execute("REVOKE ALL ON FUNCTION public.kf_immutable_case() FROM PUBLIC")
op.execute("REVOKE ALL ON FUNCTION public.kf_immutable_version() FROM PUBLIC")
# Migration owner needs CREATEROLE; runtime logins are separate, unprivileged group members.
for role in ("kf_oj_platform", "kf_oj_coordinator"):
op.execute(f"""
DO $$ BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{role}') THEN
CREATE ROLE {role} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS;
ELSIF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '{role}' AND
(rolcanlogin OR rolsuper OR rolcreatedb OR rolcreaterole OR rolbypassrls))
OR EXISTS (SELECT 1 FROM pg_auth_members m JOIN pg_roles r ON r.oid=m.member
WHERE r.rolname='{role}') THEN
RAISE EXCEPTION 'unsafe existing runtime role';
END IF;
END $$
""")
tables = (
"accounts,sessions,login_throttles,artifacts,problems,problem_versions,problem_cases,"
"submissions,evaluations,judge_attempts,case_results,outbox"
)
op.execute(f"REVOKE ALL ON {tables} FROM PUBLIC, kf_oj_platform, kf_oj_coordinator")
op.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC")
op.execute("GRANT USAGE ON SCHEMA public TO kf_oj_platform, kf_oj_coordinator")
op.execute(f"GRANT SELECT ON {tables} TO kf_oj_platform")
op.execute("GRANT INSERT,UPDATE,DELETE ON accounts,sessions,login_throttles TO kf_oj_platform")
op.execute("GRANT INSERT,SELECT,DELETE ON artifacts TO kf_oj_platform")
op.execute("GRANT UPDATE(state) ON artifacts TO kf_oj_platform")
op.execute(
"GRANT INSERT ON problems,problem_versions,problem_cases,submissions,outbox "
"TO kf_oj_platform"
)
op.execute("GRANT UPDATE(active_version_id,enabled) ON problems TO kf_oj_platform")
op.execute("GRANT UPDATE(published_at) ON problem_versions TO kf_oj_platform")
op.execute("GRANT INSERT(id,submission_id,problem_version_id) ON evaluations TO kf_oj_platform")
op.execute(
"GRANT UPDATE(failures,next_attempt_at,locked_until,lock_owner,delivered_at,"
"last_error_code) ON outbox TO kf_oj_platform"
)
op.execute(
"GRANT SELECT ON problems,problem_versions,problem_cases,artifacts,submissions,"
"evaluations,judge_attempts,case_results,outbox TO kf_oj_coordinator"
)
op.execute(
"GRANT UPDATE(state,fencing_version,attempts_count,current_attempt_id,not_before,"
"cancel_requested_at,updated_at,verdict,failure_code,final_report,final_report_digest) "
"ON evaluations TO kf_oj_coordinator"
)
op.execute("GRANT INSERT ON judge_attempts,case_results TO kf_oj_coordinator")
op.execute(
"GRANT UPDATE(expires_at,revoked_at,sequence,stage,case_index,report,report_digest,"
"cleanup_complete) ON judge_attempts TO kf_oj_coordinator"
)
# Recovery may enqueue the same immutable notification, but cannot change identity fields.
op.execute("GRANT INSERT ON outbox TO kf_oj_coordinator")
op.execute("GRANT UPDATE(next_attempt_at,delivered_at) ON outbox TO kf_oj_coordinator")
def upgrade() -> None:
op.create_table(
"accounts",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("username", sa.String(length=24), nullable=False),
sa.Column("password_hash", sa.Text(), nullable=False),
sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.CheckConstraint(
"username ~ '^[a-z0-9_]{3,24}$'", name=op.f("ck_accounts_canonical_username")
),
sa.CheckConstraint(
"char_length(password_hash) BETWEEN 40 AND 256",
name=op.f("ck_accounts_password_hash_size"),
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_accounts")),
sa.UniqueConstraint("username", name=op.f("uq_accounts_username")),
)
op.create_table(
"artifacts",
sa.Column("digest", sa.String(length=64), nullable=False),
sa.Column("bytes", sa.BigInteger(), nullable=False),
sa.Column("state", sa.String(length=8), server_default="ready", nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.CheckConstraint("digest ~ '^[0-9a-f]{64}$'", name=op.f("ck_artifacts_digest")),
sa.CheckConstraint("state IN ('ready', 'deleting')", name=op.f("ck_artifacts_state")),
sa.CheckConstraint("bytes BETWEEN 0 AND 67108864", name=op.f("ck_artifacts_bytes")),
sa.PrimaryKeyConstraint("digest", name=op.f("pk_artifacts")),
sa.UniqueConstraint("digest", "state", name=op.f("uq_artifacts_digest")),
)
op.create_table(
"login_throttles",
sa.Column("bucket_hash", sa.String(length=64), nullable=False),
sa.Column("attempts", sa.Integer(), nullable=False),
sa.Column("reset_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint(
"bucket_hash ~ '^[0-9a-f]{64}$'", name=op.f("ck_login_throttles_bucket_digest")
),
sa.CheckConstraint(
"attempts BETWEEN 1 AND 10", name=op.f("ck_login_throttles_attempt_budget")
),
sa.PrimaryKeyConstraint("bucket_hash", name=op.f("pk_login_throttles")),
)
op.create_index("ix_login_throttles_reset", "login_throttles", ["reset_at"], unique=False)
op.create_table(
"problems",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("slug", sa.String(length=64), nullable=False),
sa.Column("active_version_id", sa.Uuid(), nullable=True),
sa.Column("enabled", sa.Boolean(), server_default="true", nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.CheckConstraint("slug ~ '^[a-z0-9][a-z0-9_-]{0,63}$'", name=op.f("ck_problems_slug")),
sa.PrimaryKeyConstraint("id", name=op.f("pk_problems")),
sa.UniqueConstraint("slug", name=op.f("uq_problems_slug")),
)
op.create_table(
"problem_versions",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("problem_id", sa.Uuid(), nullable=False),
sa.Column("revision", sa.Integer(), nullable=False),
sa.Column("title", sa.String(length=160), nullable=False),
sa.Column("statement", sa.Text(), nullable=False),
sa.Column("languages", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("toolchain", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("compile_limits", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("run_limits", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("job_wall_time_us", sa.BigInteger(), nullable=False),
sa.Column("comparison", sa.String(length=16), server_default="exact_bytes", nullable=False),
sa.Column("public_samples", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("published_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.CheckConstraint(
"comparison = 'exact_bytes'", name=op.f("ck_problem_versions_comparison")
),
sa.CheckConstraint(
"char_length(title) BETWEEN 1 AND 160", name=op.f("ck_problem_versions_title_size")
),
sa.CheckConstraint(
"job_wall_time_us BETWEEN 1000 AND 3600000000",
name=op.f("ck_problem_versions_job_wall"),
),
sa.CheckConstraint(
"octet_length(statement) BETWEEN 1 AND 262144",
name=op.f("ck_problem_versions_statement_bytes"),
),
sa.CheckConstraint("revision > 0", name=op.f("ck_problem_versions_revision")),
sa.ForeignKeyConstraint(
["problem_id"], ["problems.id"], name=op.f("fk_problem_versions_problem_id_problems")
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_problem_versions")),
sa.UniqueConstraint("id", "problem_id", name=op.f("uq_problem_versions_id")),
sa.UniqueConstraint("problem_id", "revision", name=op.f("uq_problem_versions_problem_id")),
)
op.create_index(
"ix_problem_versions_public",
"problem_versions",
["problem_id", "published_at"],
unique=False,
)
op.create_table(
"sessions",
sa.Column("token_hash", sa.String(length=64), nullable=False),
sa.Column("account_id", sa.Uuid(), nullable=False),
sa.Column("csrf_hash", sa.String(length=64), nullable=False),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.CheckConstraint("csrf_hash ~ '^[0-9a-f]{64}$'", name=op.f("ck_sessions_csrf_digest")),
sa.CheckConstraint("token_hash ~ '^[0-9a-f]{64}$'", name=op.f("ck_sessions_token_digest")),
sa.CheckConstraint("expires_at > created_at", name=op.f("ck_sessions_expiry")),
sa.ForeignKeyConstraint(
["account_id"], ["accounts.id"], name=op.f("fk_sessions_account_id_accounts")
),
sa.PrimaryKeyConstraint("token_hash", name=op.f("pk_sessions")),
)
op.create_index(
"ix_sessions_account_expiry", "sessions", ["account_id", "expires_at"], unique=False
)
op.create_table(
"problem_cases",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("version_id", sa.Uuid(), nullable=False),
sa.Column("position", sa.Integer(), nullable=False),
sa.Column("input_digest", sa.String(length=64), nullable=False),
sa.Column("expected_digest", sa.String(length=64), nullable=False),
sa.Column(
"artifact_state",
sa.String(length=8),
sa.Computed("'ready'", persisted=True),
nullable=False,
),
sa.CheckConstraint("position BETWEEN 0 AND 255", name=op.f("ck_problem_cases_position")),
sa.ForeignKeyConstraint(
["expected_digest", "artifact_state"],
["artifacts.digest", "artifacts.state"],
name="fk_problem_cases_expected_digest_artifacts",
),
sa.ForeignKeyConstraint(
["input_digest", "artifact_state"],
["artifacts.digest", "artifacts.state"],
name=op.f("fk_problem_cases_input_digest_artifacts"),
),
sa.ForeignKeyConstraint(
["version_id"],
["problem_versions.id"],
name=op.f("fk_problem_cases_version_id_problem_versions"),
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_problem_cases")),
sa.UniqueConstraint("id", "version_id", name=op.f("uq_problem_cases_id")),
sa.UniqueConstraint("version_id", "position", name=op.f("uq_problem_cases_version_id")),
)
op.create_table(
"submissions",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("account_id", sa.Uuid(), nullable=False),
sa.Column("problem_id", sa.Uuid(), nullable=False),
sa.Column("problem_version_id", sa.Uuid(), nullable=False),
sa.Column("language", sa.String(length=8), nullable=False),
sa.Column("source_digest", sa.String(length=64), nullable=False),
sa.Column(
"artifact_state",
sa.String(length=8),
sa.Computed("'ready'", persisted=True),
nullable=False,
),
sa.Column("idempotency_key", sa.Uuid(), nullable=False),
sa.Column("request_digest", sa.String(length=64), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.CheckConstraint("language IN ('c17', 'cpp20')", name=op.f("ck_submissions_language")),
sa.CheckConstraint(
"request_digest ~ '^[0-9a-f]{64}$'", name=op.f("ck_submissions_request_digest")
),
sa.ForeignKeyConstraint(
["account_id"], ["accounts.id"], name=op.f("fk_submissions_account_id_accounts")
),
sa.ForeignKeyConstraint(
["problem_version_id", "problem_id"],
["problem_versions.id", "problem_versions.problem_id"],
name=op.f("fk_submissions_problem_version_id_problem_versions"),
),
sa.ForeignKeyConstraint(
["source_digest", "artifact_state"],
["artifacts.digest", "artifacts.state"],
name=op.f("fk_submissions_source_digest_artifacts"),
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_submissions")),
sa.UniqueConstraint(
"account_id", "idempotency_key", name=op.f("uq_submissions_account_id")
),
sa.UniqueConstraint("id", "problem_version_id", name=op.f("uq_submissions_id")),
)
op.create_index(
"ix_submissions_account_history",
"submissions",
["account_id", "created_at", "id"],
unique=False,
)
op.create_table(
"evaluations",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("submission_id", sa.Uuid(), nullable=False),
sa.Column("problem_version_id", sa.Uuid(), nullable=False),
sa.Column("state", sa.String(length=16), server_default="queued", nullable=False),
sa.Column("fencing_version", sa.BigInteger(), server_default="0", nullable=False),
sa.Column("attempts_count", sa.Integer(), server_default="0", nullable=False),
sa.Column("max_attempts", sa.Integer(), server_default="3", nullable=False),
sa.Column("current_attempt_id", sa.Uuid(), nullable=True),
sa.Column(
"not_before",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.Column("cancel_requested_at", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"updated_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.Column("verdict", sa.String(length=32), nullable=True),
sa.Column("failure_code", sa.String(length=32), nullable=True),
sa.Column("final_report", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("final_report_digest", sa.String(length=64), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.CheckConstraint(
"state IN ('queued','claimed','compiling','running','cancelling',"
"'finished','cancelled','failed')",
name=op.f("ck_evaluations_state"),
),
sa.CheckConstraint(
"state NOT IN ('claimed','compiling','running','cancelling') "
"OR current_attempt_id IS NOT NULL",
name=op.f("ck_evaluations_active_attempt"),
),
sa.CheckConstraint(
"attempts_count BETWEEN 0 AND max_attempts", name=op.f("ck_evaluations_attempts_count")
),
sa.CheckConstraint("fencing_version >= 0", name=op.f("ck_evaluations_fence")),
sa.CheckConstraint(
"max_attempts BETWEEN 1 AND 5", name=op.f("ck_evaluations_max_attempts")
),
sa.ForeignKeyConstraint(
["submission_id", "problem_version_id"],
["submissions.id", "submissions.problem_version_id"],
name=op.f("fk_evaluations_submission_id_submissions"),
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_evaluations")),
sa.UniqueConstraint("id", "problem_version_id", name=op.f("uq_evaluations_id")),
sa.UniqueConstraint("submission_id", name=op.f("uq_evaluations_submission_id")),
)
op.create_index(
"ix_evaluations_recovery", "evaluations", ["state", "not_before", "id"], unique=False
)
op.create_table(
"judge_attempts",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("evaluation_id", sa.Uuid(), nullable=False),
sa.Column("worker_id", sa.Uuid(), nullable=False),
sa.Column("number", sa.Integer(), nullable=False),
sa.Column("fencing_version", sa.BigInteger(), nullable=False),
sa.Column("lease_token_hash", sa.String(length=64), nullable=False),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("manifest", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("manifest_digest", sa.String(length=64), nullable=False),
sa.Column("sequence", sa.BigInteger(), server_default="0", nullable=False),
sa.Column("stage", sa.String(length=16), nullable=True),
sa.Column("case_index", sa.Integer(), nullable=True),
sa.Column("report", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("report_digest", sa.String(length=64), nullable=True),
sa.Column("cleanup_complete", sa.Boolean(), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.CheckConstraint(
"lease_token_hash ~ '^[0-9a-f]{64}$'", name=op.f("ck_judge_attempts_token_digest")
),
sa.CheckConstraint(
"manifest_digest ~ '^[0-9a-f]{64}$'", name=op.f("ck_judge_attempts_manifest_digest")
),
sa.CheckConstraint("fencing_version > 0", name=op.f("ck_judge_attempts_fence")),
sa.CheckConstraint("number BETWEEN 1 AND 5", name=op.f("ck_judge_attempts_number")),
sa.CheckConstraint("sequence >= 0", name=op.f("ck_judge_attempts_sequence")),
sa.ForeignKeyConstraint(
["evaluation_id"],
["evaluations.id"],
name=op.f("fk_judge_attempts_evaluation_id_evaluations"),
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_judge_attempts")),
sa.UniqueConstraint(
"evaluation_id", "fencing_version", name="uq_judge_attempts_evaluation_fence"
),
sa.UniqueConstraint("evaluation_id", "number", name="uq_judge_attempts_evaluation_number"),
sa.UniqueConstraint("id", "evaluation_id", name=op.f("uq_judge_attempts_id")),
)
op.create_index(
"ix_judge_attempts_expiry", "judge_attempts", ["expires_at", "evaluation_id"], unique=False
)
op.create_table(
"outbox",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("evaluation_id", sa.Uuid(), nullable=False),
sa.Column("kind", sa.String(length=32), server_default="judge.evaluation", nullable=False),
sa.Column("failures", sa.Integer(), server_default="0", nullable=False),
sa.Column(
"next_attempt_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.Column("locked_until", sa.DateTime(timezone=True), nullable=True),
sa.Column("lock_owner", sa.Uuid(), nullable=True),
sa.Column("delivered_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error_code", sa.String(length=32), nullable=True),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.CheckConstraint("kind = 'judge.evaluation'", name=op.f("ck_outbox_kind")),
sa.CheckConstraint("failures BETWEEN 0 AND 100", name=op.f("ck_outbox_failure_budget")),
sa.ForeignKeyConstraint(
["evaluation_id"], ["evaluations.id"], name=op.f("fk_outbox_evaluation_id_evaluations")
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_outbox")),
sa.UniqueConstraint("evaluation_id", name=op.f("uq_outbox_evaluation_id")),
)
op.create_index(
"ix_outbox_delivery", "outbox", ["delivered_at", "next_attempt_at", "id"], unique=False
)
op.create_table(
"case_results",
sa.Column("attempt_id", sa.Uuid(), nullable=False),
sa.Column("case_id", sa.Uuid(), nullable=False),
sa.Column("evaluation_id", sa.Uuid(), nullable=False),
sa.Column("problem_version_id", sa.Uuid(), nullable=False),
sa.Column("position", sa.Integer(), nullable=False),
sa.Column("verdict", sa.String(length=32), nullable=False),
sa.Column("cpu_time_us", sa.BigInteger(), nullable=True),
sa.Column("memory_peak_bytes", sa.BigInteger(), nullable=True),
sa.CheckConstraint(
"cpu_time_us IS NULL OR cpu_time_us >= 0", name=op.f("ck_case_results_cpu")
),
sa.CheckConstraint(
"memory_peak_bytes IS NULL OR memory_peak_bytes >= 0",
name=op.f("ck_case_results_memory"),
),
sa.CheckConstraint("position BETWEEN 0 AND 255", name=op.f("ck_case_results_position")),
sa.ForeignKeyConstraint(
["attempt_id", "evaluation_id"],
["judge_attempts.id", "judge_attempts.evaluation_id"],
name=op.f("fk_case_results_attempt_id_judge_attempts"),
),
sa.ForeignKeyConstraint(
["case_id", "problem_version_id"],
["problem_cases.id", "problem_cases.version_id"],
name=op.f("fk_case_results_case_id_problem_cases"),
),
sa.ForeignKeyConstraint(
["evaluation_id", "problem_version_id"],
["evaluations.id", "evaluations.problem_version_id"],
name=op.f("fk_case_results_evaluation_id_evaluations"),
),
sa.PrimaryKeyConstraint("attempt_id", "case_id", name=op.f("pk_case_results")),
sa.UniqueConstraint("attempt_id", "position", name=op.f("uq_case_results_attempt_id")),
)
op.create_foreign_key(
"fk_problems_active_version",
"problems",
"problem_versions",
["active_version_id", "id"],
["id", "problem_id"],
deferrable=True,
initially="DEFERRED",
)
op.create_foreign_key(
"fk_evaluations_current_attempt",
"evaluations",
"judge_attempts",
["current_attempt_id", "id"],
["id", "evaluation_id"],
deferrable=True,
initially="DEFERRED",
)
install_guards_and_permissions()
def downgrade() -> None:
op.drop_constraint("fk_evaluations_current_attempt", "evaluations", type_="foreignkey")
op.drop_constraint("fk_problems_active_version", "problems", type_="foreignkey")
op.drop_table("case_results")
op.drop_index("ix_outbox_delivery", table_name="outbox")
op.drop_table("outbox")
op.drop_index("ix_judge_attempts_expiry", table_name="judge_attempts")
op.drop_table("judge_attempts")
op.drop_index("ix_evaluations_recovery", table_name="evaluations")
op.drop_table("evaluations")
op.drop_index("ix_submissions_account_history", table_name="submissions")
op.drop_table("submissions")
op.drop_table("problem_cases")
op.drop_index("ix_sessions_account_expiry", table_name="sessions")
op.drop_table("sessions")
op.drop_index("ix_problem_versions_public", table_name="problem_versions")
op.drop_table("problem_versions")
op.drop_table("problems")
op.drop_index("ix_login_throttles_reset", table_name="login_throttles")
op.drop_table("login_throttles")
op.drop_table("artifacts")
op.drop_table("accounts")
op.execute("DROP FUNCTION public.kf_immutable_version()")
op.execute("DROP FUNCTION public.kf_immutable_case()")
# NOLOGIN groups are intentionally retained: a downgrade never deletes operator roles.
backend/migrations/versions/0002_multi_runtime_versions.py
-247
View File
@@ -1,247 +0,0 @@
"""Immutable per-language judging configuration and extended submission language IDs.
Revision ID: 0002_multi_runtime
Revises: 0001_judging_foundation
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.postgresql import JSONB
revision = "0002_multi_runtime"
down_revision = "0001_judging_foundation"
branch_labels = None
depends_on = None
CONFIGURATION_CHECK = """
(judging_schema_version = 1 AND language_configurations IS NULL
AND toolchain IS NOT NULL AND compile_limits IS NOT NULL
AND run_limits IS NOT NULL AND job_wall_time_us IS NOT NULL)
OR
(judging_schema_version = 2 AND language_configurations IS NOT NULL
AND toolchain IS NULL AND compile_limits IS NULL
AND run_limits IS NULL AND job_wall_time_us IS NULL
AND public.kf_valid_language_configurations(languages, language_configurations))
"""
def upgrade() -> None:
# Metadata-only additions/widening; do not UPDATE any immutable published rows.
op.add_column(
"problem_versions",
sa.Column("judging_schema_version", sa.SmallInteger(), nullable=False, server_default="1"),
)
op.add_column(
"problem_versions",
sa.Column("language_configurations", JSONB(none_as_null=True), nullable=True),
)
for column, kind in (
("toolchain", JSONB()),
("compile_limits", JSONB()),
("run_limits", JSONB()),
("job_wall_time_us", sa.BigInteger()),
):
op.alter_column("problem_versions", column, existing_type=kind, nullable=True)
op.alter_column(
"submissions",
"language",
existing_type=sa.String(8),
type_=sa.String(16),
existing_nullable=False,
)
op.drop_constraint(op.f("ck_submissions_language"), "submissions", type_="check")
op.create_check_constraint(
op.f("ck_submissions_language"),
"submissions",
"language IN ('c17','cpp20','python313','java21')",
)
op.execute("""
CREATE FUNCTION public.kf_valid_runtime_limits(limits jsonb)
RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT
SET search_path = pg_catalog, public AS $$
DECLARE entry record;
BEGIN
IF jsonb_typeof(limits) != 'object' OR
(SELECT count(*) FROM jsonb_object_keys(limits)) != 10 THEN RETURN false; END IF;
FOR entry IN SELECT * FROM (VALUES
('cpu_time_us',1000::numeric,60000000::numeric),
('wall_time_us',1000,120000000), ('memory_bytes',1,2147483648),
('stack_bytes',1,268435456), ('processes',1,128), ('open_files',3,1024),
('stdout_bytes',1,16777216), ('stderr_bytes',1,16777216),
('file_bytes',1,67108864), ('workdir_bytes',1,268435456)
) AS bounds(name,lo,hi) LOOP
IF NOT limits ? entry.name OR jsonb_typeof(limits->entry.name) != 'number'
OR (limits->>entry.name) !~ '^(0|[1-9][0-9]*)$'
OR length(limits->>entry.name) > 19 THEN RETURN false; END IF;
IF (limits->>entry.name)::numeric NOT BETWEEN entry.lo AND entry.hi
THEN RETURN false; END IF;
END LOOP;
RETURN (limits->>'stack_bytes')::numeric <= (limits->>'memory_bytes')::numeric
AND (limits->>'file_bytes')::numeric <= (limits->>'workdir_bytes')::numeric
AND (limits->>'stdout_bytes')::numeric <= (limits->>'file_bytes')::numeric
AND (limits->>'stderr_bytes')::numeric <= (limits->>'file_bytes')::numeric
AND (limits->>'stdout_bytes')::numeric + (limits->>'stderr_bytes')::numeric
<= (limits->>'workdir_bytes')::numeric;
END $$
""")
op.execute("""
CREATE FUNCTION public.kf_valid_language_configurations(languages jsonb, configs jsonb)
RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT
SET search_path = pg_catalog, public AS $$
DECLARE item record; cfg jsonb; runtime jsonb; lang text; seen text[] := ARRAY[]::text[];
phase jsonb; preparing boolean;
BEGIN
IF jsonb_typeof(languages) != 'array' OR jsonb_typeof(configs) != 'array'
THEN RETURN false; END IF;
IF jsonb_array_length(configs) NOT BETWEEN 1 AND 4 OR
jsonb_array_length(languages) != jsonb_array_length(configs)
THEN RETURN false; END IF;
FOR item IN SELECT value, ordinality FROM jsonb_array_elements(configs)
WITH ORDINALITY LOOP
cfg := item.value;
IF jsonb_typeof(cfg) != 'object' OR
(SELECT count(*) FROM jsonb_object_keys(cfg)) != 5 OR NOT cfg ?&
ARRAY['language','toolchain','compile_limits','run_limits','job_wall_time_us']
THEN RETURN false; END IF;
IF jsonb_typeof(cfg->'language') != 'string' THEN RETURN false; END IF;
lang := cfg->>'language';
IF lang NOT IN ('c17','cpp20','python313','java21') OR lang = ANY(seen)
OR languages->(item.ordinality::int-1) != cfg->'language'
THEN RETURN false; END IF;
seen := array_append(seen,lang); runtime := cfg->'toolchain';
IF jsonb_typeof(runtime) != 'object' OR
(SELECT count(*) FROM jsonb_object_keys(runtime)) != 5 OR NOT runtime ?&
ARRAY['toolchain_id','image_kind','image_revision','image_digest','runtime_version']
THEN RETURN false; END IF;
IF jsonb_typeof(runtime->'toolchain_id') != 'string' OR
jsonb_typeof(runtime->'image_kind') != 'string' OR
jsonb_typeof(runtime->'runtime_version') != 'string' OR
jsonb_typeof(runtime->'image_digest') != 'string' OR
(runtime->>'image_digest') !~ '^[0-9a-f]{64}$' OR
(runtime->>'runtime_version') !~ '^[A-Za-z0-9][A-Za-z0-9_.+-]{0,63}$' OR
jsonb_typeof(runtime->'image_revision') != 'number' OR
(runtime->>'image_revision') !~ '^[1-9][0-9]{0,9}$'
THEN RETURN false; END IF;
IF (runtime->>'image_revision')::numeric > 4294967295 THEN RETURN false; END IF;
IF lang IN ('c17','cpp20') AND (runtime->>'toolchain_id' != 'gnu-cxx-v1' OR
runtime->>'image_kind' != 'linux_gnu_cxx_x86_64_v1') THEN RETURN false; END IF;
IF lang = 'python313' AND (runtime->>'toolchain_id' != 'cpython313-v1' OR
runtime->>'image_kind' != 'linux_cpython313_x86_64_v1' OR
runtime->>'runtime_version' != '3.13.16') THEN RETURN false; END IF;
IF lang = 'java21' AND (runtime->>'toolchain_id' != 'openjdk21-v1' OR
runtime->>'image_kind' != 'linux_openjdk21_x86_64_v1' OR
(runtime->>'runtime_version') !~
'^21[.][0-9]+[.][0-9]+([.][0-9]+)?[+][1-9][0-9]*(-LTS)?$')
THEN RETURN false; END IF;
IF NOT public.kf_valid_runtime_limits(cfg->'compile_limits') OR
NOT public.kf_valid_runtime_limits(cfg->'run_limits') THEN RETURN false; END IF;
IF jsonb_typeof(cfg->'job_wall_time_us') != 'number' OR
(cfg->>'job_wall_time_us') !~ '^[1-9][0-9]{0,9}$' THEN RETURN false; END IF;
IF (cfg->>'job_wall_time_us')::numeric NOT BETWEEN 1000 AND 3600000000 OR
(cfg->'compile_limits'->>'wall_time_us')::numeric +
(cfg->'run_limits'->>'wall_time_us')::numeric >
(cfg->>'job_wall_time_us')::numeric
THEN RETURN false; END IF;
FOR preparing, phase IN SELECT true,cfg->'compile_limits' UNION ALL
SELECT false,cfg->'run_limits' LOOP
IF lang = 'python313' AND ((phase->>'cpu_time_us')::numeric < 10000 OR
(phase->>'wall_time_us')::numeric < 50000 OR
(phase->>'memory_bytes')::numeric < 33554432 OR
(phase->>'open_files')::numeric < 16) THEN RETURN false; END IF;
IF lang = 'java21' AND ((phase->>'cpu_time_us')::numeric <
CASE WHEN preparing THEN 1000000 ELSE 200000 END OR
(phase->>'wall_time_us')::numeric <
CASE WHEN preparing THEN 3000000 ELSE 1000000 END OR
(phase->>'memory_bytes')::numeric <
CASE WHEN preparing THEN 402653184 ELSE 268435456 END OR
(phase->>'processes')::numeric < 16 OR
(phase->>'open_files')::numeric < 32 OR
(phase->>'stack_bytes')::numeric < 262144 OR
(phase->>'memory_bytes')::numeric - 167772160 -
(phase->>'processes')::numeric * 262144 < 33554432)
THEN RETURN false; END IF;
END LOOP;
END LOOP;
RETURN true;
END $$
""")
op.execute("REVOKE ALL ON FUNCTION public.kf_valid_runtime_limits(jsonb) FROM PUBLIC")
op.execute(
"REVOKE ALL ON FUNCTION public.kf_valid_language_configurations(jsonb,jsonb) FROM PUBLIC"
)
op.execute("GRANT EXECUTE ON FUNCTION public.kf_valid_runtime_limits(jsonb) TO kf_oj_platform")
op.execute(
"GRANT EXECUTE ON FUNCTION public.kf_valid_language_configurations(jsonb,jsonb) "
"TO kf_oj_platform"
)
op.create_check_constraint(
op.f("ck_problem_versions_judging_schema"),
"problem_versions",
"judging_schema_version IN (1,2)",
)
op.create_check_constraint(
op.f("ck_problem_versions_language_configurations"), "problem_versions", CONFIGURATION_CHECK
)
op.execute("""
CREATE FUNCTION public.kf_submission_language() RETURNS trigger LANGUAGE plpgsql
SET search_path = pg_catalog, public AS $$
DECLARE supported jsonb;
BEGIN
SELECT languages INTO supported FROM public.problem_versions
WHERE id=NEW.problem_version_id;
IF FOUND AND NOT supported @> jsonb_build_array(NEW.language) THEN
RAISE EXCEPTION 'submission language not supported by version'
USING ERRCODE = '23514';
END IF;
RETURN NEW;
END $$
""")
op.execute("REVOKE ALL ON FUNCTION public.kf_submission_language() FROM PUBLIC")
op.execute("""
CREATE TRIGGER kf_submission_language BEFORE INSERT OR UPDATE OF
language,problem_version_id ON public.submissions
FOR EACH ROW EXECUTE FUNCTION public.kf_submission_language()
""")
def downgrade() -> None:
# Losing a runtime configuration is not a rollback strategy. Preserve all user data.
op.execute("""
DO $$ BEGIN
IF EXISTS (SELECT 1 FROM problem_versions WHERE judging_schema_version = 2) OR
EXISTS (SELECT 1 FROM submissions WHERE language NOT IN ('c17','cpp20')) THEN
RAISE EXCEPTION 'cannot downgrade while multi-runtime data exists'
USING ERRCODE = '23514';
END IF;
END $$
""")
op.drop_constraint(
op.f("ck_problem_versions_language_configurations"), "problem_versions", type_="check"
)
op.drop_constraint(
op.f("ck_problem_versions_judging_schema"), "problem_versions", type_="check"
)
op.execute("DROP TRIGGER kf_submission_language ON public.submissions")
op.execute("DROP FUNCTION public.kf_submission_language()")
op.execute("DROP FUNCTION public.kf_valid_language_configurations(jsonb,jsonb)")
op.execute("DROP FUNCTION public.kf_valid_runtime_limits(jsonb)")
op.drop_column("problem_versions", "language_configurations")
op.drop_column("problem_versions", "judging_schema_version")
for column, kind in (
("toolchain", JSONB()),
("compile_limits", JSONB()),
("run_limits", JSONB()),
("job_wall_time_us", sa.BigInteger()),
):
op.alter_column("problem_versions", column, existing_type=kind, nullable=False)
op.drop_constraint(op.f("ck_submissions_language"), "submissions", type_="check")
op.create_check_constraint(
op.f("ck_submissions_language"), "submissions", "language IN ('c17','cpp20')"
)
op.alter_column(
"submissions",
"language",
existing_type=sa.String(16),
type_=sa.String(8),
existing_nullable=False,
)
backend/migrations/versions/0003_submission_performance.py
-119
View File
@@ -1,119 +0,0 @@
"""Trusted final performance projection; no re-encoding or promotion of old reports."""
import sqlalchemy as sa
from alembic import op
revision = "0003_submission_performance"
down_revision = "0002_multi_runtime"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("judge_attempts", sa.Column("measurement_epoch", sa.String(64), nullable=True))
op.create_check_constraint(
op.f("ck_judge_attempts_measurement_epoch"),
"judge_attempts",
"measurement_epoch IS NULL OR measurement_epoch ~ '^[0-9a-f]{64}$'",
)
op.create_table(
"evaluation_performance",
sa.Column("evaluation_id", sa.Uuid(), primary_key=True),
sa.Column("submission_id", sa.Uuid(), nullable=False, unique=True),
sa.Column("attempt_id", sa.Uuid(), nullable=False, unique=True),
sa.Column("problem_version_id", sa.Uuid(), nullable=False),
sa.Column("language", sa.String(16), nullable=False),
sa.Column("comparison_group", sa.String(64), nullable=False),
sa.Column("measurement_epoch", sa.String(64), nullable=False),
sa.Column("report_digest", sa.String(64), nullable=False),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("cpu_time_us", sa.BigInteger(), nullable=True),
sa.Column("memory_peak_bytes", sa.BigInteger(), nullable=True),
sa.ForeignKeyConstraint(
["evaluation_id", "problem_version_id"],
["evaluations.id", "evaluations.problem_version_id"],
),
sa.ForeignKeyConstraint(
["submission_id", "problem_version_id"],
["submissions.id", "submissions.problem_version_id"],
),
sa.ForeignKeyConstraint(
["attempt_id", "evaluation_id"], ["judge_attempts.id", "judge_attempts.evaluation_id"]
),
sa.CheckConstraint("language IN ('c17','cpp20','python313','java21')", name="language"),
sa.CheckConstraint("comparison_group ~ '^[0-9a-f]{64}$'", name="comparison_group"),
sa.CheckConstraint("measurement_epoch ~ '^[0-9a-f]{64}$'", name="measurement_epoch"),
sa.CheckConstraint("report_digest ~ '^[0-9a-f]{64}$'", name="report_digest"),
sa.CheckConstraint("cpu_time_us IS NULL OR cpu_time_us >= 0", name="cpu"),
sa.CheckConstraint("memory_peak_bytes IS NULL OR memory_peak_bytes >= 0", name="memory"),
)
for metric in ("cpu_time_us", "memory_peak_bytes"):
op.create_index(
f"ix_performance_{metric}_cohort",
"evaluation_performance",
["comparison_group", sa.text("finished_at DESC"), sa.text("evaluation_id DESC")],
postgresql_where=sa.text(f"{metric} IS NOT NULL"),
)
op.create_index(
"ix_submissions_owned_problem_history",
"submissions",
["account_id", "problem_version_id", sa.text("created_at DESC"), sa.text("id DESC")],
)
op.execute("""
CREATE FUNCTION public.kf_trusted_performance() RETURNS trigger LANGUAGE plpgsql
SET search_path = pg_catalog, public AS $$
BEGIN
IF TG_OP != 'INSERT' THEN
RAISE EXCEPTION 'final performance is immutable' USING ERRCODE = '23514';
END IF;
IF NOT EXISTS (
SELECT 1 FROM public.evaluations e
JOIN public.judge_attempts a ON a.id=e.current_attempt_id AND a.evaluation_id=e.id
JOIN public.submissions s ON s.id=e.submission_id
WHERE e.id=NEW.evaluation_id AND e.state='finished' AND e.verdict='accepted'
AND e.failure_code IS NULL AND a.cleanup_complete IS TRUE
AND e.current_attempt_id=NEW.attempt_id AND s.id=NEW.submission_id
AND s.problem_version_id=NEW.problem_version_id AND s.language=NEW.language
AND a.measurement_epoch=NEW.measurement_epoch
AND e.final_report_digest=NEW.report_digest AND a.report_digest=NEW.report_digest
AND e.updated_at=NEW.finished_at
AND e.final_report->>'cleanup_complete'='true'
AND (e.final_report->>'run_cpu_time_us')::bigint
IS NOT DISTINCT FROM NEW.cpu_time_us
AND (e.final_report->>'run_memory_peak_bytes')::bigint
IS NOT DISTINCT FROM NEW.memory_peak_bytes
) THEN
RAISE EXCEPTION 'performance requires a bound trusted final AC'
USING ERRCODE = '23514';
END IF;
RETURN NEW;
END $$
""")
op.execute("REVOKE ALL ON FUNCTION public.kf_trusted_performance() FROM PUBLIC")
op.execute("""
CREATE TRIGGER kf_trusted_performance BEFORE INSERT OR UPDATE OR DELETE
ON public.evaluation_performance FOR EACH ROW
EXECUTE FUNCTION public.kf_trusted_performance()
""")
op.execute("REVOKE ALL ON evaluation_performance FROM PUBLIC,kf_oj_platform,kf_oj_coordinator")
op.execute("GRANT SELECT ON evaluation_performance TO kf_oj_platform,kf_oj_coordinator")
op.execute("GRANT INSERT ON evaluation_performance TO kf_oj_coordinator")
# Existing attempt INSERT permission covers the new column. No UPDATE epoch permission:
# an unbound historical attempt must never be promoted by changing current configuration.
def downgrade() -> None:
op.execute("""
DO $$ BEGIN
IF EXISTS(SELECT 1 FROM evaluation_performance) OR
EXISTS(SELECT 1 FROM judge_attempts WHERE measurement_epoch IS NOT NULL) THEN
RAISE EXCEPTION 'cannot discard trusted measurement bindings'
USING ERRCODE = '23514';
END IF;
END $$
""")
op.drop_table("evaluation_performance")
op.execute("DROP FUNCTION public.kf_trusted_performance()")
op.drop_index("ix_submissions_owned_problem_history", table_name="submissions")
op.drop_constraint(op.f("ck_judge_attempts_measurement_epoch"), "judge_attempts", type_="check")
op.drop_column("judge_attempts", "measurement_epoch")
backend/migrations/versions/0004_performance_watermark.py
-21
View File
@@ -1,21 +0,0 @@
"""Indexed group watermark for bounded aggregate cache invalidation."""
import sqlalchemy as sa
from alembic import op
revision = "0004_performance_watermark"
down_revision = "0003_submission_performance"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_index(
"ix_performance_group_watermark",
"evaluation_performance",
["comparison_group", sa.text("finished_at DESC"), sa.text("evaluation_id DESC")],
)
def downgrade() -> None:
op.drop_index("ix_performance_group_watermark", table_name="evaluation_performance")
backend/migrations/versions/0005_identity_authority.py
-150
View File
@@ -1,150 +0,0 @@
"""Default-denied administrator authority and separate local operator/import roles."""
import sqlalchemy as sa
from alembic import op
revision = "0005_identity_authority"
down_revision = "0004_performance_watermark"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
"accounts",
sa.Column("is_administrator", sa.Boolean(), server_default="false", nullable=False),
)
op.add_column(
"accounts",
sa.Column("authorization_version", sa.BigInteger(), server_default="1", nullable=False),
)
op.create_check_constraint(
op.f("ck_accounts_authorization_version"), "accounts", "authorization_version >= 1"
)
op.create_index(
"ix_accounts_active_administrators",
"accounts",
["id"],
postgresql_where=sa.text("is_administrator AND enabled"),
)
# Logins and secrets are provisioned separately by an operator. Neither group
# may inherit an existing privileged role or quietly reuse a privileged login.
for role in ("kf_oj_identity_operator", "kf_oj_problem_import"):
op.execute(f"""
DO $$ BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{role}') THEN
CREATE ROLE {role} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE
NOREPLICATION NOBYPASSRLS;
ELSIF EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{role}' AND
(rolcanlogin OR rolsuper OR rolcreatedb OR rolcreaterole
OR rolreplication OR rolbypassrls))
OR EXISTS (SELECT 1 FROM pg_auth_members m JOIN pg_roles r ON r.oid=m.member
WHERE r.rolname='{role}') THEN
RAISE EXCEPTION 'unsafe existing authority or import role';
END IF;
END $$
""")
op.execute(
"REVOKE ALL ON ALL TABLES IN SCHEMA public "
"FROM kf_oj_identity_operator,kf_oj_problem_import"
)
# Table REVOKE does not remove explicit column grants. Remove those too,
# including grants left by an earlier downgrade or manual role provisioning.
op.execute("""
DO $$ DECLARE t record; BEGIN
FOR t IN SELECT c.relname, string_agg(quote_ident(a.attname),',' ORDER BY a.attnum) cols
FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
JOIN pg_attribute a ON a.attrelid=c.oid
WHERE n.nspname='public' AND c.relkind IN ('r','p','v','f')
AND a.attnum>0 AND NOT a.attisdropped
GROUP BY c.oid,c.relname LOOP
EXECUTE format('REVOKE SELECT (%s),INSERT (%s),UPDATE (%s),REFERENCES (%s) '
'ON TABLE public.%I FROM kf_oj_identity_operator,kf_oj_problem_import',
t.cols,t.cols,t.cols,t.cols,t.relname);
END LOOP;
END $$
""")
op.execute("GRANT USAGE ON SCHEMA public TO kf_oj_identity_operator,kf_oj_problem_import")
op.execute("REVOKE INSERT,UPDATE,DELETE ON accounts FROM kf_oj_platform")
op.execute(
"REVOKE INSERT(id,username,password_hash,enabled,created_at,"
"is_administrator,authorization_version),"
"UPDATE(id,username,password_hash,enabled,created_at,"
"is_administrator,authorization_version) "
"ON accounts FROM kf_oj_platform"
)
op.execute(
"GRANT INSERT(id,username,password_hash,enabled,created_at) ON accounts TO kf_oj_platform"
)
op.execute("GRANT UPDATE(enabled) ON accounts TO kf_oj_platform")
op.execute("GRANT SELECT ON accounts,sessions TO kf_oj_identity_operator")
op.execute("GRANT UPDATE(is_administrator,enabled) ON accounts TO kf_oj_identity_operator")
op.execute("GRANT UPDATE(revoked_at) ON sessions TO kf_oj_identity_operator")
op.execute("REVOKE ALL ON accounts,sessions FROM kf_oj_coordinator,kf_oj_problem_import")
# No import queue exists yet. Its role deliberately has no table privileges
# until the import migration grants only the queue/staging tables it needs.
op.execute("""
CREATE FUNCTION public.kf_account_authority_version() RETURNS trigger LANGUAGE plpgsql
SET search_path = pg_catalog, public AS $$
BEGIN
IF NEW.authorization_version IS DISTINCT FROM OLD.authorization_version THEN
RAISE EXCEPTION 'authorization version is server derived' USING ERRCODE='23514';
END IF;
IF OLD.is_administrator AND NEW.enabled IS DISTINCT FROM OLD.enabled
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','USAGE') THEN
RAISE EXCEPTION 'administrator enablement requires the local operator'
USING ERRCODE='42501';
END IF;
IF NEW.is_administrator IS DISTINCT FROM OLD.is_administrator
OR NEW.enabled IS DISTINCT FROM OLD.enabled THEN
NEW.authorization_version := OLD.authorization_version + 1;
END IF;
RETURN NEW;
END $$
""")
op.execute("""
CREATE FUNCTION public.kf_revoke_authority_sessions() RETURNS trigger LANGUAGE plpgsql
SET search_path = pg_catalog, public AS $$
BEGIN
IF NEW.authorization_version != OLD.authorization_version THEN
UPDATE public.sessions SET revoked_at=coalesce(revoked_at,clock_timestamp())
WHERE account_id=NEW.id AND revoked_at IS NULL;
END IF;
RETURN NEW;
END $$
""")
for function in ("kf_account_authority_version", "kf_revoke_authority_sessions"):
op.execute(f"REVOKE ALL ON FUNCTION public.{function}() FROM PUBLIC")
op.execute("""
CREATE TRIGGER kf_account_authority_version BEFORE UPDATE ON public.accounts
FOR EACH ROW EXECUTE FUNCTION public.kf_account_authority_version()
""")
op.execute("""
CREATE TRIGGER kf_revoke_authority_sessions AFTER UPDATE ON public.accounts
FOR EACH ROW EXECUTE FUNCTION public.kf_revoke_authority_sessions()
""")
def downgrade() -> None:
op.execute("""
DO $$ BEGIN
IF EXISTS(SELECT 1 FROM accounts
WHERE is_administrator OR authorization_version != 1) THEN
RAISE EXCEPTION 'cannot discard authority or revocation history'
USING ERRCODE='23514';
END IF;
END $$
""")
for name in ("kf_revoke_authority_sessions", "kf_account_authority_version"):
op.execute(f"DROP TRIGGER {name} ON public.accounts")
op.execute(f"DROP FUNCTION public.{name}()")
op.execute("REVOKE ALL ON accounts,sessions FROM kf_oj_identity_operator")
op.execute("REVOKE UPDATE(is_administrator,enabled) ON accounts FROM kf_oj_identity_operator")
op.execute("REVOKE UPDATE(revoked_at) ON sessions FROM kf_oj_identity_operator")
op.drop_index("ix_accounts_active_administrators", table_name="accounts")
op.drop_constraint(op.f("ck_accounts_authorization_version"), "accounts", type_="check")
op.drop_column("accounts", "authorization_version")
op.drop_column("accounts", "is_administrator")
op.execute("GRANT INSERT,UPDATE,DELETE ON accounts TO kf_oj_platform")
# Shared NOLOGIN groups survive downgrade, as in the foundation migration;
# removing a cluster role could break another database using the same groups.
backend/migrations/versions/0006_authority_operations.py
-226
View File
@@ -1,226 +0,0 @@
"""Fixed local authority operations and serialized last-administrator protection."""
import sqlalchemy as sa
from alembic import op
revision = "0006_authority_operations"
down_revision = "0005_identity_authority"
branch_labels = None
depends_on = None
OWNER = "kf_oj_identity_authority_owner"
def upgrade() -> None:
op.execute(f"""
DO $$ BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{OWNER}') THEN
CREATE ROLE {OWNER} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE
NOREPLICATION NOBYPASSRLS;
ELSIF EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{OWNER}' AND
(rolcanlogin OR rolsuper OR rolcreatedb OR rolcreaterole
OR rolreplication OR rolbypassrls))
OR EXISTS (SELECT 1 FROM pg_auth_members m JOIN pg_roles r ON r.oid=m.member
WHERE r.rolname='{OWNER}') THEN
RAISE EXCEPTION 'unsafe existing authority function owner';
END IF;
END $$
""")
op.create_table(
"identity_authority_guard",
sa.Column("id", sa.SmallInteger(), primary_key=True),
sa.Column("generation", sa.BigInteger(), nullable=False, server_default="0"),
sa.CheckConstraint("id = 1", name="singleton"),
sa.CheckConstraint("generation >= 0", name="generation"),
)
op.execute("INSERT INTO public.identity_authority_guard(id) VALUES(1)")
for role in (OWNER, "kf_oj_identity_operator"):
op.execute(f"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM {role}")
op.execute(f"""
REVOKE SELECT(id,username,password_hash,enabled,created_at,
is_administrator,authorization_version),
UPDATE(id,username,password_hash,enabled,created_at,
is_administrator,authorization_version) ON public.accounts FROM {role}
""")
op.execute(f"""
REVOKE SELECT(token_hash,account_id,csrf_hash,expires_at,revoked_at,created_at),
UPDATE(token_hash,account_id,csrf_hash,expires_at,revoked_at,created_at)
ON public.sessions FROM {role}
""")
op.execute(f"GRANT USAGE ON SCHEMA public TO {OWNER}")
op.execute(f"""
GRANT SELECT(id,username,enabled,is_administrator,authorization_version),
UPDATE(enabled,is_administrator) ON public.accounts TO {OWNER}
""")
op.execute(f"GRANT SELECT(account_id,revoked_at),UPDATE(revoked_at) ON sessions TO {OWNER}")
op.execute(f"GRANT SELECT,UPDATE(generation) ON identity_authority_guard TO {OWNER}")
# Every account write obtains the guard BEFORE it can lock an account row.
# Updating (rather than merely locking) the singleton also makes an old
# REPEATABLE READ/SERIALIZABLE snapshot fail safely with a serialization error.
op.execute("""
CREATE FUNCTION public.kf_serialize_account_authority() RETURNS trigger
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
BEGIN
UPDATE public.identity_authority_guard SET generation=generation+1 WHERE id=1;
IF NOT FOUND THEN
RAISE EXCEPTION 'authority guard unavailable' USING ERRCODE='23514';
END IF;
RETURN NULL;
END $$
""")
op.execute("""
CREATE FUNCTION public.kf_protect_last_administrator() RETURNS trigger
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
BEGIN
IF OLD.is_administrator AND OLD.enabled AND
(TG_OP='DELETE' OR NOT (NEW.is_administrator AND NEW.enabled)) AND
(SELECT count(*) FROM public.accounts WHERE is_administrator AND enabled) <= 1
THEN
RAISE EXCEPTION 'last enabled administrator is protected' USING ERRCODE='23514';
END IF;
IF TG_OP='DELETE' THEN RETURN OLD; END IF;
RETURN NEW;
END $$
""")
op.execute("""
CREATE OR REPLACE FUNCTION public.kf_account_authority_version() RETURNS trigger
LANGUAGE plpgsql SET search_path = pg_catalog, public AS $$
BEGIN
IF NEW.authorization_version IS DISTINCT FROM OLD.authorization_version THEN
RAISE EXCEPTION 'authorization version is server derived' USING ERRCODE='23514';
END IF;
IF OLD.is_administrator AND NEW.enabled IS DISTINCT FROM OLD.enabled
AND current_user != 'kf_oj_identity_authority_owner'
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','USAGE') THEN
RAISE EXCEPTION 'administrator enablement requires the local operator'
USING ERRCODE='42501';
END IF;
IF NEW.is_administrator IS DISTINCT FROM OLD.is_administrator
OR NEW.enabled IS DISTINCT FROM OLD.enabled THEN
NEW.authorization_version := OLD.authorization_version + 1;
END IF;
RETURN NEW;
END $$
""")
op.execute("""
CREATE FUNCTION public.kf_manage_account_authority(p_username text,p_operation text)
RETURNS TABLE(account_id uuid,username text,enabled boolean,is_administrator boolean,
authorization_version bigint,changed boolean)
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
DECLARE target record; desired_admin boolean; desired_enabled boolean; active bigint;
BEGIN
IF p_username IS NULL OR p_username !~ '^[a-z0-9_]{3,24}$' OR
p_operation IS NULL OR
p_operation NOT IN ('bootstrap','grant','revoke','enable','disable') THEN
RAISE EXCEPTION 'invalid authority operation' USING ERRCODE='22023';
END IF;
UPDATE public.identity_authority_guard g SET generation=g.generation+1 WHERE g.id=1;
IF NOT FOUND THEN
RAISE EXCEPTION 'authority guard unavailable' USING ERRCODE='23514';
END IF;
SELECT a.id,a.username,a.enabled,a.is_administrator,a.authorization_version
INTO target FROM public.accounts a WHERE a.username=p_username FOR UPDATE;
IF NOT FOUND THEN
RAISE EXCEPTION 'account not found' USING ERRCODE='P0002';
END IF;
desired_admin:=target.is_administrator; desired_enabled:=target.enabled;
SELECT count(*) INTO active FROM public.accounts a
WHERE a.is_administrator AND a.enabled;
IF p_operation='bootstrap' THEN
IF active>0 AND NOT (target.is_administrator AND target.enabled) THEN
RAISE EXCEPTION 'bootstrap requires no enabled administrator'
USING ERRCODE='23514';
END IF;
desired_admin:=true; desired_enabled:=true;
ELSIF p_operation='grant' THEN
IF active=0 THEN
RAISE EXCEPTION 'first administrator requires explicit bootstrap'
USING ERRCODE='23514';
END IF;
IF NOT target.enabled THEN
RAISE EXCEPTION 'enable the account before granting authority'
USING ERRCODE='23514';
END IF;
desired_admin:=true;
ELSIF p_operation='revoke' THEN desired_admin:=false;
ELSIF p_operation='enable' THEN desired_enabled:=true;
ELSE desired_enabled:=false;
END IF;
changed:=desired_admin IS DISTINCT FROM target.is_administrator OR
desired_enabled IS DISTINCT FROM target.enabled;
IF changed THEN
UPDATE public.accounts a SET enabled=desired_enabled,is_administrator=desired_admin
WHERE a.id=target.id;
END IF;
RETURN QUERY SELECT a.id,a.username::text,a.enabled,a.is_administrator,
a.authorization_version,changed FROM public.accounts a WHERE a.id=target.id;
END $$
""")
# A narrow NOLOGIN owner holds only these column privileges. The operator
# receives EXECUTE, never table writes or access to password/session digests.
op.execute(f"GRANT CREATE ON SCHEMA public TO {OWNER}")
for signature in (
"kf_serialize_account_authority()",
"kf_protect_last_administrator()",
"kf_manage_account_authority(text,text)",
):
op.execute(f"ALTER FUNCTION public.{signature} OWNER TO {OWNER}")
op.execute(f"REVOKE ALL ON FUNCTION public.{signature} FROM PUBLIC")
op.execute(f"REVOKE CREATE ON SCHEMA public FROM {OWNER}")
op.execute("""
GRANT EXECUTE ON FUNCTION public.kf_manage_account_authority(text,text)
TO kf_oj_identity_operator
""")
op.execute("""
CREATE TRIGGER kf_serialize_account_authority BEFORE INSERT OR UPDATE OR DELETE
ON public.accounts FOR EACH STATEMENT
EXECUTE FUNCTION public.kf_serialize_account_authority()
""")
op.execute("""
CREATE TRIGGER kf_protect_last_administrator BEFORE UPDATE OR DELETE
ON public.accounts FOR EACH ROW EXECUTE FUNCTION public.kf_protect_last_administrator()
""")
def downgrade() -> None:
op.execute("""
DO $$ BEGIN
IF EXISTS(SELECT 1 FROM public.accounts WHERE is_administrator) THEN
RAISE EXCEPTION 'cannot remove active authority safeguards' USING ERRCODE='23514';
END IF;
END $$
""")
for name in ("kf_protect_last_administrator", "kf_serialize_account_authority"):
op.execute(f"DROP TRIGGER {name} ON public.accounts")
op.execute(f"DROP FUNCTION public.{name}()")
op.execute("DROP FUNCTION public.kf_manage_account_authority(text,text)")
op.drop_table("identity_authority_guard")
op.execute(f"REVOKE ALL ON accounts,sessions FROM {OWNER}")
op.execute(f"""
REVOKE SELECT(id,username,enabled,is_administrator,authorization_version),
UPDATE(enabled,is_administrator) ON accounts FROM {OWNER}
""")
op.execute(f"REVOKE SELECT(account_id,revoked_at),UPDATE(revoked_at) ON sessions FROM {OWNER}")
op.execute("GRANT SELECT ON accounts,sessions TO kf_oj_identity_operator")
op.execute("GRANT UPDATE(is_administrator,enabled) ON accounts TO kf_oj_identity_operator")
op.execute("GRANT UPDATE(revoked_at) ON sessions TO kf_oj_identity_operator")
# Preserve the v0005 trigger policy without referring to the function owner.
op.execute("""
CREATE OR REPLACE FUNCTION public.kf_account_authority_version() RETURNS trigger
LANGUAGE plpgsql SET search_path = pg_catalog, public AS $$
BEGIN
IF NEW.authorization_version IS DISTINCT FROM OLD.authorization_version THEN
RAISE EXCEPTION 'authorization version is server derived' USING ERRCODE='23514';
END IF;
IF OLD.is_administrator AND NEW.enabled IS DISTINCT FROM OLD.enabled
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','USAGE') THEN
RAISE EXCEPTION 'administrator enablement requires the local operator'
USING ERRCODE='42501';
END IF;
IF NEW.is_administrator IS DISTINCT FROM OLD.is_administrator
OR NEW.enabled IS DISTINCT FROM OLD.enabled THEN
NEW.authorization_version := OLD.authorization_version + 1;
END IF;
RETURN NEW;
END $$
""")
backend/migrations/versions/0007_problem_imports.py
-212
View File
@@ -1,212 +0,0 @@
"""Durable imports/candidates, append-only events and accepted-authority worker port."""
import sqlalchemy as sa
from alembic import op
from kf_oj.modules.operations.models import MaintenanceEvent
from kf_oj.modules.problems.import_models import (
CandidateCase,
ImportPool,
ProblemCandidate,
ProblemImportOperation,
)
revision = "0007_problem_imports"
down_revision = "0006_authority_operations"
branch_labels = depends_on = None
def sql(script):
# asyncpg prepares one statement at a time; dollar-quoted bodies stay intact.
buffer, quote, dollar, i = [], False, False, 0
while i < len(script):
if not quote and script[i : i + 2] == "$$":
dollar = not dollar
buffer.append("$$")
i += 2
continue
character = script[i]
if not dollar and character == "'":
if quote and script[i : i + 2] == "''":
buffer.append("''")
i += 2
continue
quote = not quote
if character == ";" and not dollar and not quote:
if "".join(buffer).strip():
op.execute("".join(buffer))
buffer = []
else:
buffer.append(character)
i += 1
if "".join(buffer).strip():
op.execute("".join(buffer))
def upgrade():
connection = op.get_bind()
# Cyclic candidate/operation link is installed only after both tables exist.
for model in (
ImportPool,
ProblemImportOperation,
ProblemCandidate,
CandidateCase,
):
model.__table__.create(connection)
# Freeze the historical shape. A later operations model must not cause a
# fresh database to skip ahead and conflict with a subsequent ALTER migration.
op.create_table(
"problem_maintenance_events",
sa.Column("id", sa.Uuid(), primary_key=True),
sa.Column(
"created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
),
sa.Column("actor_id", sa.Uuid(), sa.ForeignKey("accounts.id"), nullable=False),
sa.Column("authorization_version", sa.BigInteger(), nullable=False),
sa.Column("operation_id", sa.Uuid(), nullable=False),
sa.Column("action", sa.String(24), nullable=False),
sa.Column("target_id", sa.Uuid()),
sa.CheckConstraint("authorization_version > 0", name="authority"),
sa.CheckConstraint(
"action IN ('upload_created','upload_received','import_ready','import_failed',"
"'import_discarded','import_expired','published','disabled','restored')",
name="action",
),
)
op.create_index(
"ix_problem_maintenance_events_page", "problem_maintenance_events", ["created_at", "id"]
)
op.create_foreign_key(
"fk_problem_import_candidate",
"problem_import_operations",
"problem_candidates",
["candidate_id"],
["id"],
)
op.execute("INSERT INTO problem_import_pool(id,reserved_bytes) VALUES(1,0)")
sql("""
CREATE FUNCTION public.kf_lock_reserved_maintenance(actor uuid, epoch bigint)
RETURNS void LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$
DECLARE permitted boolean;
BEGIN
SELECT enabled AND is_administrator AND authorization_version=epoch
INTO permitted FROM public.accounts WHERE id=actor FOR SHARE;
IF permitted IS DISTINCT FROM true THEN
RAISE EXCEPTION 'problem maintenance required' USING ERRCODE='42501';
END IF;
END $$;
GRANT CREATE ON SCHEMA public TO kf_oj_identity_authority_owner;
ALTER FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
OWNER TO kf_oj_identity_authority_owner;
REVOKE CREATE ON SCHEMA public FROM kf_oj_identity_authority_owner;
REVOKE ALL ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint) FROM PUBLIC;
GRANT EXECUTE ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
TO kf_oj_problem_import;
GRANT SELECT,INSERT,UPDATE ON problem_import_operations TO kf_oj_platform;
GRANT SELECT,UPDATE ON problem_import_pool TO kf_oj_platform,kf_oj_problem_import;
GRANT SELECT ON problem_candidates,problem_candidate_cases TO kf_oj_platform;
GRANT SELECT ON problem_import_operations TO kf_oj_problem_import;
GRANT UPDATE(state,updated_at,lease_until,attempts,fence,
reserved_bytes,failure_code,candidate_id)
ON problem_import_operations TO kf_oj_problem_import;
GRANT SELECT,INSERT,DELETE ON problem_candidates,problem_candidate_cases
TO kf_oj_problem_import;
GRANT SELECT,INSERT ON artifacts TO kf_oj_problem_import;
GRANT INSERT ON problem_maintenance_events TO kf_oj_platform,kf_oj_problem_import;
GRANT SELECT ON problem_maintenance_events TO kf_oj_platform;
""")
sql("""
CREATE FUNCTION public.kf_immutable_import_candidate() RETURNS trigger
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
DECLARE candidate uuid; current_state text;
BEGIN
IF TG_TABLE_NAME='problem_candidates' THEN candidate=OLD.id;
ELSE candidate=OLD.candidate_id; END IF;
SELECT o.state INTO current_state FROM public.problem_candidates c
JOIN public.problem_import_operations o ON o.id=c.operation_id WHERE c.id=candidate;
IF TG_OP='UPDATE' OR current_state NOT IN ('discarded','expired','failed') THEN
RAISE EXCEPTION 'immutable candidate' USING ERRCODE='23514';
END IF;
RETURN OLD;
END $$;
CREATE TRIGGER immutable_candidate BEFORE UPDATE OR DELETE ON problem_candidates
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
CREATE TRIGGER immutable_candidate_case BEFORE UPDATE OR DELETE ON problem_candidate_cases
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
CREATE FUNCTION public.kf_insert_import_candidate() RETURNS trigger
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
DECLARE operation uuid; current_state text;
BEGIN
IF TG_TABLE_NAME='problem_candidates' THEN operation=NEW.operation_id;
ELSE SELECT operation_id INTO operation FROM public.problem_candidates
WHERE id=NEW.candidate_id; END IF;
SELECT state INTO current_state FROM public.problem_import_operations
WHERE id=operation FOR SHARE;
IF current_state IS DISTINCT FROM 'validating' THEN
RAISE EXCEPTION 'candidate is not being validated' USING ERRCODE='23514';
END IF; RETURN NEW;
END $$;
CREATE TRIGGER insert_candidate BEFORE INSERT ON problem_candidates
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
CREATE TRIGGER insert_candidate_case BEFORE INSERT ON problem_candidate_cases
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
CREATE FUNCTION public.kf_fixed_import_receipt() RETURNS trigger
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ BEGIN
IF NEW.id IS DISTINCT FROM OLD.id OR NEW.owner_id IS DISTINCT FROM OLD.owner_id
OR NEW.authorization_version IS DISTINCT FROM OLD.authorization_version
OR NEW.archive_digest IS DISTINCT FROM OLD.archive_digest
OR NEW.archive_bytes IS DISTINCT FROM OLD.archive_bytes
OR NEW.created_at IS DISTINCT FROM OLD.created_at
OR NEW.expires_at IS DISTINCT FROM OLD.expires_at THEN
RAISE EXCEPTION 'immutable import receipt' USING ERRCODE='23514';
END IF;
IF OLD.state IN ('ready','published') AND NEW.state IN ('ready','published')
AND NEW.candidate_id IS DISTINCT FROM OLD.candidate_id THEN
RAISE EXCEPTION 'immutable completed candidate receipt' USING ERRCODE='23514';
END IF;
IF OLD.state='published' AND (NEW.state IS DISTINCT FROM OLD.state
OR NEW.published_version_id IS DISTINCT FROM OLD.published_version_id) THEN
RAISE EXCEPTION 'immutable published import receipt' USING ERRCODE='23514';
END IF;
IF NEW.state='ready' THEN
IF (OLD.state IS DISTINCT FROM 'ready' AND
(NOT pg_has_role(current_user,'kf_oj_problem_import','USAGE')
OR OLD.state!='validating')) OR NEW.candidate_id IS NULL
OR NOT EXISTS(SELECT 1 FROM public.problem_candidates c
WHERE c.id=NEW.candidate_id AND c.operation_id=NEW.id
AND c.case_count=(SELECT count(*) FROM public.problem_candidate_cases b
WHERE b.candidate_id=c.id)
AND c.case_count-1=(SELECT max(position) FROM public.problem_candidate_cases b
WHERE b.candidate_id=c.id)) THEN
RAISE EXCEPTION 'invalid candidate completion' USING ERRCODE='23514';
END IF;
END IF; RETURN NEW;
END $$;
CREATE TRIGGER fixed_import_receipt BEFORE UPDATE ON problem_import_operations
FOR EACH ROW EXECUTE FUNCTION public.kf_fixed_import_receipt();
""")
def downgrade():
connection = op.get_bind()
if connection.exec_driver_sql(
"SELECT EXISTS(SELECT 1 FROM problem_import_operations)"
).scalar():
raise RuntimeError("cannot discard durable import history")
op.execute("DROP FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)")
op.drop_constraint(
"fk_problem_import_candidate", "problem_import_operations", type_="foreignkey"
)
for model in (
MaintenanceEvent,
CandidateCase,
ProblemCandidate,
ProblemImportOperation,
ImportPool,
):
model.__table__.drop(connection)
sql(
"DROP FUNCTION public.kf_immutable_import_candidate(); "
"DROP FUNCTION public.kf_fixed_import_receipt(); "
"DROP FUNCTION public.kf_insert_import_candidate()"
)
backend/migrations/versions/0008_problem_maintenance.py
-117
View File
@@ -1,117 +0,0 @@
"""Immutable maintenance receipts, explicit operator audit and read-only history."""
import sqlalchemy as sa
from alembic import op
from kf_oj.modules.problems.maintenance_models import ProblemMaintenanceOperation
revision = "0008_problem_maintenance"
down_revision = "0007_problem_imports"
branch_labels = depends_on = None
OWNER = "kf_oj_identity_authority_owner"
def upgrade():
ProblemMaintenanceOperation.__table__.create(op.get_bind())
op.execute("GRANT SELECT,INSERT ON problem_maintenance_operations TO kf_oj_platform")
op.add_column(
"problem_maintenance_events",
sa.Column("actor_kind", sa.String(8), nullable=False, server_default=sa.text("'account'")),
)
op.add_column("problem_maintenance_events", sa.Column("operator_name", sa.String(63)))
op.alter_column("problem_maintenance_events", "actor_id", nullable=True)
op.drop_constraint(op.f("ck_problem_maintenance_events_action"), "problem_maintenance_events")
op.create_check_constraint(
"action",
"problem_maintenance_events",
"action IN ('upload_created','upload_received','import_ready','import_failed',"
"'import_discarded','import_expired','published','disabled','restored',"
"'authority_bootstrap','authority_grant','authority_revoke','authority_enable','authority_disable')",
)
op.create_check_constraint(
"actor",
"problem_maintenance_events",
"(actor_kind='account' AND actor_id IS NOT NULL AND operator_name IS NULL) OR "
"(actor_kind='operator' AND actor_id IS NULL AND operator_name IS NOT NULL)",
)
op.execute("""CREATE FUNCTION public.kf_append_only_maintenance() RETURNS trigger
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ BEGIN
RAISE EXCEPTION 'append-only maintenance history' USING ERRCODE='23514'; END $$""")
for table in ("problem_maintenance_operations", "problem_maintenance_events"):
op.execute(f"""CREATE TRIGGER append_only_maintenance BEFORE UPDATE OR DELETE ON {table}
FOR EACH ROW EXECUTE FUNCTION public.kf_append_only_maintenance()""")
# Preserve the already-verified authority implementation; its only entry
# point becomes a fixed audited wrapper owned by the same narrow NOLOGIN role.
op.execute(
"ALTER FUNCTION public.kf_manage_account_authority(text,text) "
"RENAME TO kf_manage_account_authority_unlogged"
)
op.execute(
"REVOKE ALL ON FUNCTION public.kf_manage_account_authority_unlogged(text,text) "
"FROM PUBLIC,kf_oj_identity_operator"
)
op.execute(f"GRANT INSERT ON problem_maintenance_events TO {OWNER}")
op.execute("""
CREATE FUNCTION public.kf_manage_account_authority(p_username text,p_operation text)
RETURNS TABLE(account_id uuid,username text,enabled boolean,is_administrator boolean,
authorization_version bigint,changed boolean)
LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$
DECLARE result record; BEGIN
SELECT * INTO STRICT result FROM
public.kf_manage_account_authority_unlogged(p_username,p_operation);
IF result.changed THEN
INSERT INTO public.problem_maintenance_events
(id,created_at,actor_id,actor_kind,operator_name,authorization_version,
operation_id,action,target_id)
VALUES (gen_random_uuid(),clock_timestamp(),NULL,'operator',session_user,
result.authorization_version,gen_random_uuid(),'authority_' || p_operation,
result.account_id);
END IF;
RETURN QUERY SELECT result.account_id,result.username,result.enabled,
result.is_administrator,result.authorization_version,result.changed;
END $$""")
op.execute(f"GRANT CREATE ON SCHEMA public TO {OWNER}")
op.execute(f"ALTER FUNCTION public.kf_manage_account_authority(text,text) OWNER TO {OWNER}")
op.execute(f"REVOKE CREATE ON SCHEMA public FROM {OWNER}")
op.execute("REVOKE ALL ON FUNCTION public.kf_manage_account_authority(text,text) FROM PUBLIC")
op.execute(
"GRANT EXECUTE ON FUNCTION public.kf_manage_account_authority(text,text) "
"TO kf_oj_identity_operator"
)
def downgrade():
if (
op.get_bind()
.exec_driver_sql(
"SELECT EXISTS(SELECT 1 FROM problem_maintenance_operations) OR "
"EXISTS(SELECT 1 FROM problem_maintenance_events WHERE actor_kind='operator')"
)
.scalar()
):
raise RuntimeError("cannot discard committed maintenance history")
op.execute("DROP FUNCTION public.kf_manage_account_authority(text,text)")
op.execute(
"ALTER FUNCTION public.kf_manage_account_authority_unlogged(text,text) "
"RENAME TO kf_manage_account_authority"
)
op.execute(
"GRANT EXECUTE ON FUNCTION public.kf_manage_account_authority(text,text) "
"TO kf_oj_identity_operator"
)
op.execute(f"REVOKE INSERT ON problem_maintenance_events FROM {OWNER}")
for table in ("problem_maintenance_operations", "problem_maintenance_events"):
op.execute(f"DROP TRIGGER append_only_maintenance ON {table}")
op.execute("DROP FUNCTION public.kf_append_only_maintenance()")
ProblemMaintenanceOperation.__table__.drop(op.get_bind())
op.drop_constraint(op.f("ck_problem_maintenance_events_actor"), "problem_maintenance_events")
op.drop_constraint(op.f("ck_problem_maintenance_events_action"), "problem_maintenance_events")
op.create_check_constraint(
"action",
"problem_maintenance_events",
"action IN ('upload_created','upload_received','import_ready','import_failed',"
"'import_discarded','import_expired','published','disabled','restored')",
)
op.alter_column("problem_maintenance_events", "actor_id", nullable=False)
op.drop_column("problem_maintenance_events", "operator_name")
op.drop_column("problem_maintenance_events", "actor_kind")
backend/pyproject.toml
+1 -6
View File
@@ -17,15 +17,9 @@ dependencies = [
"asyncpg>=0.30,<1",
"alembic>=1.15,<2",
"aio-pika>=9.5,<10",
"argon2-cffi>=25.1,<26",
"httpx>=0.28,<1",
]
[project.scripts]
kf-problem-import-worker = "kf_oj.workers.problem_import:main"
kf-accounts = "kf_oj.modules.identity.cli:main"
kf-problems = "kf_oj.modules.problems.cli:main"
kf-artifacts = "kf_oj.infra.artifacts.cli:main"
kf-learning-worker = "kf_oj.workers.learning:main"
kf-outbox-publisher = "kf_oj.workers.outbox:main"
@@ -33,6 +27,7 @@ kf-outbox-publisher = "kf_oj.workers.outbox:main"
dev = [
"ruff>=0.11,<1",
"pytest>=8.3,<10",
"httpx>=0.28,<1",
"jsonschema>=4.23,<5",
]
backend/src/kf_oj/api/admin_dependencies.py
-26
View File
@@ -1,26 +0,0 @@
"""Uniform live session gates; domains still lock/recheck authority in write transactions."""
from typing import Annotated
from fastapi import Depends, HTTPException
from kf_oj.api.dependencies import current_principal, unsafe_principal
from kf_oj.modules.identity.public import Principal
async def maintenance_reader(
principal: Annotated[Principal, Depends(current_principal)],
) -> Principal:
if not principal.can_maintain_problems:
raise HTTPException(403, "problem maintenance required")
return principal
async def maintenance_writer(
principal: Annotated[Principal, Depends(unsafe_principal)],
) -> Principal:
# unsafe_principal already checks an explicit Origin and the live session's
# bound CSRF token. This capability check cannot replace the domain's port.
if not principal.can_maintain_problems:
raise HTTPException(403, "problem maintenance required")
return principal
backend/src/kf_oj/api/app.py
+2 -127
View File
@@ -2,37 +2,11 @@ import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.exceptions import RequestValidationError
from starlette.responses import JSONResponse
from kf_oj import __version__
from kf_oj.api.availability import AvailabilityMiddleware
from kf_oj.api.body_limits import BodyLimitMiddleware
from kf_oj.api.dependency_errors import install_handlers
from kf_oj.api.v1.admin_imports import router as import_router
from kf_oj.api.v1.admin_problems import router as maintenance_router
from kf_oj.api.v1.identity import router as identity_router
from kf_oj.api.v1.problems import router as problems_router
from kf_oj.api.v1.router import router
from kf_oj.api.v1.submissions import router as submission_router
from kf_oj.api.v1.worker_artifacts import router as worker_router
from kf_oj.core.logging import configure_logging
from kf_oj.core.settings import Settings
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.infra.artifacts.staging import ImportStaging
from kf_oj.infra.artifacts.store import ArtifactStore
from kf_oj.infra.coordinator import CoordinatorControl
from kf_oj.infra.db.session import build_engine, build_session_factory
from kf_oj.modules.identity.service import IdentityService
from kf_oj.modules.operations.queries import MaintenanceAuditQueries
from kf_oj.modules.operations.service import MaintenanceAudit
from kf_oj.modules.problems.imports import ProblemImports
from kf_oj.modules.problems.maintenance import ProblemMaintenance
from kf_oj.modules.problems.service import ProblemService
from kf_oj.modules.submissions.performance import SubmissionPerformanceQueries
from kf_oj.modules.submissions.queries import SubmissionQueries
from kf_oj.modules.submissions.service import SubmissionService
from kf_oj.modules.submissions.worker_access import WorkerAccess
def create_app(settings: Settings | None = None) -> FastAPI:
@@ -40,122 +14,23 @@ def create_app(settings: Settings | None = None) -> FastAPI:
@asynccontextmanager
async def lifespan(_: FastAPI):
configure_logging(settings.log_level, production=settings.environment == "production")
configure_logging(settings.log_level)
logger = logging.getLogger("kf_oj.api")
logger.info("Platform API started")
engine = None
store = None
staging = None
try:
if settings.database_url is not None:
engine = build_engine(settings)
app.state.sessions = build_session_factory(engine)
app.state.identity = IdentityService(
app.state.sessions, lifetime_seconds=settings.session_lifetime_seconds
)
app.state.submission_performance = SubmissionPerformanceQueries(app.state.sessions)
artifacts = None
if settings.artifact_root is not None:
store = ArtifactStore(
settings.artifact_root,
max_bytes=settings.artifact_max_bytes,
max_files=settings.artifact_max_files,
)
artifacts = ArtifactRepository(store)
if (
settings.import_staging_root is not None
and settings.judge_admission is not None
):
staging = ImportStaging(settings.import_staging_root)
app.state.problem_imports = ProblemImports(
app.state.sessions,
staging,
app.state.identity,
MaintenanceAudit(),
settings.judge_admission,
)
app.state.problem_maintenance = ProblemMaintenance(
app.state.sessions,
app.state.identity,
MaintenanceAudit(),
artifacts,
settings.judge_admission,
)
app.state.maintenance_audit = MaintenanceAuditQueries(
app.state.sessions, app.state.identity
)
app.state.problems = ProblemService(
app.state.sessions, artifacts, settings.judge_admission
)
if artifacts is not None:
app.state.submissions = SubmissionService(
app.state.sessions, app.state.problems, artifacts, settings.judge_admission
)
app.state.worker_access = WorkerAccess(
app.state.sessions, app.state.problems, artifacts
)
cancellation = (
CoordinatorControl(
settings.coordinator_url, settings.coordinator_token.get_secret_value()
)
if settings.coordinator_url and settings.coordinator_token
else None
)
app.state.submission_queries = SubmissionQueries(
app.state.sessions, artifacts, cancellation
)
yield
finally:
app.state.identity = None
app.state.problems = None
app.state.worker_access = None
app.state.submissions = None
app.state.submission_queries = None
app.state.submission_performance = None
app.state.problem_imports = None
app.state.problem_maintenance = None
app.state.maintenance_audit = None
if engine is not None:
await engine.dispose()
if store is not None:
store.close()
if staging is not None:
staging.close()
logger.info("Platform API stopped")
app = FastAPI(
title="KF-OnlineJudge Platform API",
version=__version__,
description="平台 API。身份使用真实可撤销会话，业务接口按模块逐项接入。",
description="工程骨架。当前只提供进程存活检查，业务接口逐项实现。",
docs_url="/api/docs" if settings.environment != "production" else None,
redoc_url=None,
openapi_url="/api/openapi.json" if settings.environment != "production" else None,
lifespan=lifespan,
)
app.state.settings = settings
app.add_middleware(BodyLimitMiddleware)
app.add_middleware(AvailabilityMiddleware, settings=settings)
@app.middleware("http")
async def private_responses(request, call_next):
response = await call_next(request)
if request.url.path.startswith(
("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal", "/api/v1/admin")
):
response.headers["Cache-Control"] = "no-store"
return response
@app.exception_handler(RequestValidationError)
async def invalid_request(_, __):
return JSONResponse({"detail": "invalid request"}, status_code=422)
install_handlers(app)
app.include_router(router, prefix="/api/v1")
app.include_router(identity_router, prefix="/api/v1")
app.include_router(problems_router, prefix="/api/v1")
app.include_router(worker_router, prefix="/api/v1")
app.include_router(submission_router, prefix="/api/v1")
app.include_router(import_router, prefix="/api/v1")
app.include_router(maintenance_router, prefix="/api/v1")
return app
backend/src/kf_oj/api/availability.py
-50
View File
@@ -1,50 +0,0 @@
"""Admission barrier while already accepted work retains its read/cancel paths."""
import re
from starlette.responses import JSONResponse
from kf_oj.infra.availability import current
def needs_open(method: str, path: str) -> bool:
if method in {"GET", "HEAD", "OPTIONS"}:
return False
if method == "POST" and path == "/api/v1/auth/logout":
return False
if method == "POST" and re.fullmatch(r"/api/v1/submissions/[0-9a-f-]{36}/cancel", path):
return False
if method == "PUT" and re.fullmatch(
r"/api/v1/admin/problem-imports/[0-9a-f-]{36}/archive", path
):
# The domain route still requires a previously authorized immutable
# receipt. This cannot create a new upload or bypass its lease/epoch.
return False
return path.startswith("/api/v1/")
class AvailabilityMiddleware:
def __init__(self, app, *, settings):
self.app, self.settings = app, settings
async def __call__(self, scope, receive, send):
if (
scope["type"] == "http"
and self.settings.environment == "production"
and needs_open(scope["method"], scope.get("path", ""))
):
try:
state = current(
self.settings.delivery_release_id, self.settings.delivery_configuration_id
)
allowed = state.state == "open"
except (OSError, ValueError, RecursionError):
allowed = False
if not allowed:
await JSONResponse(
{"detail": "platform mutations unavailable"},
status_code=503,
headers={"Cache-Control": "no-store", "Retry-After": "10"},
)(scope, receive, send)
return
await self.app(scope, receive, send)
backend/src/kf_oj/api/body_limits.py
-75
View File
@@ -1,75 +0,0 @@
"""Bound request bytes and upload time before schema parsing can allocate large inputs."""
import asyncio
import re
from starlette.responses import JSONResponse
class BodyLimitMiddleware:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http" or scope["method"] in {"GET", "HEAD", "OPTIONS"}:
await self.app(scope, receive, send)
return
if scope["method"] == "PUT" and re.fullmatch(
r"/api/v1/admin/problem-imports/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/archive",
scope["path"],
):
# Only this exact endpoint can stream. Its live administrator gate runs
# before receive(), then its service enforces actual bytes/chunks/deadline.
await self.app(scope, receive, send)
return
limit = 8192 if scope["path"] == "/api/v1/auth/login" else 1024 * 1024
if scope["path"] == "/api/v1/submissions":
# Max256KiB UTF-8 source, worst-case JSON escaping plus bounded metadata.
limit = 6 * 256 * 1024 + 8192
headers = dict(scope.get("headers", []))
length = headers.get(b"content-length")
if length is not None:
try:
size = int(length)
if size < 0 or size > limit:
raise ValueError
except ValueError:
await JSONResponse({"detail": "request body outside byte policy"}, 413)(
scope, receive, send
)
return
# Buffer a finite body so errors can be returned before response headers are sent.
chunks, size, received_chunks = [], 0, 0
try:
async with asyncio.timeout(10):
while True:
message = await receive()
if message["type"] == "http.disconnect":
return
body = message.get("body", b"")
size += len(body)
received_chunks += 1
if size > limit or received_chunks > 1024:
await JSONResponse({"detail": "request body outside byte policy"}, 413)(
scope, receive, send
)
return
chunks.append(body)
if not message.get("more_body", False):
break
except TimeoutError:
await JSONResponse({"detail": "request body deadline exceeded"}, 408)(
scope, receive, send
)
return
body = b"".join(chunks)
delivered = False
async def bounded_receive():
nonlocal delivered
if not delivered:
delivered = True
return {"type": "http.request", "body": body, "more_body": False}
return await receive()
await self.app(scope, bounded_receive, send)
backend/src/kf_oj/api/dependencies.py
-50
View File
@@ -1,50 +0,0 @@
from typing import Annotated
from fastapi import Depends, HTTPException, Request
from kf_oj.core.settings import Settings
from kf_oj.modules.identity.public import AuthenticationFailed, Principal
from kf_oj.modules.identity.service import IdentityService
SESSION_COOKIE = "kf_session"
def identity_service(request: Request) -> IdentityService:
service = getattr(request.app.state, "identity", None)
if service is None:
raise HTTPException(503, "identity service unavailable")
return service
def trusted_origin(request: Request) -> None:
settings: Settings = request.app.state.settings
if request.headers.get("origin") not in settings.browser_origins:
raise HTTPException(403, "request origin rejected")
if request.headers.get("sec-fetch-site") == "cross-site":
raise HTTPException(403, "cross-site request rejected")
async def current_principal(
request: Request, identity: Annotated[IdentityService, Depends(identity_service)]
) -> Principal:
try:
return await identity.authenticate(request.cookies.get(SESSION_COOKIE))
except AuthenticationFailed:
raise HTTPException(401, "authentication required") from None
async def unsafe_principal(
request: Request, identity: Annotated[IdentityService, Depends(identity_service)]
) -> Principal:
trusted_origin(request)
try:
await identity.authenticate(request.cookies.get(SESSION_COOKIE))
except AuthenticationFailed:
raise HTTPException(401, "authentication required") from None
csrf = request.headers.get("x-csrf-token")
if csrf is None:
raise HTTPException(403, "request token required")
try:
return await identity.authenticate(request.cookies.get(SESSION_COOKIE), csrf_token=csrf)
except AuthenticationFailed:
raise HTTPException(403, "request authentication rejected") from None
backend/src/kf_oj/api/dependency_errors.py
-42
View File
@@ -1,42 +0,0 @@
"""Bounded, non-secret API responses for transient dependency/capacity failures."""
import errno
from asyncpg import CannotConnectNowError, PostgresConnectionError
from fastapi import FastAPI
from sqlalchemy.exc import OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
from starlette.responses import JSONResponse
async def temporarily_unavailable(_, __):
# Write outcomes may be uncertain. Existing operation/submission UUIDs, not
# automatic server retries, own recovery from a lost response.
return JSONResponse(
{"detail": "service temporarily unavailable"},
status_code=503,
headers={"Cache-Control": "no-store", "Retry-After": "5"},
)
async def filesystem_failure(_, error: OSError):
if error.errno in {errno.ENOSPC, errno.EDQUOT, errno.EMFILE, errno.ENFILE}:
return await temporarily_unavailable(_, error)
# Unrelated filesystem/programming failures retain server-error semantics.
# Never serialize path/source/query/credential text from an exception.
return JSONResponse(
{"detail": "internal server error"}, status_code=500, headers={"Cache-Control": "no-store"}
)
def install_handlers(app: FastAPI) -> None:
for error in (
ConnectionError,
TimeoutError,
OperationalError,
PoolTimeoutError,
PostgresConnectionError,
CannotConnectNowError,
):
app.add_exception_handler(error, temporarily_unavailable)
app.add_exception_handler(OSError, filesystem_failure)
backend/src/kf_oj/api/v1/admin_imports.py
-139
View File
@@ -1,139 +0,0 @@
"""Administrator import contract; configured service availability is explicit."""
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy.exc import DBAPIError
from starlette.requests import ClientDisconnect
from kf_oj.api.admin_dependencies import maintenance_reader, maintenance_writer
from kf_oj.contracts.admin_problems import (
CreateProblemImport,
ProblemImportCapabilities,
ProblemImportDetail,
ProblemImportPage,
ProblemImportSummary,
)
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.identity.public import MaintenanceDenied, Principal
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
router = APIRouter(prefix="/admin/problem-imports", tags=["problem-maintenance"])
@router.get("/capabilities", response_model=ProblemImportCapabilities)
async def capabilities(
request: Request, _: Annotated[Principal, Depends(maintenance_reader)]
) -> ProblemImportCapabilities:
service = getattr(request.app.state, "problem_imports", None)
if service is None:
return ProblemImportCapabilities(uploads_available=False, validation_available=False)
return await service.capabilities()
def service(request):
result = getattr(request.app.state, "problem_imports", None)
if result is None:
raise HTTPException(503, "problem imports unavailable")
return result
async def operation(awaitable):
try:
return await awaitable
except MaintenanceDenied:
raise HTTPException(403, "problem maintenance required") from None
except ImportConflict:
raise HTTPException(409, "import request conflict") from None
except ImportMissing:
raise HTTPException(404, "import unavailable") from None
except TimeoutError:
raise HTTPException(408, "upload deadline exceeded") from None
except (ImportUnavailable, ArtifactError, OSError, DBAPIError):
raise HTTPException(503, "problem imports unavailable") from None
except ClientDisconnect:
raise HTTPException(400, "upload interrupted") from None
@router.post("", status_code=201, response_model=ProblemImportSummary)
async def create(
body: CreateProblemImport,
request: Request,
principal: Annotated[Principal, Depends(maintenance_writer)],
):
if request.headers.get("content-type", "").split(";", 1)[0] != "application/json":
raise HTTPException(415, "JSON request required")
return await operation(service(request).create(principal, body))
@router.get("", response_model=ProblemImportPage)
async def page(
request: Request,
principal: Annotated[Principal, Depends(maintenance_reader)],
after: UUID | None = None,
limit: Annotated[int, Query(ge=1, le=100)] = 20,
):
return await operation(service(request).list(principal, after, limit))
@router.get("/{operation_id}", response_model=ProblemImportDetail)
async def detail(
operation_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(maintenance_reader)],
):
return await operation(service(request).detail(principal, operation_id))
@router.put(
"/{operation_id}/archive",
response_model=ProblemImportSummary,
openapi_extra={
"requestBody": {
"required": True,
"content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}},
}
},
)
async def upload(
operation_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(maintenance_writer)],
):
if request.url.path != f"/api/v1/admin/problem-imports/{operation_id}/archive":
raise HTTPException(422, "canonical operation UUID required")
if request.headers.get("content-type", "").split(";", 1)[0] != "application/zip":
raise HTTPException(415, "ZIP request required")
if request.headers.get("content-encoding") not in (None, "identity"):
raise HTTPException(415, "encoded upload unavailable")
length = request.headers.get("content-length")
if length is not None:
try:
if not length.isascii() or not length.isdecimal() or len(length) > 12:
raise ValueError
length = int(length)
except ValueError:
raise HTTPException(413, "invalid archive length") from None
async def chunks():
while True:
message = await request.receive()
if message["type"] == "http.disconnect":
raise ClientDisconnect()
if message["type"] != "http.request":
raise ImportConflict("invalid upload frame")
yield message.get("body", b"")
if not message.get("more_body", False):
return
return await operation(service(request).upload(principal, operation_id, chunks(), length))
@router.post("/{operation_id}/discard", response_model=ProblemImportSummary)
async def discard(
operation_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(maintenance_writer)],
):
return await operation(service(request).discard(principal, operation_id))
backend/src/kf_oj/api/v1/admin_problems.py
-150
View File
@@ -1,150 +0,0 @@
"""Live administrator projections and actor-bound immutable mutations."""
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy.exc import DBAPIError
from kf_oj.api.admin_dependencies import maintenance_reader, maintenance_writer
from kf_oj.contracts.problem_maintenance import (
AuditAction,
ChangeProblemAvailability,
MaintainedProblem,
MaintainedProblemPage,
MaintainedVersionPage,
MaintenanceAuditPage,
MaintenanceReceipt,
PublishCandidate,
)
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.identity.public import MaintenanceDenied, Principal
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
from kf_oj.modules.problems.public import ProblemUnavailable
router = APIRouter(prefix="/admin", tags=["problem-maintenance"])
def service(request, name="problem_maintenance"):
result = getattr(request.app.state, name, None)
if result is None:
raise HTTPException(503, "problem maintenance unavailable")
return result
async def operation(awaitable):
try:
return await awaitable
except MaintenanceDenied:
raise HTTPException(403, "problem maintenance required") from None
except ImportConflict:
raise HTTPException(409, "maintenance precondition conflict") from None
except ImportMissing:
raise HTTPException(404, "maintenance object unavailable") from None
except (ImportUnavailable, ArtifactError, ProblemUnavailable, OSError, DBAPIError):
raise HTTPException(503, "problem maintenance unavailable") from None
except ValueError:
raise HTTPException(422, "invalid maintenance request") from None
@router.get("/problems", response_model=MaintainedProblemPage)
async def page(
request: Request,
principal: Annotated[Principal, Depends(maintenance_reader)],
after: Annotated[str | None, Query(pattern=r"^[a-z0-9][a-z0-9_.-]{0,63}$")] = None,
limit: Annotated[int, Query(ge=1, le=100)] = 20,
enabled: bool | None = None,
):
return await operation(
service(request).page(principal, after=after, limit=limit, enabled=enabled)
)
@router.get("/problem-slugs/{slug}", response_model=MaintainedProblem)
async def by_slug(
slug: str, request: Request, principal: Annotated[Principal, Depends(maintenance_reader)]
):
from pydantic import TypeAdapter
from kf_oj.contracts.judging import Label
try:
TypeAdapter(Label).validate_python(slug)
except ValueError:
raise HTTPException(422, "invalid problem slug") from None
return await operation(service(request).by_slug(principal, slug))
@router.get("/problems/{problem_id}", response_model=MaintainedProblem)
async def detail(
problem_id: UUID, request: Request, principal: Annotated[Principal, Depends(maintenance_reader)]
):
return await operation(service(request).detail(principal, problem_id))
@router.get("/problems/{problem_id}/versions", response_model=MaintainedVersionPage)
async def versions(
problem_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(maintenance_reader)],
before: Annotated[int | None, Query(ge=1)] = None,
limit: Annotated[int, Query(ge=1, le=100)] = 20,
):
return await operation(
service(request).versions(principal, problem_id, before=before, limit=limit)
)
@router.get("/problem-maintenance/{operation_id}", response_model=MaintenanceReceipt)
async def receipt(
operation_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(maintenance_reader)],
):
return await operation(service(request).receipt(principal, operation_id))
@router.post("/problem-publications", response_model=MaintenanceReceipt, status_code=201)
async def publish(
body: PublishCandidate,
request: Request,
principal: Annotated[Principal, Depends(maintenance_writer)],
):
return await operation(service(request).publish(principal, body))
@router.post("/problems/{problem_id}/disable", response_model=MaintenanceReceipt)
async def disable(
problem_id: UUID,
body: ChangeProblemAvailability,
request: Request,
principal: Annotated[Principal, Depends(maintenance_writer)],
):
return await operation(
service(request).availability(principal, problem_id, body, enabled=False)
)
@router.post("/problems/{problem_id}/restore", response_model=MaintenanceReceipt)
async def restore(
problem_id: UUID,
body: ChangeProblemAvailability,
request: Request,
principal: Annotated[Principal, Depends(maintenance_writer)],
):
return await operation(service(request).availability(principal, problem_id, body, enabled=True))
@router.get("/maintenance-audit", response_model=MaintenanceAuditPage)
async def audit(
request: Request,
principal: Annotated[Principal, Depends(maintenance_reader)],
before: Annotated[str | None, Query(max_length=100)] = None,
limit: Annotated[int, Query(ge=1, le=100)] = 20,
action: AuditAction | None = None,
):
return await operation(
service(request, "maintenance_audit").page(
principal, before=before, limit=limit, action=action
)
)
backend/src/kf_oj/api/v1/identity.py
-106
View File
@@ -1,106 +0,0 @@
from typing import Annotated
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from kf_oj.api.dependencies import (
SESSION_COOKIE,
current_principal,
identity_service,
trusted_origin,
unsafe_principal,
)
from kf_oj.contracts.identity import AccountCapabilities, CurrentUser, LoginRequest, LoginResponse
from kf_oj.modules.identity.public import AuthenticationFailed, LoginThrottled, Principal
from kf_oj.modules.identity.service import IdentityService
router = APIRouter(prefix="/auth", tags=["identity"])
def current_user(principal: Principal) -> CurrentUser:
return CurrentUser(
id=principal.account_id,
username=principal.username,
capabilities=AccountCapabilities(
problem_maintenance=principal.can_maintain_problems,
authorization_version=str(principal.authorization_version),
),
)
@router.post("/login", response_model=LoginResponse, dependencies=[Depends(trusted_origin)])
async def login(
body: LoginRequest,
request: Request,
response: Response,
identity: Annotated[IdentityService, Depends(identity_service)],
) -> LoginResponse:
if request.headers.get("content-type", "").split(";", 1)[0] != "application/json":
raise HTTPException(415, "JSON request required")
try:
issued = await identity.login(
body.username,
body.password.get_secret_value(),
peer=request.client.host if request.client else "",
)
except LoginThrottled:
raise HTTPException(
429, "login temporarily limited", headers={"Retry-After": "900"}
) from None
except (AuthenticationFailed, ValueError):
raise HTTPException(401, "invalid credentials") from None
settings = request.app.state.settings
response.set_cookie(
SESSION_COOKIE,
issued.token,
max_age=settings.session_lifetime_seconds,
httponly=True,
secure=settings.environment == "production",
samesite="strict",
path="/api/v1",
)
response.set_cookie(
"kf_csrf",
issued.csrf_token,
max_age=settings.session_lifetime_seconds,
httponly=False,
secure=settings.environment == "production",
samesite="strict",
path="/",
)
response.headers["Cache-Control"] = "no-store"
return LoginResponse(
user=current_user(issued.principal),
expires_at=issued.expires_at,
csrf_token=issued.csrf_token,
)
@router.get("/me", response_model=CurrentUser)
async def me(response: Response, principal: Annotated[Principal, Depends(current_principal)]):
response.headers["Cache-Control"] = "no-store"
return current_user(principal)
@router.post("/logout", status_code=204, dependencies=[Depends(unsafe_principal)])
async def logout(
request: Request, identity: Annotated[IdentityService, Depends(identity_service)]
) -> Response:
try:
await identity.logout(request.cookies[SESSION_COOKIE], request.headers["x-csrf-token"])
except AuthenticationFailed:
raise HTTPException(401, "authentication required") from None
response = Response(status_code=204, headers={"Cache-Control": "no-store"})
response.delete_cookie(
SESSION_COOKIE,
path="/api/v1",
httponly=True,
secure=request.app.state.settings.environment == "production",
samesite="strict",
)
response.delete_cookie(
"kf_csrf",
path="/",
secure=request.app.state.settings.environment == "production",
samesite="strict",
)
return response
backend/src/kf_oj/api/v1/problems.py
-34
View File
@@ -1,34 +0,0 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, HTTPException, Query, Request
from kf_oj.contracts.problems import ProblemDetail, ProblemPage
from kf_oj.modules.problems.public import ProblemUnavailable
from kf_oj.modules.problems.service import ProblemService
router = APIRouter(prefix="/problems", tags=["problems"])
def service(request: Request) -> ProblemService:
component = getattr(request.app.state, "problems", None)
if component is None:
raise HTTPException(503, "problem service unavailable")
return component
@router.get("", response_model=ProblemPage)
async def list_problems(
request: Request,
after: Annotated[str | None, Query(pattern=r"^[a-z0-9][a-z0-9_-]{0,63}$")] = None,
limit: Annotated[int, Query(ge=1, le=100)] = 20,
) -> ProblemPage:
return await service(request).list(after=after, limit=limit)
@router.get("/versions/{version_id}", response_model=ProblemDetail)
async def problem_version(version_id: UUID, request: Request) -> ProblemDetail:
try:
return await service(request).detail(version_id)
except ProblemUnavailable:
raise HTTPException(404, "problem version unavailable") from None
backend/src/kf_oj/api/v1/submissions.py
-140
View File
@@ -1,140 +0,0 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
from kf_oj.api.dependencies import current_principal, unsafe_principal
from kf_oj.contracts.judging_v2 import Language
from kf_oj.contracts.submission_performance import SubmissionPerformance
from kf_oj.contracts.submissions import (
CancellationAccepted,
CreateSubmission,
SubmissionCreated,
SubmissionDetail,
SubmissionPage,
SubmissionSource,
)
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.identity.public import Principal
from kf_oj.modules.submissions.public import (
CancelUnavailable,
IdempotencyConflict,
SubmissionBusy,
SubmissionUnavailable,
)
router = APIRouter(prefix="/submissions", tags=["submissions"])
@router.post("", status_code=201, response_model=SubmissionCreated)
async def create(
body: CreateSubmission,
request: Request,
response: Response,
principal: Annotated[Principal, Depends(unsafe_principal)],
):
if request.headers.get("content-type", "").split(";", 1)[0] != "application/json":
raise HTTPException(415, "JSON request required")
service = getattr(request.app.state, "submissions", None)
if service is None:
raise HTTPException(503, "submission service unavailable")
try:
result = await service.create(principal, body)
except IdempotencyConflict:
raise HTTPException(409, "idempotency request conflict") from None
except SubmissionBusy:
raise HTTPException(429, "pending submission capacity exhausted") from None
except SubmissionUnavailable:
raise HTTPException(404, "submission capability unavailable") from None
except ArtifactError:
raise HTTPException(503, "source storage unavailable") from None
except ValueError:
raise HTTPException(422, "submission outside admission policy") from None
response.headers["Cache-Control"] = "no-store"
return result
def queries(request: Request):
service = getattr(request.app.state, "submission_queries", None)
if service is None:
raise HTTPException(503, "submission service unavailable")
return service
@router.get("", response_model=SubmissionPage)
async def history(
request: Request,
principal: Annotated[Principal, Depends(current_principal)],
after: Annotated[str | None, Query(max_length=128)] = None,
limit: Annotated[int, Query(ge=1, le=100)] = 20,
problem_version_id: UUID | None = None,
language: Language | None = None,
):
try:
return await queries(request).page(principal, after, limit, problem_version_id, language)
except ValueError:
raise HTTPException(422, "invalid history request") from None
@router.get("/{submission_id}", response_model=SubmissionDetail)
async def detail(
submission_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(current_principal)],
):
try:
return await queries(request).detail(principal, submission_id)
except SubmissionUnavailable:
raise HTTPException(404, "submission unavailable") from None
except ValueError:
raise HTTPException(503, "stored result unavailable") from None
@router.get("/{submission_id}/performance", response_model=SubmissionPerformance)
async def performance(
submission_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(current_principal)],
):
service = getattr(request.app.state, "submission_performance", None)
if service is None:
raise HTTPException(503, "performance service unavailable")
try:
return await service.detail(principal, submission_id)
except SubmissionUnavailable:
raise HTTPException(404, "submission unavailable") from None
except ValueError:
raise HTTPException(503, "performance unavailable") from None
@router.get("/{submission_id}/source", response_model=SubmissionSource)
async def source(
submission_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(current_principal)],
):
service = queries(request)
try:
result = await service.detail(principal, submission_id)
data = await service.owned_source(principal, submission_id)
return SubmissionSource(
submission_id=submission_id, language=result.language, source=data.decode("utf-8")
)
except SubmissionUnavailable:
raise HTTPException(404, "submission unavailable") from None
except ValueError:
raise HTTPException(503, "stored source unavailable") from None
@router.post("/{submission_id}/cancel", status_code=202, response_model=CancellationAccepted)
async def cancel(
submission_id: UUID,
request: Request,
principal: Annotated[Principal, Depends(unsafe_principal)],
):
try:
return await queries(request).cancel(principal, submission_id)
except SubmissionUnavailable:
raise HTTPException(404, "submission unavailable") from None
except CancelUnavailable:
raise HTTPException(503, "cancellation service unavailable") from None
backend/src/kf_oj/api/v1/worker_artifacts.py
-96
View File
@@ -1,96 +0,0 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Header, HTTPException, Request, Response, Security
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from kf_oj.contracts.judging import JobManifest, LeaseBinding
from kf_oj.modules.submissions.public import WorkerAccessDenied, WorkerReadBusy
from kf_oj.modules.submissions.worker_access import WorkerAccess
router = APIRouter(prefix="/internal/evaluations", tags=["worker artifacts"])
bearer = HTTPBearer(auto_error=False, scheme_name="WorkerLease")
def grant(
evaluation_id: UUID,
attempt_id: UUID,
worker: UUID,
fence: int,
credentials: HTTPAuthorizationCredentials | None,
) -> tuple[LeaseBinding, str]:
if (
credentials is None
or credentials.scheme.lower() != "bearer"
or len(credentials.credentials) != 64
):
raise HTTPException(401, "worker lease required")
try:
return LeaseBinding(
evaluation_id=evaluation_id,
attempt_id=attempt_id,
worker_id=worker,
fencing_version=fence,
), credentials.credentials
except ValueError:
raise HTTPException(404, "worker grant rejected") from None
def component(request: Request) -> WorkerAccess:
access = getattr(request.app.state, "worker_access", None)
if access is None:
raise HTTPException(503, "worker artifact service unavailable")
return access
@router.get("/{evaluation_id}/attempts/{attempt_id}/manifest", response_model=JobManifest)
async def manifest(
evaluation_id: UUID,
attempt_id: UUID,
request: Request,
worker: Annotated[UUID, Header(alias="X-Worker-ID")],
fence: Annotated[int, Header(alias="X-Fencing-Version", ge=1)],
credentials: Annotated[HTTPAuthorizationCredentials | None, Security(bearer)],
):
binding, token = grant(evaluation_id, attempt_id, worker, fence, credentials)
try:
payload = await component(request).manifest(binding, token)
except WorkerAccessDenied:
raise HTTPException(404, "worker grant rejected") from None
except WorkerReadBusy:
raise HTTPException(429, "worker reads temporarily limited") from None
return Response(payload, media_type="application/json", headers={"Cache-Control": "no-store"})
@router.get(
"/{evaluation_id}/attempts/{attempt_id}/artifacts/{digest}",
response_class=Response,
responses={
200: {
"content": {
"application/octet-stream": {"schema": {"type": "string", "format": "binary"}}
}
}
},
)
async def artifact(
evaluation_id: UUID,
attempt_id: UUID,
digest: str,
request: Request,
worker: Annotated[UUID, Header(alias="X-Worker-ID")],
fence: Annotated[int, Header(alias="X-Fencing-Version", ge=1)],
credentials: Annotated[HTTPAuthorizationCredentials | None, Security(bearer)],
):
binding, token = grant(evaluation_id, attempt_id, worker, fence, credentials)
try:
payload = await component(request).artifact(binding, token, digest)
except WorkerAccessDenied:
raise HTTPException(404, "worker grant rejected") from None
except WorkerReadBusy:
raise HTTPException(429, "worker reads temporarily limited") from None
return Response(
payload,
media_type="application/octet-stream",
headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"},
)
backend/src/kf_oj/contracts/admin_problems.py
-180
View File
@@ -1,180 +0,0 @@
"""Versioned problem-import DTOs; hidden bytes, paths and toolchain selection stay private."""
from datetime import datetime
from enum import StrEnum
from typing import Annotated, Literal
from pydantic import Field, StrictBool, model_validator
from kf_oj.contracts.judging import Digest, Identifier, Label, RunLimits, Version, WireModel
from kf_oj.contracts.judging_v2 import Language
from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample
MAX_ARCHIVE_BYTES = 64 * 1024**2
MAX_UNPACKED_BYTES = 64 * 1024**2
MAX_ARCHIVE_FILES = 534
MAX_ARCHIVE_NAME_BYTES = 128
MAX_UPLOAD_CHUNKS = 4096
MAX_UPLOAD_CHUNK_BYTES = 1024**2
UPLOAD_DEADLINE_SECONDS = 30
VALIDATION_DEADLINE_SECONDS = 60
MAX_IMPORT_ATTEMPTS = 2
IMPORT_LIFETIME_SECONDS = 86400
MAX_OWNER_CANDIDATES = 10
MAX_TOTAL_CANDIDATES = 32
STAGING_MAX_BYTES = 256 * 1024**2
STAGING_RESERVATION_BYTES = MAX_ARCHIVE_BYTES + MAX_UNPACKED_BYTES
class ImportState(StrEnum):
UPLOADING = "uploading"
UPLOADED = "uploaded"
VALIDATING = "validating"
READY = "ready"
FAILED = "failed"
PUBLISHED = "published"
DISCARDED = "discarded"
EXPIRED = "expired"
class ImportFailure(StrEnum):
INVALID_ARCHIVE = "invalid_archive"
INVALID_MANIFEST = "invalid_manifest"
INVALID_DATA = "invalid_data"
DIGEST_MISMATCH = "digest_mismatch"
POLICY_REJECTED = "policy_rejected"
UPLOAD_INTERRUPTED = "upload_interrupted"
UPLOAD_DEADLINE = "upload_deadline"
VALIDATION_DEADLINE = "validation_deadline"
VALIDATION_UNAVAILABLE = "validation_unavailable"
STORAGE_UNAVAILABLE = "storage_unavailable"
AUTHORITY_REVOKED = "authority_revoked"
EXPIRED = "expired"
class ProblemImportLimits(WireModel):
schema_version: Version = 1
archive_format: Literal["single_level_zip"] = "single_level_zip"
manifest_name: Literal["problem.json"] = "problem.json"
manifest_schema_versions: tuple[Literal[1, 2], ...] = (1, 2)
languages: tuple[Language, ...] = tuple(Language)
archive_bytes: int = MAX_ARCHIVE_BYTES
unpacked_bytes: int = MAX_UNPACKED_BYTES
files: int = MAX_ARCHIVE_FILES
filename_ascii_bytes: int = MAX_ARCHIVE_NAME_BYTES
upload_chunks: int = MAX_UPLOAD_CHUNKS
chunk_bytes: int = MAX_UPLOAD_CHUNK_BYTES
upload_deadline_seconds: int = UPLOAD_DEADLINE_SECONDS
validation_deadline_seconds: int = VALIDATION_DEADLINE_SECONDS
global_concurrency: Literal[1] = 1
owner_concurrency: Literal[1] = 1
owner_candidates: int = MAX_OWNER_CANDIDATES
total_candidates: int = MAX_TOTAL_CANDIDATES
staging_bytes: int = STAGING_MAX_BYTES
candidate_lifetime_seconds: int = IMPORT_LIFETIME_SECONDS
class ProblemImportCapabilities(WireModel):
schema_version: Version = 1
limits: ProblemImportLimits = ProblemImportLimits()
uploads_available: StrictBool
validation_available: StrictBool
class CreateProblemImport(WireModel):
schema_version: Version = 1
operation_id: Identifier
archive_digest: Digest
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
class LegacyImportBudget(WireModel):
compile_limits: RunLimits
run_limits: RunLimits
job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=3600000000)]
class CandidatePreview(WireModel):
candidate_id: Identifier
slug: Label
title: Annotated[str, Field(strict=True, min_length=1, max_length=160)]
statement: Annotated[str, Field(strict=True, min_length=1, max_length=262144)]
samples: Annotated[tuple[PublicSample, ...], Field(max_length=10)]
judging_schema_version: Annotated[int, Field(strict=True, ge=1, le=2)]
languages: Annotated[tuple[Language, ...], Field(min_length=1, max_length=4)]
legacy_budget: LegacyImportBudget | None
language_configurations: Annotated[
tuple[PublicLanguageConfiguration, ...], Field(max_length=4)
] = ()
case_count: Annotated[int, Field(strict=True, ge=1, le=256)]
@model_validator(mode="after")
def bounded_text(self):
if len(self.statement.encode("utf-8")) > 262144 or "\x00" in self.statement + self.title:
raise ValueError("invalid public text")
languages = [c.language for c in self.language_configurations]
if len(set(languages)) != len(languages):
raise ValueError("duplicate language configuration")
if len(set(self.languages)) != len(self.languages):
raise ValueError("duplicate language")
if self.judging_schema_version == 1:
if (
self.language_configurations
or self.legacy_budget is None
or any(
language not in (Language.C17, Language.CPP20) for language in self.languages
)
):
raise ValueError("invalid legacy candidate")
elif tuple(languages) != self.languages or self.legacy_budget is not None:
raise ValueError("invalid multi-runtime candidate")
return self
class ProblemImportSummary(WireModel):
schema_version: Version = 1
operation_id: Identifier
state: ImportState
expected_archive_digest: Digest
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
created_at: datetime
expires_at: datetime
updated_at: datetime
failure_code: ImportFailure | None
candidate_id: Identifier | None
published_version_id: Identifier | None
@model_validator(mode="after")
def consistent_state(self):
if self.expires_at <= self.created_at or self.updated_at < self.created_at:
raise ValueError("invalid import time range")
if (self.state in (ImportState.READY, ImportState.PUBLISHED)) != (
self.candidate_id is not None
):
raise ValueError("candidate state mismatch")
if (self.state == ImportState.PUBLISHED) != (self.published_version_id is not None):
raise ValueError("published state mismatch")
if self.state in (ImportState.FAILED, ImportState.EXPIRED):
if self.failure_code is None:
raise ValueError("failure code required")
elif self.failure_code is not None:
raise ValueError("unexpected failure code")
return self
class ProblemImportDetail(ProblemImportSummary):
candidate: CandidatePreview | None
@model_validator(mode="after")
def candidate_matches(self):
if (self.candidate_id is None) != (self.candidate is None):
raise ValueError("candidate preview missing")
if self.candidate and self.candidate.candidate_id != self.candidate_id:
raise ValueError("candidate preview mismatch")
return self
class ProblemImportPage(WireModel):
schema_version: Version = 1
items: Annotated[tuple[ProblemImportSummary, ...], Field(max_length=100)]
next_after: Identifier | None
backend/src/kf_oj/contracts/availability.py
-18
View File
@@ -1,18 +0,0 @@
"""Root-to-API availability binding; independent from immutable judging protocols."""
from typing import Annotated, Literal
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
from kf_oj.contracts.judging import integer_version
AVAILABILITY_PATH = "/etc/kf-admission/status.json"
class AvailabilityRecord(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
release_id: Annotated[str, Field(pattern=r"^sd-[0-9a-f]{12}$")]
configuration_id: Annotated[str, Field(pattern=r"^cfg-[0-9a-f]{24}$")]
state: Literal["open", "draining", "quiescent"]
backend/src/kf_oj/contracts/export.py
-58
View File
@@ -5,23 +5,7 @@ import json
from pathlib import Path
from kf_oj.api.app import create_app
from kf_oj.contracts.admin_problems import (
CreateProblemImport,
ProblemImportCapabilities,
ProblemImportDetail,
)
from kf_oj.contracts.availability import AvailabilityRecord
from kf_oj.contracts.judge import JudgeTaskNotification
from kf_oj.contracts.judging import WIRE_MODELS
from kf_oj.contracts.judging_v2 import WIRE_MODELS as WIRE_MODELS_V2
from kf_oj.contracts.problem_maintenance import (
ChangeProblemAvailability,
MaintenanceAuditPage,
MaintenanceReceipt,
PublishCandidate,
)
from kf_oj.contracts.restore_io import RestoreIOReply, RestoreIORequest
from kf_oj.contracts.submission_performance import SubmissionPerformance
from kf_oj.core.settings import Settings
@@ -39,48 +23,6 @@ def main() -> None:
Settings(_env_file=None, environment="test")
).openapi(),
}
for name, model in WIRE_MODELS.items():
document = model.model_json_schema()
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
document["$id"] = f"urn:kf-oj:{name}:v1"
document["$comment"] = (
"Structural schema; also run versioned model validation for cross-field "
"budgets, result consistency, UUID canonical form and UTF-8 byte limits."
)
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
performance = SubmissionPerformance.model_json_schema()
performance["$schema"] = "https://json-schema.org/draft/2020-12/schema"
performance["$id"] = "urn:kf-oj:submission-performance:v1"
performance["$comment"] = (
"Anonymous bounded aggregates, decimal raw units and strict same-snapshot "
"counts; also run model validation and server-side ownership/environment checks."
)
outputs[root / "contracts/schemas/submission-performance.v1.schema.json"] = performance
for name, model in {
"delivery-availability": AvailabilityRecord,
"restore-io-request": RestoreIORequest,
"restore-io-reply": RestoreIOReply,
"problem-publish": PublishCandidate,
"problem-availability": ChangeProblemAvailability,
"problem-maintenance": MaintenanceReceipt,
"problem-maintenance-audit": MaintenanceAuditPage,
"problem-import-create": CreateProblemImport,
"problem-import": ProblemImportDetail,
"problem-import-capabilities": ProblemImportCapabilities,
}.items():
document = model.model_json_schema()
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
document["$id"] = f"urn:kf-oj:{name}:v1"
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
for name, model in WIRE_MODELS_V2.items():
document = model.model_json_schema()
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
document["$id"] = f"urn:kf-oj:{name}:v2"
document["$comment"] = (
"Multi-runtime protocol. Also validate runtime/source/attempt bindings, startup "
"and node budgets, canonical bounded class sets and result consistency."
)
outputs[root / f"contracts/schemas/{name}.v2.schema.json"] = document
stale = []
for path, document in outputs.items():
content = json.dumps(document, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
backend/src/kf_oj/contracts/identity.py
-29
View File
@@ -1,29 +0,0 @@
from datetime import datetime
from typing import Annotated, Literal
from pydantic import Field, SecretStr
from kf_oj.contracts.judging import Digest, Identifier, WireModel
class LoginRequest(WireModel):
username: Annotated[str, Field(strict=True, pattern=r"^[a-z0-9_]{3,24}$")]
password: Annotated[SecretStr, Field(min_length=12, max_length=128)]
class AccountCapabilities(WireModel):
schema_version: Literal[1] = 1
problem_maintenance: bool = False
authorization_version: Annotated[str, Field(pattern=r"^[1-9][0-9]{0,18}$")] = "1"
class CurrentUser(WireModel):
id: Identifier
username: str
capabilities: AccountCapabilities = Field(default_factory=AccountCapabilities)
class LoginResponse(WireModel):
user: CurrentUser
expires_at: datetime
csrf_token: Digest = Field(repr=False)
backend/src/kf_oj/contracts/judge.py
+3 -5
View File
@@ -1,9 +1,7 @@
from typing import Annotated, Literal
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, BeforeValidator, ConfigDict
from kf_oj.contracts.judging import integer_version
from pydantic import BaseModel, ConfigDict
class JudgeTaskNotification(BaseModel):
@@ -11,6 +9,6 @@ class JudgeTaskNotification(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
schema_version: Literal[1]
kind: Literal["judge.evaluation"]
evaluation_id: UUID
backend/src/kf_oj/contracts/judge_state.py
-48
View File
@@ -1,48 +0,0 @@
"""Pure transition/fencing rules; callers still need a locked database transaction."""
from enum import StrEnum
from kf_oj.contracts.judging import LeaseBinding
class EvaluationState(StrEnum):
QUEUED = "queued"
CLAIMED = "claimed"
COMPILING = "compiling"
RUNNING = "running"
CANCELLING = "cancelling"
FINISHED = "finished"
CANCELLED = "cancelled"
FAILED = "failed"
def allows(self, target: "EvaluationState") -> bool:
transitions = {
self.QUEUED: {self.CLAIMED, self.CANCELLED},
self.CLAIMED: {self.COMPILING, self.CANCELLING, self.FAILED, self.QUEUED},
self.COMPILING: {
self.RUNNING,
self.FINISHED,
self.CANCELLING,
self.FAILED,
self.QUEUED,
},
self.RUNNING: {self.FINISHED, self.CANCELLING, self.FAILED, self.QUEUED},
self.CANCELLING: {self.CANCELLED, self.FAILED, self.QUEUED},
}
return target in transitions.get(self, set())
def require_current_lease(
expected: LeaseBinding,
supplied: LeaseBinding,
expires_at_unix_ms: int,
now_unix_ms: int,
) -> None:
"""Identity/version equality and a strict deadline; never trust client time."""
if expected != supplied or now_unix_ms < 0 or now_unix_ms >= expires_at_unix_ms:
raise ValueError("stale or foreign execution authority")
def require_transition(current: EvaluationState, target: EvaluationState) -> None:
if not current.allows(target):
raise ValueError("invalid evaluation transition")
backend/src/kf_oj/contracts/judging.py
-403
View File
@@ -1,403 +0,0 @@
"""Versioned judge wire models. Authorization and persistence belong to services."""
import json
from enum import StrEnum
from typing import Annotated, Literal, Self
from uuid import UUID
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
StrictBool,
model_validator,
)
MIB = 1024 * 1024
I64_MAX = (1 << 63) - 1
MAX_SOURCE_BYTES = 256 * 1024
MAX_PACKAGE_BYTES = 64 * MIB
MAX_CASES = 256
MAX_JOB_WALL_US = 3_600_000_000
MAX_WIRE_BYTES = 256 * 1024
def integer_version(value: object) -> object:
if type(value) is not int:
raise ValueError("schema version must be an integer")
return value
def canonical_id(value: object) -> UUID:
try:
parsed = UUID(str(value))
except (ValueError, TypeError, AttributeError) as error:
raise ValueError("invalid identifier") from error
if parsed.int == 0 or str(value) != str(parsed):
raise ValueError("identifier must be non-nil and canonical")
return parsed
Version = Annotated[Literal[1], BeforeValidator(integer_version)]
Identifier = Annotated[UUID, BeforeValidator(canonical_id)]
Digest = Annotated[str, Field(strict=True, min_length=64, max_length=64, pattern=r"^[0-9a-f]{64}$")]
Label = Annotated[str, Field(strict=True, pattern=r"^[a-z0-9][a-z0-9_-]{0,63}$")]
Counter = Annotated[int, Field(strict=True, ge=1, le=I64_MAX)]
Measurement = Annotated[int, Field(strict=True, ge=0, le=I64_MAX)]
class WireModel(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
class Language(StrEnum):
C17 = "c17"
CPP20 = "cpp20"
class Verdict(StrEnum):
ACCEPTED = "accepted"
WRONG_ANSWER = "wrong_answer"
COMPILATION_ERROR = "compilation_error"
RUNTIME_ERROR = "runtime_error"
TIME_LIMIT = "time_limit_exceeded"
MEMORY_LIMIT = "memory_limit_exceeded"
OUTPUT_LIMIT = "output_limit_exceeded"
CANCELLED = "cancelled"
SYSTEM_ERROR = "system_error"
class CompilationStatus(StrEnum):
SUCCEEDED = "succeeded"
COMPILATION_ERROR = "compilation_error"
COMPILATION_LIMIT = "compilation_limit"
CANCELLED = "cancelled"
SYSTEM_ERROR = "system_error"
class FailureCode(StrEnum):
ARTIFACT_UNAVAILABLE = "artifact_unavailable"
SANDBOX_UNAVAILABLE = "sandbox_unavailable"
CLEANUP_FAILED = "cleanup_failed"
LEASE_LOST = "lease_lost"
JOB_DEADLINE = "job_deadline"
class ArtifactReference(WireModel):
digest: Digest
bytes: Annotated[int, Field(strict=True, ge=0, le=MAX_PACKAGE_BYTES)]
class Toolchain(WireModel):
toolchain_id: Label
image_kind: Literal["linux_gnu_cxx_x86_64_v1"]
image_revision: Annotated[int, Field(strict=True, ge=1, le=(1 << 32) - 1)]
class RunLimits(WireModel):
cpu_time_us: Annotated[int, Field(strict=True, ge=1000, le=60_000_000)]
wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=120_000_000)]
memory_bytes: Annotated[int, Field(strict=True, ge=1, le=2048 * MIB)]
stack_bytes: Annotated[int, Field(strict=True, ge=1, le=256 * MIB)]
processes: Annotated[int, Field(strict=True, ge=1, le=128)]
open_files: Annotated[int, Field(strict=True, ge=3, le=1024)]
stdout_bytes: Annotated[int, Field(strict=True, ge=1, le=16 * MIB)]
stderr_bytes: Annotated[int, Field(strict=True, ge=1, le=16 * MIB)]
file_bytes: Annotated[int, Field(strict=True, ge=1, le=64 * MIB)]
workdir_bytes: Annotated[int, Field(strict=True, ge=1, le=256 * MIB)]
@model_validator(mode="after")
def consistent(self) -> Self:
if (
self.stack_bytes > self.memory_bytes
or self.file_bytes > self.workdir_bytes
or self.stdout_bytes > self.file_bytes
or self.stderr_bytes > self.file_bytes
or self.stdout_bytes + self.stderr_bytes > self.workdir_bytes
):
raise ValueError("inconsistent resource limits")
return self
def fits(self, ceiling: "RunLimits") -> bool:
return all(
getattr(self, name) <= getattr(ceiling, name) for name in type(self).model_fields
)
class TestCase(WireModel):
case_id: Identifier
input: ArtifactReference
expected: ArtifactReference
class JobManifest(WireModel):
schema_version: Version
kind: Literal["judge.manifest"]
evaluation_id: Identifier
submission_id: Identifier
problem_version_id: Identifier
language: Language
toolchain: Toolchain
source: ArtifactReference
cases: Annotated[tuple[TestCase, ...], Field(min_length=1, max_length=MAX_CASES)]
comparison: Literal["exact_bytes"]
compile_limits: RunLimits
run_limits: RunLimits
job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=MAX_JOB_WALL_US)]
def package_bytes(self) -> int:
# Count references, including duplicates: this bounds actual staged/read bytes.
return self.source.bytes + sum(c.input.bytes + c.expected.bytes for c in self.cases)
@model_validator(mode="after")
def consistent(self) -> Self:
if not 1 <= self.source.bytes <= MAX_SOURCE_BYTES:
raise ValueError("source size outside policy")
if len({c.case_id for c in self.cases}) != len(self.cases):
raise ValueError("duplicate testcase identifiers")
if self.package_bytes() > MAX_PACKAGE_BYTES:
raise ValueError("test package outside policy")
if any(c.expected.bytes > self.run_limits.stdout_bytes for c in self.cases):
raise ValueError("expected output exceeds output budget")
required_wall = (
self.compile_limits.wall_time_us + len(self.cases) * self.run_limits.wall_time_us
)
if required_wall > self.job_wall_time_us:
raise ValueError("job deadline cannot cover execution budgets")
return self
def manifest_bytes(manifest: JobManifest) -> bytes:
"""Stable UTF-8 bytes for persisted snapshots and their SHA-256 identity."""
return json.dumps(
manifest.model_dump(mode="json"),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
allow_nan=False,
).encode("utf-8")
class LeaseBinding(WireModel):
evaluation_id: Identifier
attempt_id: Identifier
worker_id: Identifier
fencing_version: Counter
class LeaseGrant(WireModel):
schema_version: Version
kind: Literal["judge.lease"]
binding: LeaseBinding
lease_token: Digest = Field(repr=False)
attempt_number: Annotated[int, Field(strict=True, ge=1, le=5)]
expires_at_unix_ms: Counter
manifest_digest: Digest
class StageReport(WireModel):
schema_version: Version
kind: Literal["judge.stage"]
binding: LeaseBinding
sequence: Counter
stage: Literal["compiling", "running"]
case_index: Annotated[int, Field(strict=True, ge=0, lt=MAX_CASES)] | None
@model_validator(mode="after")
def consistent(self) -> Self:
if (self.stage == "running") != (self.case_index is not None):
raise ValueError("testcase index only belongs to running phase")
return self
class CompilationReport(WireModel):
status: CompilationStatus
program: ArtifactReference | None
diagnostics: Annotated[str, Field(strict=True, max_length=65536)] = Field(repr=False)
diagnostics_truncated: StrictBool
cpu_time_us: Measurement | None
memory_peak_bytes: Measurement | None
@model_validator(mode="after")
def consistent(self) -> Self:
if len(self.diagnostics.encode("utf-8")) > 65536:
raise ValueError("compiler diagnostics exceed byte budget")
if (self.status == CompilationStatus.SUCCEEDED) != (self.program is not None):
raise ValueError("compiled artifact only belongs to successful compilation")
if self.program is not None and self.program.bytes == 0:
raise ValueError("compiled artifact cannot be empty")
return self
class CaseReport(WireModel):
case_id: Identifier
verdict: Verdict
cpu_time_us: Measurement | None
memory_peak_bytes: Measurement | None
@model_validator(mode="after")
def consistent(self) -> Self:
if self.verdict == Verdict.COMPILATION_ERROR:
raise ValueError("compilation failure is not a testcase outcome")
return self
class JudgementReport(WireModel):
schema_version: Version
kind: Literal["judge.result"]
binding: LeaseBinding
manifest_digest: Digest
compilation: CompilationReport
cases: Annotated[tuple[CaseReport, ...], Field(max_length=MAX_CASES)]
verdict: Verdict
cleanup_complete: StrictBool
failure_code: FailureCode | None
run_cpu_time_us: Measurement | None
run_memory_peak_bytes: Measurement | None
@model_validator(mode="after")
def consistent(self) -> Self:
if len({c.case_id for c in self.cases}) != len(self.cases):
raise ValueError("duplicate testcase results")
if not self.cleanup_complete and self.failure_code != FailureCode.CLEANUP_FAILED:
raise ValueError("incomplete cleanup requires its own failure evidence")
if self.cleanup_complete and self.failure_code == FailureCode.CLEANUP_FAILED:
raise ValueError("cleanup failure cannot claim completed cleanup")
if (self.verdict == Verdict.SYSTEM_ERROR) != (self.failure_code is not None):
raise ValueError("infrastructure failure and verdict must agree")
terminal = next(
(
i
for i, c in enumerate(self.cases)
if c.verdict in (Verdict.SYSTEM_ERROR, Verdict.CANCELLED)
),
None,
)
if terminal is not None and terminal != len(self.cases) - 1:
raise ValueError("results after fatal testcase outcome")
if self.compilation.status != CompilationStatus.SUCCEEDED:
mapped = {
CompilationStatus.COMPILATION_ERROR: Verdict.COMPILATION_ERROR,
CompilationStatus.COMPILATION_LIMIT: Verdict.COMPILATION_ERROR,
CompilationStatus.CANCELLED: Verdict.CANCELLED,
CompilationStatus.SYSTEM_ERROR: Verdict.SYSTEM_ERROR,
}[self.compilation.status]
if self.cases or self.verdict != mapped:
raise ValueError("failed compilation cannot have runtime results")
elif self.failure_code is None:
if any(c.verdict == Verdict.SYSTEM_ERROR for c in self.cases):
raise ValueError("infrastructure failure requires failure code")
if not self.cases and self.verdict != Verdict.CANCELLED:
raise ValueError("missing testcase results cannot be a completed judgement")
if self.cases and self.verdict != Verdict.CANCELLED:
failures = (Verdict.SYSTEM_ERROR, Verdict.CANCELLED)
summary = next(
(c.verdict for c in self.cases if c.verdict in failures),
next(
(c.verdict for c in self.cases if c.verdict != Verdict.ACCEPTED),
Verdict.ACCEPTED,
),
)
if self.verdict != summary:
raise ValueError("testcase summary disagrees")
cpu = (
sum(c.cpu_time_us for c in self.cases if c.cpu_time_us is not None)
if self.cases and all(c.cpu_time_us is not None for c in self.cases)
else None
)
memory = (
max(c.memory_peak_bytes for c in self.cases if c.memory_peak_bytes is not None)
if self.cases and all(c.memory_peak_bytes is not None for c in self.cases)
else None
)
if self.run_cpu_time_us != cpu or self.run_memory_peak_bytes != memory:
raise ValueError("runtime measurement summary disagrees")
return self
def validate_for_manifest(self, manifest: JobManifest, digest: str) -> None:
if self.binding.evaluation_id != manifest.evaluation_id or self.manifest_digest != digest:
raise ValueError("report does not belong to authorized manifest")
if [c.case_id for c in self.cases] != [
c.case_id for c in manifest.cases[: len(self.cases)]
]:
raise ValueError("testcase results differ from authorized order")
if self.verdict not in (Verdict.COMPILATION_ERROR, Verdict.CANCELLED, Verdict.SYSTEM_ERROR):
if len(self.cases) != len(manifest.cases):
raise ValueError("incomplete ordinary judgement")
class AdmissionPolicy(WireModel):
"""Trusted node policy. A manifest never supplies or replaces this policy."""
max_concurrent_jobs: Annotated[int, Field(strict=True, ge=1, le=64)] = 1
prefetch: Annotated[int, Field(strict=True, ge=1, le=64)] = 1
max_retries: Annotated[int, Field(strict=True, ge=0, le=4)] = 2
max_job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=MAX_JOB_WALL_US)]
max_package_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_PACKAGE_BYTES)]
toolchain: Toolchain
compile_ceiling: RunLimits
run_ceiling: RunLimits
@model_validator(mode="after")
def consistent(self) -> Self:
if self.prefetch > self.max_concurrent_jobs:
raise ValueError("prefetch exceeds available job slots")
return self
def admit(self, manifest: JobManifest, attempt_number: int) -> None:
if type(attempt_number) is not int or not 1 <= attempt_number <= self.max_retries + 1:
raise ValueError("attempt retry budget exhausted")
if (
manifest.toolchain != self.toolchain
or not manifest.compile_limits.fits(self.compile_ceiling)
or not manifest.run_limits.fits(self.run_ceiling)
or manifest.job_wall_time_us > self.max_job_wall_time_us
or manifest.package_bytes() > self.max_package_bytes
):
raise ValueError("job exceeds trusted node budget")
class StoppedReport(WireModel):
schema_version: Version
kind: Literal["judge.stopped"]
binding: LeaseBinding
cleanup_complete: StrictBool
WIRE_MODELS = {
"judge-manifest": JobManifest,
"judge-lease": LeaseGrant,
"judge-stage": StageReport,
"judge-result": JudgementReport,
"judge-admission": AdmissionPolicy,
"judge-stopped": StoppedReport,
}
def decode_wire[T: BaseModel](model: type[T], payload: bytes) -> T:
"""UTF-8, bounded size and no ambiguous duplicate members before model validation."""
if len(payload) > MAX_WIRE_BYTES:
raise ValueError("wire payload exceeds byte budget")
def unique_members(pairs: list[tuple[str, object]]) -> dict[str, object]:
result = {}
for key, value in pairs:
if key in result:
raise ValueError("duplicate JSON member")
result[key] = value
return result
def invalid_constant(_: str) -> None:
raise ValueError("invalid JSON constant")
try:
value = json.loads(
payload.decode("utf-8"),
object_pairs_hook=unique_members,
parse_constant=invalid_constant,
)
return model.model_validate(value)
except (ValueError, TypeError):
# Errors at the transport boundary must not echo credentials or hidden inputs.
raise ValueError("invalid wire payload") from None
backend/src/kf_oj/contracts/judging_v2.py
-358
View File
@@ -1,358 +0,0 @@
"""Explicit multi-runtime protocol; existing v1 snapshots remain byte-for-byte stable."""
import re
from enum import StrEnum
from typing import Annotated, Literal, Self
from pydantic import BeforeValidator, Field, model_validator
from kf_oj.contracts import judging as v1
Version = Annotated[Literal[2], BeforeValidator(v1.integer_version)]
MAX_CLASS_FILES = 256
JVM_NATIVE_RESERVE_BYTES = 160 * v1.MIB
JVM_THREAD_STACK_BYTES = 256 * 1024
class Language(StrEnum):
C17 = "c17"
CPP20 = "cpp20"
PYTHON313 = "python313"
JAVA21 = "java21"
class PreparationKind(StrEnum):
COMPILE = "compile"
SYNTAX_CHECK = "syntax_check"
class ProgramKind(StrEnum):
NATIVE = "native_executable"
PYTHON = "python_source"
JAVA = "java_classes"
class RuntimeIdentity(v1.WireModel):
"""An exact identity matched against trusted configuration, never a host path."""
toolchain_id: v1.Label
image_kind: Literal[
"linux_gnu_cxx_x86_64_v1", "linux_cpython313_x86_64_v1", "linux_openjdk21_x86_64_v1"
]
image_revision: Annotated[int, Field(strict=True, ge=1, le=(1 << 32) - 1)]
image_digest: v1.Digest
runtime_version: Annotated[
str, Field(strict=True, pattern=r"^[A-Za-z0-9][A-Za-z0-9_.+-]{0,63}$")
]
def runtime_contract(language: Language) -> tuple[str, str, str, ProgramKind, PreparationKind]:
"""Fixed source/profile names. Runtime callers cannot introduce argv or paths."""
return {
Language.C17: (
"main.c",
"compile_c",
"native",
ProgramKind.NATIVE,
PreparationKind.COMPILE,
),
Language.CPP20: (
"main.cpp",
"compile_cpp",
"native",
ProgramKind.NATIVE,
PreparationKind.COMPILE,
),
Language.PYTHON313: (
"main.py",
"python_check",
"python_run",
ProgramKind.PYTHON,
PreparationKind.SYNTAX_CHECK,
),
Language.JAVA21: (
"Main.java",
"java_compile",
"java_run",
ProgramKind.JAVA,
PreparationKind.COMPILE,
),
}[language]
def validate_runtime(language: Language, identity: RuntimeIdentity) -> None:
expected = {
Language.C17: ("gnu-cxx-v1", "linux_gnu_cxx_x86_64_v1"),
Language.CPP20: ("gnu-cxx-v1", "linux_gnu_cxx_x86_64_v1"),
Language.PYTHON313: ("cpython313-v1", "linux_cpython313_x86_64_v1"),
Language.JAVA21: ("openjdk21-v1", "linux_openjdk21_x86_64_v1"),
}[language]
if (identity.toolchain_id, identity.image_kind) != expected:
raise ValueError("runtime family differs from language contract")
if language == Language.PYTHON313 and identity.runtime_version != "3.13.16":
raise ValueError("unsupported CPython version")
if language == Language.JAVA21 and not re.fullmatch(
r"21\.[0-9]+\.[0-9]+(?:\.[0-9]+)?\+[1-9][0-9]*(?:-LTS)?", identity.runtime_version
):
raise ValueError("JDK version must identify a fixed Java 21 build")
def validate_startup_budget(language: Language, limits: v1.RunLimits, *, preparing: bool) -> None:
"""Admission floors, not multipliers on problem limits; real execution still required."""
if language == Language.PYTHON313:
floor = (10_000, 50_000, 32 * v1.MIB, 1, 16)
elif language == Language.JAVA21:
floor = (
(1_000_000, 3_000_000, 384 * v1.MIB, 16, 32)
if preparing
else (200_000, 1_000_000, 256 * v1.MIB, 16, 32)
)
else:
return
actual = (
limits.cpu_time_us,
limits.wall_time_us,
limits.memory_bytes,
limits.processes,
limits.open_files,
)
if any(value < minimum for value, minimum in zip(actual, floor, strict=True)):
raise ValueError("budget below runtime startup policy")
if language == Language.JAVA21:
jvm_heap_bytes(limits)
def jvm_heap_bytes(limits: v1.RunLimits) -> int:
"""Reserve capped metaspace/code/native and every allowed thread stack inside cgroup."""
reserve = JVM_NATIVE_RESERVE_BYTES + limits.processes * JVM_THREAD_STACK_BYTES
heap = (limits.memory_bytes - reserve) // v1.MIB * v1.MIB
if heap < 32 * v1.MIB or limits.stack_bytes < JVM_THREAD_STACK_BYTES:
raise ValueError("budget cannot cover JVM heap and explicit non-heap reserve")
return heap
class LanguageConfiguration(v1.WireModel):
language: Language
toolchain: RuntimeIdentity
compile_limits: v1.RunLimits
run_limits: v1.RunLimits
job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=v1.MAX_JOB_WALL_US)]
@model_validator(mode="after")
def runtime_and_budget(self) -> Self:
validate_runtime(self.language, self.toolchain)
validate_startup_budget(self.language, self.compile_limits, preparing=True)
validate_startup_budget(self.language, self.run_limits, preparing=False)
if self.compile_limits.wall_time_us + self.run_limits.wall_time_us > self.job_wall_time_us:
raise ValueError("job deadline cannot cover preparation and execution")
return self
class JobManifest(v1.JobManifest):
schema_version: Version
language: Language
toolchain: RuntimeIdentity
@model_validator(mode="after")
def runtime_and_budget(self) -> Self:
validate_runtime(self.language, self.toolchain)
validate_startup_budget(self.language, self.compile_limits, preparing=True)
validate_startup_budget(self.language, self.run_limits, preparing=False)
return self
class ProgramFile(v1.WireModel):
path: Annotated[str, Field(strict=True, min_length=1, max_length=128)]
artifact: v1.ArtifactReference
class CompiledProgram(v1.WireModel):
"""Sealed file identities bound to a particular source, runtime and lease attempt."""
kind: ProgramKind
binding: v1.LeaseBinding
source: v1.ArtifactReference
toolchain: RuntimeIdentity
files: Annotated[tuple[ProgramFile, ...], Field(min_length=1, max_length=MAX_CLASS_FILES)]
@property
def bytes(self) -> int:
return sum(file.artifact.bytes for file in self.files)
@model_validator(mode="after")
def bounded_files(self) -> Self:
names = [file.path for file in self.files]
if names != sorted(set(names)):
raise ValueError("program files must have unique canonical order")
if not 1 <= self.source.bytes <= v1.MAX_SOURCE_BYTES:
raise ValueError("program source outside policy")
if self.bytes > v1.MAX_PACKAGE_BYTES or any(
file.artifact.bytes == 0 for file in self.files
):
raise ValueError("program files empty or outside byte policy")
if self.kind == ProgramKind.NATIVE:
validate_runtime(Language.C17, self.toolchain)
if names != ["main"] or self.toolchain.image_kind != "linux_gnu_cxx_x86_64_v1":
raise ValueError("native program requires its fixed executable and runtime")
elif self.kind == ProgramKind.PYTHON:
validate_runtime(Language.PYTHON313, self.toolchain)
if names != ["main.py"] or self.files[0].artifact != self.source:
raise ValueError("Python program must be the exact checked source")
else:
validate_runtime(Language.JAVA21, self.toolchain)
if "Main.class" not in names or any(
not re.fullmatch(r"[A-Za-z_$][A-Za-z0-9_$]*\.class", name) for name in names
):
raise ValueError("Java requires a bounded default-package Main class set")
return self
class CompilationReport(v1.CompilationReport):
program: CompiledProgram | None
class JudgementReport(v1.JudgementReport):
schema_version: Version
compilation: CompilationReport
@model_validator(mode="after")
def program_attempt(self) -> Self:
if (
self.compilation.program is not None
and self.compilation.program.binding != self.binding
):
raise ValueError("program belongs to another attempt")
return self
def validate_for_manifest(self, manifest: JobManifest, digest: str) -> None:
super().validate_for_manifest(manifest, digest)
program = self.compilation.program
if program is not None and (
program.source != manifest.source
or program.toolchain != manifest.toolchain
or program.kind != runtime_contract(manifest.language)[3]
):
raise ValueError("program differs from authorized source or runtime")
class LeaseGrant(v1.LeaseGrant):
schema_version: Version
class StageReport(v1.StageReport):
schema_version: Version
preparation_kind: PreparationKind
def validate_for_manifest(self, manifest: JobManifest) -> None:
if self.binding.evaluation_id != manifest.evaluation_id or (
self.preparation_kind != runtime_contract(manifest.language)[4]
):
raise ValueError("stage differs from authorized language preparation")
class StoppedReport(v1.StoppedReport):
schema_version: Version
class RuntimeCapability(v1.WireModel):
language: Language
toolchain: RuntimeIdentity
compile_ceiling: v1.RunLimits
run_ceiling: v1.RunLimits
@model_validator(mode="after")
def runtime_and_budget(self) -> Self:
validate_runtime(self.language, self.toolchain)
validate_startup_budget(self.language, self.compile_ceiling, preparing=True)
validate_startup_budget(self.language, self.run_ceiling, preparing=False)
return self
class AdmissionPolicy(v1.WireModel):
schema_version: Version
max_concurrent_jobs: Annotated[int, Field(strict=True, ge=1, le=64)] = 1
prefetch: Annotated[int, Field(strict=True, ge=1, le=64)] = 1
max_retries: Annotated[int, Field(strict=True, ge=0, le=4)] = 2
max_job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=v1.MAX_JOB_WALL_US)]
max_package_bytes: Annotated[int, Field(strict=True, ge=1, le=v1.MAX_PACKAGE_BYTES)]
capabilities: Annotated[tuple[RuntimeCapability, ...], Field(min_length=1, max_length=4)]
@model_validator(mode="after")
def consistent(self) -> Self:
if self.prefetch > self.max_concurrent_jobs:
raise ValueError("prefetch exceeds available job slots")
if len({cap.language for cap in self.capabilities}) != len(self.capabilities):
raise ValueError("duplicate language capability")
return self
def admit(self, manifest: JobManifest, attempt_number: int) -> None:
# Revalidate constructed/copied values as well as transport-decoded models.
manifest = JobManifest.model_validate(manifest.model_dump(mode="json"))
AdmissionPolicy.model_validate(self.model_dump(mode="json"))
if type(attempt_number) is not int or not 1 <= attempt_number <= self.max_retries + 1:
raise ValueError("attempt retry budget exhausted")
cap = next((cap for cap in self.capabilities if cap.language == manifest.language), None)
if cap is None or (
cap.toolchain != manifest.toolchain
or not manifest.compile_limits.fits(cap.compile_ceiling)
or not manifest.run_limits.fits(cap.run_ceiling)
or manifest.job_wall_time_us > self.max_job_wall_time_us
or manifest.package_bytes() > self.max_package_bytes
):
raise ValueError("job exceeds trusted runtime capability or budget")
class AdmissionRegistry(v1.WireModel):
"""Explicit operator opt-in for both protocol generations; no silent conversion."""
schema_version: Version
legacy: v1.AdmissionPolicy | None
multi_runtime: AdmissionPolicy | None
@model_validator(mode="after")
def at_least_one_protocol(self) -> Self:
if self.legacy is None and self.multi_runtime is None:
raise ValueError("node must advertise at least one explicit protocol capability")
if self.legacy is not None and self.multi_runtime is not None:
for field in ("max_concurrent_jobs", "prefetch", "max_retries"):
if getattr(self.legacy, field) != getattr(self.multi_runtime, field):
raise ValueError("protocol policies disagree on node capacity or retries")
return self
def admit(self, manifest: v1.JobManifest | JobManifest, attempt_number: int) -> None:
AdmissionRegistry.model_validate(self.model_dump(mode="json"))
policy = self.legacy if manifest.schema_version == 1 else self.multi_runtime
admit_manifest(policy, manifest, attempt_number)
NodeAdmission = v1.AdmissionPolicy | AdmissionPolicy | AdmissionRegistry
def admit_manifest(
policy: NodeAdmission | None, manifest: v1.JobManifest | JobManifest, attempt_number: int
) -> None:
if type(manifest) not in (v1.JobManifest, JobManifest):
raise ValueError("unsupported manifest model")
# Copied/constructed values must pass the original strict protocol validators too.
manifest = type(manifest).model_validate(manifest.model_dump(mode="json"))
if isinstance(policy, AdmissionRegistry):
policy.admit(manifest, attempt_number)
elif manifest.schema_version == 1 and type(policy) is v1.AdmissionPolicy:
v1.AdmissionPolicy.model_validate(policy.model_dump(mode="json")).admit(
manifest, attempt_number
)
elif manifest.schema_version == 2 and type(policy) is AdmissionPolicy:
policy.admit(manifest, attempt_number)
else:
raise ValueError("node does not advertise this manifest protocol")
WIRE_MODELS = {
"judge-manifest": JobManifest,
"judge-lease": LeaseGrant,
"judge-stage": StageReport,
"judge-result": JudgementReport,
"judge-admission": AdmissionPolicy,
"judge-stopped": StoppedReport,
"judge-registry": AdmissionRegistry,
}
backend/src/kf_oj/contracts/judging_versions.py
-35
View File
@@ -1,35 +0,0 @@
"""Version dispatch at storage/transport boundaries; never rewrite legacy snapshots."""
from pydantic import ConfigDict, RootModel
from kf_oj.contracts import judging as v1
from kf_oj.contracts import judging_v2 as v2
Manifest = v1.JobManifest | v2.JobManifest
Report = v1.JudgementReport | v2.JudgementReport
class ManifestEnvelope(RootModel[Manifest]):
model_config = ConfigDict(frozen=True)
class ReportEnvelope(RootModel[Report]):
model_config = ConfigDict(frozen=True)
def decode_manifest(payload: bytes) -> Manifest:
return v1.decode_wire(ManifestEnvelope, payload).root
def decode_report(payload: bytes) -> Report:
return v1.decode_wire(ReportEnvelope, payload).root
def manifest_bytes(manifest: Manifest) -> bytes:
return v1.manifest_bytes(manifest)
def validate_report(report: Report, manifest: Manifest, digest: str) -> None:
if report.schema_version != manifest.schema_version:
raise ValueError("report protocol differs from authorized manifest")
report.validate_for_manifest(manifest, digest)
backend/src/kf_oj/contracts/problem_maintenance.py
-103
View File
@@ -1,103 +0,0 @@
"""Bounded administrator projections and immutable operation receipts, separate from judging."""
from datetime import datetime
from typing import Annotated, Literal
from pydantic import Field, StrictBool
from kf_oj.contracts.admin_problems import CandidatePreview
from kf_oj.contracts.judging import Identifier, Label, Version, WireModel
from kf_oj.contracts.problems import ProblemSummary
MaintenanceAction = Literal["publish", "disable", "restore"]
AuditAction = Literal[
"upload_created",
"upload_received",
"import_ready",
"import_failed",
"import_discarded",
"import_expired",
"published",
"disabled",
"restored",
"authority_bootstrap",
"authority_grant",
"authority_revoke",
"authority_enable",
"authority_disable",
]
class MaintainedProblem(WireModel):
id: Identifier
slug: Label
enabled: StrictBool
active_version_id: Identifier | None
revision: Annotated[int, Field(strict=True, ge=0)]
title: Annotated[str | None, Field(max_length=160)]
created_at: datetime
class MaintainedProblemPage(WireModel):
schema_version: Version = 1
items: Annotated[tuple[MaintainedProblem, ...], Field(max_length=100)]
next_after: Label | None
class MaintainedVersion(ProblemSummary):
published_at: datetime
case_count: Annotated[int, Field(strict=True, ge=1, le=256)]
class MaintainedVersionPage(WireModel):
schema_version: Version = 1
items: Annotated[tuple[MaintainedVersion, ...], Field(max_length=100)]
next_before: Annotated[int | None, Field(strict=True, ge=1)]
class PublishCandidate(WireModel):
schema_version: Version = 1
operation_id: Identifier
import_operation_id: Identifier
candidate_id: Identifier
expected_active_version_id: Identifier | None
class ChangeProblemAvailability(WireModel):
schema_version: Version = 1
operation_id: Identifier
expected_active_version_id: Identifier
expected_enabled: StrictBool
class MaintenanceReceipt(WireModel):
schema_version: Version = 1
operation_id: Identifier
action: MaintenanceAction
completed_at: datetime
problem: MaintainedProblem
published_version_id: Identifier | None = None
class MaintenanceAuditEvent(WireModel):
id: Identifier
created_at: datetime
actor_kind: Literal["account", "operator"]
actor_id: Identifier | None
operator_name: Annotated[str | None, Field(max_length=63)]
authorization_version: Annotated[str, Field(pattern=r"^[1-9][0-9]{0,18}$")]
operation_id: Identifier
action: AuditAction
target_id: Identifier | None
result: Literal["committed"] = "committed"
class MaintenanceAuditPage(WireModel):
schema_version: Version = 1
items: Annotated[tuple[MaintenanceAuditEvent, ...], Field(max_length=100)]
next_before: Annotated[str | None, Field(max_length=100)]
# This alias deliberately uses the existing sanitized candidate contract. No new
# endpoint exposes the private manifest, artifact digest or hidden case names.
PublicCandidate = CandidatePreview
backend/src/kf_oj/contracts/problems.py
-58
View File
@@ -1,58 +0,0 @@
from typing import Annotated, Literal
from pydantic import Field
from kf_oj.contracts.judging import Identifier, Label, RunLimits, WireModel
from kf_oj.contracts.judging_v2 import Language, PreparationKind, runtime_contract
class PublicSample(WireModel):
encoding: Literal["base64"] = "base64"
input: Annotated[str, Field(max_length=87384)]
output: Annotated[str, Field(max_length=87384)]
class ProblemSummary(WireModel):
id: Identifier
slug: Label
version_id: Identifier
revision: int
title: str
languages: tuple[Language, ...]
class PublicLanguageConfiguration(WireModel):
language: Language
runtime_version: str
source_name: str
entrypoint: str
preparation_kind: PreparationKind
compile_limits: RunLimits
run_limits: RunLimits
job_wall_time_us: int
@classmethod
def from_configuration(cls, configuration):
source, _, _, _, preparation = runtime_contract(configuration.language)
return cls(
language=configuration.language,
runtime_version=configuration.toolchain.runtime_version,
source_name=source,
entrypoint="Main" if configuration.language == Language.JAVA21 else source,
preparation_kind=preparation,
compile_limits=configuration.compile_limits,
run_limits=configuration.run_limits,
job_wall_time_us=configuration.job_wall_time_us,
)
class ProblemDetail(ProblemSummary):
statement: str
samples: tuple[PublicSample, ...]
run_limits: RunLimits | None
language_configurations: tuple[PublicLanguageConfiguration, ...] = ()
class ProblemPage(WireModel):
items: tuple[ProblemSummary, ...]
next_after: Label | None
backend/src/kf_oj/contracts/restore_io.py
-129
View File
@@ -1,129 +0,0 @@
"""Private offline restore byte-copy protocol; conveys no execution authority."""
import re
from typing import Annotated, Literal
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
RootModel,
StrictInt,
model_validator,
)
from kf_oj.contracts.judging import integer_version
MAX_FILES = 98304
MAX_LOGICAL_BYTES = 12 * 1024**3
MAX_FILE_BYTES = 256 * 1024**2
MAX_BATCH_BYTES = 64 * 1024**2
DOMAINS = frozenset(
"postgres artifacts imports app-config judge-config factory operations releases "
"images dependencies proxy recovery".split()
)
Count = Annotated[StrictInt, Field(ge=0, le=MAX_FILES)]
ByteCount = Annotated[StrictInt, Field(ge=0, le=MAX_LOGICAL_BYTES)]
class Record(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
class RestoreFile(Record):
path: Annotated[str, Field(min_length=1, max_length=512)]
uid: Annotated[StrictInt, Field(ge=0, le=1205)]
gid: Annotated[StrictInt, Field(ge=0, le=1205)]
mode: StrictInt
bytes: Annotated[StrictInt, Field(ge=0, le=MAX_FILE_BYTES)]
sha256: Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
@model_validator(mode="after")
def check_member(self) -> "RestoreFile":
parts = self.path.split("/")
if (
len(parts) < 2
or parts[0] not in DOMAINS
or any(
p in {"", ".", "..", ".git", ".ssh", ".localplan", "docs"}
or re.fullmatch(r"[A-Za-z0-9_.+@-]{1,255}", p) is None
for p in parts
)
):
raise ValueError("restore file must be below a fixed backup domain")
if self.uid not in {0, 1200, 1201, 1202, 1203, 1204, 1205} or self.gid not in (
{0, 1203, 1204, 1205} if self.uid == 0 else {self.uid}
):
raise ValueError("unsupported restore identity")
if self.mode not in {0o400, 0o444, 0o600, 0o640, 0o644, 0o555, 0o755}:
raise ValueError("unsupported restore file mode")
if self.sha256 in {"a" * 64, "0" * 64}:
raise ValueError("placeholder digest refused")
return self
class Open(Record):
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
kind: Literal["restore.io.open"]
backup_id: Annotated[str, Field(pattern=r"^bk-[0-9a-f]{32}$")]
recovery_id: Annotated[str, Field(pattern=r"^rr-[0-9a-f]{32}$")]
expected_files: Annotated[StrictInt, Field(ge=1, le=MAX_FILES)]
expected_logical_bytes: ByteCount
deadline_monotonic_ns: Annotated[StrictInt, Field(ge=1, le=2**63 - 1)]
class Batch(Record):
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
kind: Literal["restore.io.batch"]
batch_id: Annotated[StrictInt, Field(ge=1, le=MAX_FILES)]
files: Annotated[list[RestoreFile], Field(min_length=1, max_length=256)]
@model_validator(mode="after")
def check_batch(self) -> "Batch":
if len(self.files) > 1 and sum(f.bytes for f in self.files) > MAX_BATCH_BYTES:
raise ValueError("restore batch byte bound exceeded")
if len({f.path for f in self.files}) != len(self.files):
raise ValueError("duplicate restore batch member")
return self
class Finish(Record):
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
kind: Literal["restore.io.finish"]
class RestoreIORequest(RootModel[Annotated[Open | Batch | Finish, Field(discriminator="kind")]]):
pass
class RestoreIOReply(Record):
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
kind: Literal["restore.io.reply"]
stage: Literal["opened", "verified", "finished", "refused"]
batch_id: Count
verified_files: Count
verified_logical_bytes: ByteCount
execution_verified: Annotated[
Literal[False],
BeforeValidator(lambda value: value if type(value) is bool else _bad_boolean()),
]
error: Literal["none", "protocol", "guard", "deadline", "filesystem", "bytes", "identity"]
@model_validator(mode="after")
def check_reply(self) -> "RestoreIOReply":
if (self.stage == "refused") != (self.error != "none"):
raise ValueError("restore refusal state differs")
if self.stage == "opened" and (
self.batch_id or self.verified_files or self.verified_logical_bytes
):
raise ValueError("restore open cannot claim verified data")
if self.stage in {"verified", "finished"} and not (
1 <= self.batch_id <= self.verified_files
):
raise ValueError("restore completion has no verified batch")
return self
def _bad_boolean() -> bool:
raise ValueError("execution verification must be a boolean false")
backend/src/kf_oj/contracts/submission_performance.py
-126
View File
@@ -1,126 +0,0 @@
"""Bounded anonymous aggregates; raw measurements never pass through JS floats."""
from datetime import datetime
from typing import Annotated, Literal, Self
from pydantic import AfterValidator, Field, StrictBool, model_validator
from kf_oj.contracts.judging import I64_MAX, Digest, Identifier, Version, WireModel
from kf_oj.contracts.judging_v2 import Language
MAX_PERFORMANCE_SAMPLES = 10_000
MIN_PERFORMANCE_SAMPLES = 5
MAX_PERFORMANCE_BUCKETS = 64
def bounded_decimal(value: str) -> str:
if int(value) > I64_MAX:
raise ValueError("measurement exceeds signed storage range")
return value
DecimalMeasurement = Annotated[
str,
Field(strict=True, min_length=1, max_length=19, pattern=r"^(0|[1-9][0-9]*)$"),
AfterValidator(bounded_decimal),
]
SampleCount = Annotated[int, Field(strict=True, ge=0, le=MAX_PERFORMANCE_SAMPLES)]
UnavailableReason = Literal[
"not_accepted", "measurement_missing", "environment_unverified", "insufficient_samples"
]
class PerformanceBucket(WireModel):
start: DecimalMeasurement
end: DecimalMeasurement
upper_inclusive: StrictBool
count: SampleCount
@model_validator(mode="after")
def interval(self) -> Self:
if int(self.end) <= int(self.start):
raise ValueError("empty or reversed interval")
return self
class PerformanceMetric(WireModel):
unit: Literal["cpu_microseconds", "bytes"]
current: DecimalMeasurement | None
available: StrictBool
reason: UnavailableReason | None
sample_count: SampleCount
window_truncated: StrictBool
oldest_sample_at: datetime | None
# These counts are computed against the original integers, never rounded bins.
strictly_slower_count: SampleCount | None
tied_count: SampleCount | None
buckets: Annotated[tuple[PerformanceBucket, ...], Field(max_length=MAX_PERFORMANCE_BUCKETS)]
@model_validator(mode="after")
def aggregate(self) -> Self:
if self.oldest_sample_at is not None and self.oldest_sample_at.utcoffset() is None:
raise ValueError("sample timestamp must have a timezone")
if self.available:
if (
self.reason is not None
or self.current is None
or self.sample_count < MIN_PERFORMANCE_SAMPLES
or self.strictly_slower_count is None
or self.tied_count is None
or self.oldest_sample_at is None
or not self.buckets
):
raise ValueError("incomplete available aggregate")
if self.strictly_slower_count + self.tied_count > self.sample_count:
raise ValueError("comparison counts exceed the cohort")
if sum(b.count for b in self.buckets) != self.sample_count:
raise ValueError("histogram loses samples")
if int(self.buckets[0].start) != 0:
raise ValueError("histogram must include zero")
for index, bucket in enumerate(self.buckets):
if bucket.upper_inclusive != (index == len(self.buckets) - 1):
raise ValueError("only the last interval includes its upper endpoint")
if index and self.buckets[index - 1].end != bucket.start:
raise ValueError("histogram intervals must be contiguous")
elif (
self.reason is None
or self.buckets
or self.strictly_slower_count is not None
or self.tied_count is not None
):
raise ValueError("unavailable aggregate cannot expose comparison data")
if self.reason == "measurement_missing" and self.current is not None:
raise ValueError("missing measurement cannot claim a value")
if self.reason == "insufficient_samples" and (
self.sample_count >= MIN_PERFORMANCE_SAMPLES or self.current is None
):
raise ValueError("inconsistent insufficient cohort")
return self
class SubmissionPerformance(WireModel):
schema_version: Version = 1
kind: Literal["judge.performance"] = "judge.performance"
submission_id: Identifier
problem_version_id: Identifier
language: Language
as_of: datetime
# Opaque trusted comparison identity, not a worker name, path or credential.
comparison_group: Digest | None
window_limit: Literal[10_000] = MAX_PERFORMANCE_SAMPLES
minimum_samples: Literal[5] = MIN_PERFORMANCE_SAMPLES
cpu: PerformanceMetric
memory: PerformanceMetric
@model_validator(mode="after")
def snapshot(self) -> Self:
if self.as_of.utcoffset() is None:
raise ValueError("snapshot timestamp must have a timezone")
if self.cpu.unit != "cpu_microseconds" or self.memory.unit != "bytes":
raise ValueError("metric unit mismatch")
for metric in (self.cpu, self.memory):
if metric.available and self.comparison_group is None:
raise ValueError("comparison requires trusted environment binding")
if metric.oldest_sample_at is not None and metric.oldest_sample_at > self.as_of:
raise ValueError("sample is newer than the snapshot")
return self
backend/src/kf_oj/contracts/submissions.py
-92
View File
@@ -1,92 +0,0 @@
from datetime import datetime
from typing import Annotated, Literal
from pydantic import Field, model_validator
from kf_oj.contracts.judging import (
MAX_SOURCE_BYTES,
CompilationStatus,
FailureCode,
Identifier,
Measurement,
Verdict,
WireModel,
)
from kf_oj.contracts.judging_v2 import Language
EvaluationState = Literal[
"queued", "claimed", "compiling", "running", "cancelling", "finished", "cancelled", "failed"
]
class CreateSubmission(WireModel):
problem_version_id: Identifier
language: Language
idempotency_key: Identifier
source: Annotated[
str, Field(strict=True, min_length=1, max_length=MAX_SOURCE_BYTES, repr=False)
]
@model_validator(mode="after")
def bounded_source(self):
try:
if len(self.source.encode("utf-8")) > MAX_SOURCE_BYTES:
raise ValueError("source exceeds byte policy")
except UnicodeError:
raise ValueError("source must be UTF-8") from None
return self
class SubmissionCreated(WireModel):
submission_id: Identifier
evaluation_id: Identifier
problem_version_id: Identifier
language: Language
state: EvaluationState
created_at: datetime
class SubmissionSummary(SubmissionCreated):
updated_at: datetime
attempts: int
verdict: Verdict | None
failure_code: FailureCode | None
class PublicCaseResult(WireModel):
position: int
verdict: Verdict
cpu_time_us: Measurement | None
memory_peak_bytes: Measurement | None
class PublicJudgement(WireModel):
compilation_status: CompilationStatus
diagnostics: Annotated[str, Field(max_length=65536)]
diagnostics_truncated: bool
cpu_time_us: Measurement | None
memory_peak_bytes: Measurement | None
cleanup_complete: bool
cases: tuple[PublicCaseResult, ...]
class SubmissionDetail(SubmissionSummary):
result: PublicJudgement | None
class SubmissionPage(WireModel):
items: tuple[SubmissionSummary, ...]
next_after: str | None
class SubmissionSource(WireModel):
submission_id: Identifier
language: Language
source: Annotated[str, Field(max_length=MAX_SOURCE_BYTES, repr=False)]
class CancellationAccepted(WireModel):
submission_id: Identifier
evaluation_id: Identifier
# Acceptance means Coordinator durably accepted intent, not that processes have stopped.
accepted: Literal[True] = True
backend/src/kf_oj/core/logging.py
+2 -48
View File
@@ -16,53 +16,7 @@ class JsonFormatter(logging.Formatter):
return json.dumps(payload, ensure_ascii=False)
class ProductionFormatter(logging.Formatter):
"""Closed event vocabulary; arbitrary messages/arguments/tracebacks are private data."""
EVENTS = {
("kf_oj.api", "Platform API started"): "api_started",
("kf_oj.api", "Platform API stopped"): "api_stopped",
("kf_oj.outbox", "outbox broker unavailable"): "outbox_broker_unavailable",
("kf_oj.outbox", "outbox database unavailable"): "outbox_database_unavailable",
}
LOGGERS = {"kf_oj.api", "kf_oj.outbox", "uvicorn", "uvicorn.error", "uvicorn.access"}
LEVELS = {
logging.DEBUG: "DEBUG",
logging.INFO: "INFO",
logging.WARNING: "WARNING",
logging.ERROR: "ERROR",
logging.CRITICAL: "CRITICAL",
}
def format(self, record: logging.LogRecord) -> str:
# Do not call getMessage/formatException: library errors can contain
# connection URIs, query parameters, source or diagnostic body bytes.
event = "infrastructure_event"
if isinstance(record.msg, str) and not record.args:
event = self.EVENTS.get((record.name, record.msg), event)
return json.dumps(
{
"timestamp": datetime.fromtimestamp(record.created, UTC).isoformat(),
"level": self.LEVELS.get(record.levelno, "UNKNOWN"),
"logger": record.name if record.name in self.LOGGERS else "infrastructure",
"message": event,
"exception_present": record.exc_info is not None,
},
ensure_ascii=False,
)
def configure_logging(level: str, *, production: bool = False) -> None:
def configure_logging(level: str) -> None:
handler = logging.StreamHandler()
handler.setFormatter(ProductionFormatter() if production else JsonFormatter())
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=level, handlers=[handler], force=True)
if production:
# Uvicorn installs its own handlers before lifespan, which otherwise
# bypass the root formatter when reporting an unhandled application error.
for name in ("uvicorn", "uvicorn.error"):
logger = logging.getLogger(name)
logger.handlers.clear()
logger.propagate = True
access = logging.getLogger("uvicorn.access")
access.handlers.clear()
access.propagate = False
backend/src/kf_oj/core/settings.py
+1 -71
View File
@@ -1,11 +1,8 @@
from pathlib import Path
from typing import Literal
from pydantic import Field, SecretStr, model_validator
from pydantic import SecretStr
from pydantic_settings import BaseSettings, SettingsConfigDict
from kf_oj.contracts.judging_v2 import NodeAdmission
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_prefix="KF_OJ_", env_file=".env", extra="ignore")
@@ -13,71 +10,4 @@ class Settings(BaseSettings):
environment: Literal["development", "test", "production"] = "development"
log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
database_url: SecretStr | None = None
identity_operator_database_url: SecretStr | None = None
amqp_url: SecretStr | None = None
browser_origins: tuple[str, ...] = (
"http://localhost:5173",
"http://localhost:8000",
"http://127.0.0.1:5173",
"http://127.0.0.1:8000",
)
session_lifetime_seconds: int = Field(default=43200, ge=300, le=86400)
artifact_root: Path | None = None
import_staging_root: Path | None = None
import_worker_cgroup: Path | None = None
artifact_max_bytes: int = Field(default=536870912, ge=1, le=16 * 1024**3)
artifact_max_files: int = Field(default=4096, ge=1, le=65536)
judge_admission: NodeAdmission | None = None
coordinator_url: str | None = None
coordinator_token: SecretStr | None = None
delivery_release_id: str | None = Field(default=None, pattern=r"^sd-[0-9a-f]{12}$")
delivery_configuration_id: str | None = Field(default=None, pattern=r"^cfg-[0-9a-f]{24}$")
@model_validator(mode="after")
def secure_origins(self):
from ipaddress import ip_address
from urllib.parse import urlsplit
if (self.coordinator_url is None) != (self.coordinator_token is None):
raise ValueError("both coordinator URL and token required")
if self.coordinator_url is not None:
parts = urlsplit(self.coordinator_url)
try:
valid = parts.hostname is not None and ip_address(parts.hostname).is_loopback
port = parts.port
except ValueError:
valid, port = False, None
if (
not valid
or parts.scheme != "http"
or port is None
or parts.path not in {"", "/"}
or parts.username
or parts.password
or parts.query
or parts.fragment
):
raise ValueError("coordinator must be an explicit numeric loopback HTTP origin")
import re
if not re.fullmatch(r"[0-9a-f]{64}", self.coordinator_token.get_secret_value()):
raise ValueError("invalid coordinator service credential")
if not self.browser_origins or len(self.browser_origins) > 8:
raise ValueError("one to eight browser origins required")
for origin in self.browser_origins:
parts = urlsplit(origin)
if (
parts.scheme not in {"http", "https"}
or not parts.hostname
or parts.username
or parts.password
or parts.path
or parts.query
or parts.fragment
or (self.environment == "production" and parts.scheme != "https")
):
raise ValueError(
"browser origins must be explicit origins; production requires HTTPS"
)
return self
backend/src/kf_oj/infra/artifacts/README.md
-20
View File
@@ -1,20 +0,0 @@
# 不可变工件
存储根配置于 `KF_OJ_ARTIFACT_ROOT`，由服务账号持有，目录 0700、文件 0600。
单件至多 64 MiB；默认总字节 512 MiB / 文件 4096，可在有界配置中调整。
发布采用摘要名称、临时文件、文件/目录 fsync 和跨进程锁；已有工件必须通过长度/摘要核验，
拒绝静默覆盖/修复，普通读取不跟随链接。锁等待最多五秒。临时文件也计入容量。
适配器执行已经授权的操作，不自行识别用户、租约或题库。公开样例由 problems 的公开 DTO
提供；个人源码由 submissions 的已验证主体与归属端口读取；隐藏数据只经有效当前租约取得。
没有匿名工件下载或由文件路径选择的读取接口。
发布事务先取得 digest 锁，再持久化字节与元数据，业务引用最后提交；事务回滚最多留下
无元数据的耐久孤立文件，不留下数据库引用指向未发布字节。回收使用同一 digest 锁，
业务引用的数据库生成 ready 状态及复合外键阻止被引用元数据转为 deleting。
`uv run --locked kf-artifacts --limit 100 [--after <digest>]` 执行有限回收批次：恢复 deleting、
标记超过一天的未引用元数据、删除耐久字节/元数据及超过一天的无元数据文件/临时文件。
每次扫描至多 100 行；显示游标，后续调用接续，避免固定前页上的保留工件阻塞扫描。
引用工件不能被回收。文件删除后事务失败仍保持 deleting，下一批可恢复；新引用无法加入。
需要已迁移数据库和有限平台权限；回收不遍历业务域 ORM，也不删除不可变题目版本。
backend/src/kf_oj/infra/artifacts/__init__.py
+1 -1
View File
@@ -1 +1 @@
"""Digest-addressed immutable storage; business ports enforce authorization."""
"""Reserved for authenticated artifact access and digest verification."""
backend/src/kf_oj/infra/artifacts/archive_reader.py
-340
View File
@@ -1,340 +0,0 @@
"""Bounded single-level ZIP reader; no extractall, metadata restoration or content execution."""
import binascii
import hashlib
import os
import re
import stat
import struct
import time
import zlib
from collections.abc import Iterator
from dataclasses import dataclass
from kf_oj.contracts.admin_problems import (
MAX_ARCHIVE_BYTES,
MAX_ARCHIVE_FILES,
MAX_UNPACKED_BYTES,
)
NAME = re.compile(rb"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z")
CHUNK = 65536
LOCAL = struct.Struct("<4s5H3I2H")
CENTRAL = struct.Struct("<4s6H3I5H2I")
END = struct.Struct("<4s4H2IH")
class ArchiveError(ValueError):
"""A fixed sanitized failure; never echo an archive name, data or local path."""
@dataclass(frozen=True)
class ArchiveMember:
name: str
method: int
crc32: int
compressed_bytes: int
bytes: int
offset: int
flags: int
class ArchiveReader:
def __init__(self, descriptor: int, expected_bytes: int, *, deadline: float):
self.descriptor, self.deadline = descriptor, deadline
self.before = os.fstat(descriptor)
if (
not stat.S_ISREG(self.before.st_mode)
or self.before.st_uid != os.getuid()
or self.before.st_nlink != 1
or stat.S_IMODE(self.before.st_mode) != 0o600
or not 22 <= expected_bytes <= MAX_ARCHIVE_BYTES
or self.before.st_size != expected_bytes
):
raise ArchiveError("invalid_archive")
self.size = expected_bytes
self.unpacked_bytes, self.steps = 0, 0
self.read_members: set[str] = set()
self.members = self._members()
def _tick(self) -> None:
self.steps += 1
if self.steps > 8192 or time.monotonic() >= self.deadline:
raise ArchiveError("archive_deadline")
def _read(self, offset: int, size: int) -> bytes:
self._tick()
if not 0 <= size <= 1024**2 or offset < 0 or offset + size > self.size:
raise ArchiveError("invalid_archive")
data = os.pread(self.descriptor, size, offset)
if len(data) != size:
raise ArchiveError("invalid_archive")
return data
@staticmethod
def _extra(data: bytes) -> None:
offset, fields = 0, 0
# These bounded timestamps/UID attributes are ignored, never restored.
# ZIP64, Unicode aliases and all other extension semantics are unsupported.
while offset < len(data):
if offset + 4 > len(data) or fields >= 32:
raise ArchiveError("invalid_archive")
kind, size = struct.unpack_from("<HH", data, offset)
offset += 4
if kind not in (0x5455, 0x7875, 0x5855, 0x7855, 0x000A) or offset + size > len(data):
raise ArchiveError("invalid_archive")
offset += size
fields += 1
def _members(self) -> tuple[ArchiveMember, ...]:
tail_offset = max(0, self.size - 1046)
tail = self._read(tail_offset, self.size - tail_offset)
position = tail.rfind(b"PK\x05\x06")
if position < 0 or position + END.size > len(tail):
raise ArchiveError("invalid_archive")
sig, disk, cd_disk, disk_count, count, cd_bytes, cd_offset, comment = END.unpack_from(
tail, position
)
end_offset = tail_offset + position
if (
sig != b"PK\x05\x06"
or disk != 0
or cd_disk != 0
or disk_count != count
or not 1 <= count <= MAX_ARCHIVE_FILES
or comment > 1024
or end_offset + END.size + comment != self.size
or not 46 <= cd_bytes <= 1024**2
or cd_offset + cd_bytes != end_offset
):
raise ArchiveError("invalid_archive")
directory = self._read(cd_offset, cd_bytes)
offset, records, names = 0, [], set()
for _ in range(count):
if offset + CENTRAL.size > len(directory):
raise ArchiveError("invalid_archive")
values = CENTRAL.unpack_from(directory, offset)
(
sig,
_,
needed,
flags,
method,
_,
_,
crc,
compressed,
size,
name_size,
extra_size,
comment_size,
disk,
_,
attributes,
local_offset,
) = values
allowed_flags = 0x0808 | (0x0006 if method == 8 else 0)
if (
sig != b"PK\x01\x02"
or needed not in (10, 20)
or method not in (0, 8)
or (method == 8 and needed != 20)
or flags & ~allowed_flags
or disk != 0
or not 1 <= name_size <= 128
or extra_size > 1024
or comment_size > 256
or size > MAX_UNPACKED_BYTES
or compressed > MAX_ARCHIVE_BYTES
or local_offset + LOCAL.size > cd_offset
or attributes & 0x10
or stat.S_IFMT(attributes >> 16) not in (0, stat.S_IFREG)
):
raise ArchiveError("invalid_archive")
start = offset + CENTRAL.size
finish = start + name_size + extra_size + comment_size
if finish > len(directory):
raise ArchiveError("invalid_archive")
name_bytes = directory[start : start + name_size]
if not NAME.fullmatch(name_bytes):
raise ArchiveError("invalid_archive")
name = name_bytes.decode("ascii")
if name.lower() in names:
raise ArchiveError("invalid_archive")
names.add(name.lower())
self._extra(directory[start + name_size : start + name_size + extra_size])
records.append(ArchiveMember(name, method, crc, compressed, size, local_offset, flags))
offset = finish
if offset != len(directory) or sum(m.bytes for m in records) > MAX_UNPACKED_BYTES:
raise ArchiveError("invalid_archive")
cursor, members = 0, []
for member in sorted(records, key=lambda m: m.offset):
if member.offset != cursor:
raise ArchiveError("invalid_archive")
sig, needed, flags, method, _, _, crc, compressed, size, name_size, extra_size = (
LOCAL.unpack(self._read(cursor, LOCAL.size))
)
if (
sig != b"PK\x03\x04"
or needed not in (10, 20)
or flags != member.flags
or method != member.method
or (method == 8 and needed != 20)
or name_size != len(member.name)
or extra_size > 1024
):
raise ArchiveError("invalid_archive")
metadata = self._read(cursor + LOCAL.size, name_size + extra_size)
if metadata[:name_size] != member.name.encode("ascii"):
raise ArchiveError("invalid_archive")
self._extra(metadata[name_size:])
data_offset = cursor + LOCAL.size + name_size + extra_size
cursor = data_offset + member.compressed_bytes
if cursor > cd_offset or (method == 0 and member.compressed_bytes != member.bytes):
raise ArchiveError("invalid_archive")
if flags & 8:
if (crc, compressed, size) not in (
(0, 0, 0),
(member.crc32, member.compressed_bytes, member.bytes),
):
raise ArchiveError("invalid_archive")
marker = self._read(cursor, 4)
descriptor_size = 16 if marker == b"PK\x07\x08" else 12
descriptor = self._read(cursor, descriptor_size)
described = struct.unpack("<III", descriptor[-12:])
if described != (member.crc32, member.compressed_bytes, member.bytes):
raise ArchiveError("invalid_archive")
cursor += descriptor_size
elif (crc, compressed, size) != (member.crc32, member.compressed_bytes, member.bytes):
raise ArchiveError("invalid_archive")
members.append(
ArchiveMember(
member.name,
method,
member.crc32,
member.compressed_bytes,
member.bytes,
data_offset,
flags,
)
)
if cursor != cd_offset:
raise ArchiveError("invalid_archive")
return tuple(members)
def verify_digest(self, expected: str) -> None:
digest = hashlib.sha256()
for offset in range(0, self.size, CHUNK):
digest.update(self._read(offset, min(CHUNK, self.size - offset)))
if digest.hexdigest() != expected:
raise ArchiveError("archive_digest_mismatch")
def verify_unchanged(self) -> None:
after = os.fstat(self.descriptor)
fields = (
"st_dev",
"st_ino",
"st_nlink",
"st_mode",
"st_uid",
"st_size",
"st_mtime_ns",
"st_ctime_ns",
)
if any(getattr(self.before, f) != getattr(after, f) for f in fields):
raise ArchiveError("archive_changed")
@staticmethod
def _nested(prefix: bytes) -> bool:
return prefix.startswith(
(
b"PK\x03\x04",
b"PK\x05\x06",
b"PK\x07\x08",
b"\x1f\x8b",
b"BZh",
b"\xfd7zXZ\x00",
b"Rar!\x1a\x07",
b"7z\xbc\xaf\x27\x1c",
)
) or (len(prefix) >= 262 and prefix[257:262] == b"ustar")
def chunks(self, name: str, maximum: int) -> Iterator[bytes]:
member = next((m for m in self.members if m.name == name), None)
if member is None or name in self.read_members or member.bytes > maximum:
raise ArchiveError("invalid_archive")
self.read_members.add(name)
used, crc, prefix = 0, 0, bytearray()
inflater = zlib.decompressobj(-15) if member.method == 8 else None
try:
for offset in range(0, member.compressed_bytes, CHUNK):
pending = self._read(
member.offset + offset, min(CHUNK, member.compressed_bytes - offset)
)
while pending:
self._tick()
if inflater:
data = inflater.decompress(pending, min(CHUNK, member.bytes - used + 1))
pending = inflater.unconsumed_tail
if inflater.unused_data:
raise ArchiveError("invalid_archive")
else:
data, pending = pending, b""
used += len(data)
self.unpacked_bytes += len(data)
if (
used > member.bytes
or used > maximum
or self.unpacked_bytes > MAX_UNPACKED_BYTES
):
raise ArchiveError("invalid_archive")
prefix.extend(data[: max(0, 512 - len(prefix))])
if self._nested(bytes(prefix)):
raise ArchiveError("invalid_archive")
crc = binascii.crc32(data, crc)
if data:
yield data
if (inflater and not inflater.eof) or used != member.bytes or crc != member.crc32:
raise ArchiveError("invalid_archive")
except zlib.error:
raise ArchiveError("invalid_archive") from None
def read(self, name: str, maximum: int) -> bytes:
return b"".join(self.chunks(name, maximum))
def extract_remaining(self, directory: int) -> dict[str, tuple[str, int]]:
info = os.fstat(directory)
if (
not stat.S_ISDIR(info.st_mode)
or info.st_uid != os.getuid()
or stat.S_IMODE(info.st_mode) != 0o700
):
raise ArchiveError("invalid_extraction_directory")
references = {}
for member in self.members:
if member.name in self.read_members:
continue
fd = os.open(
member.name,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
0o600,
dir_fd=directory,
)
try:
digest, size = hashlib.sha256(), 0
for chunk in self.chunks(member.name, member.bytes):
digest.update(chunk)
size += len(chunk)
remaining = memoryview(chunk)
while remaining:
written = os.write(fd, remaining)
if written <= 0:
raise ArchiveError("extraction_unavailable")
remaining = remaining[written:]
os.fsync(fd)
references[member.name] = (digest.hexdigest(), size)
finally:
os.close(fd)
os.fsync(directory)
self.verify_unchanged()
return references
backend/src/kf_oj/infra/artifacts/backup.py
-126
View File
@@ -1,126 +0,0 @@
"""Read-only private backup inventory, separate from garbage collection and grading."""
import argparse
import asyncio
import hashlib
import json
import os
import stat
from pathlib import Path
from sqlalchemy import select, text
from sqlalchemy.exc import SQLAlchemyError
from kf_oj.contracts.judging import ArtifactReference
from kf_oj.core.settings import Settings
from kf_oj.infra.artifacts.models import Artifact
from kf_oj.infra.db.session import build_engine, build_session_factory
MAX_REFERENCES = 65536
def verify_existing(root: Path, reference: ArtifactReference) -> None:
"""No mkdir, lock creation or writable descriptor, even for a missing store."""
root = root.absolute()
if root.resolve(strict=True) != root:
raise ValueError("backup storage path contains links")
directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
try:
info = os.fstat(directory)
if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
raise ValueError("backup storage root identity differs")
descriptor = os.open(
reference.digest,
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
dir_fd=directory,
)
with os.fdopen(descriptor, "rb") as source:
before = os.fstat(source.fileno())
if (
not stat.S_ISREG(before.st_mode)
or before.st_nlink != 1
or before.st_uid != os.getuid()
or stat.S_IMODE(before.st_mode) != 0o600
or before.st_size != reference.bytes
):
raise ValueError("backup artifact identity or length differs")
digest = hashlib.sha256()
remaining = reference.bytes
while remaining:
block = source.read(min(65536, remaining))
if not block:
raise ValueError("backup artifact truncated")
digest.update(block)
remaining -= len(block)
after = os.fstat(source.fileno())
if (
source.read(1)
or digest.hexdigest() != reference.digest
or any(
getattr(before, key) != getattr(after, key)
for key in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns")
)
):
raise ValueError("backup artifact changed or digest differs")
finally:
os.close(directory)
async def inventory() -> dict:
"""Every ready registry object covers all enforced ready-state foreign keys.
The operator separately proves a stopped-writing cut. This CLI neither
claims that cut nor reads domain ORM models or changes a lifecycle state.
"""
settings = Settings()
if settings.artifact_root is None:
raise ValueError("artifact root required")
engine = build_engine(settings)
try:
sessions = build_session_factory(engine)
async with sessions.begin() as session:
await session.execute(text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"))
rows = await session.execute(
select(Artifact.digest, Artifact.bytes)
.where(Artifact.state == "ready")
.order_by(Artifact.digest)
.limit(MAX_REFERENCES + 1)
)
objects = list(rows)
if len(objects) > MAX_REFERENCES:
raise ValueError("backup registry count exceeds bound")
references = []
for digest, size in objects:
reference = ArtifactReference(digest=digest, bytes=size)
# A 64-KiB buffer checks actual bytes without source/case output.
await asyncio.to_thread(verify_existing, settings.artifact_root, reference)
references.append(reference.model_dump(mode="json"))
return {
"schema_version": 1,
"kind": "kf.private.artifact-backup-inventory",
"read_only": True,
"quiescence_verified": False,
"boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(),
"storage_identity": {
"device": settings.artifact_root.stat().st_dev,
"inode": settings.artifact_root.stat().st_ino,
"uid": settings.artifact_root.stat().st_uid,
},
"objects": references,
}
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description="Private read-only artifact backup inventory")
parser.parse_args()
try:
result = asyncio.run(asyncio.wait_for(inventory(), timeout=300))
print(json.dumps(result, separators=(",", ":"), sort_keys=True))
except (ValueError, OSError, SQLAlchemyError, TimeoutError):
parser.exit(1, "Backup artifact inventory refused; no metadata or grade was modified.\n")
if __name__ == "__main__":
main()
backend/src/kf_oj/infra/artifacts/cli.py
-48
View File
@@ -1,48 +0,0 @@
import argparse
import asyncio
from sqlalchemy.exc import SQLAlchemyError
from kf_oj.core.settings import Settings
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.infra.artifacts.store import ArtifactStore
from kf_oj.infra.db.session import build_engine, build_session_factory
async def run(limit: int, after: str | None) -> None:
settings = Settings()
if settings.artifact_root is None:
raise ValueError("artifact root required")
store = ArtifactStore(
settings.artifact_root,
max_bytes=settings.artifact_max_bytes,
max_files=settings.artifact_max_files,
)
engine = build_engine(settings)
try:
sessions = build_session_factory(engine)
repository = ArtifactRepository(store)
recovered = await repository.finish_deleting(sessions, limit=limit)
marked, cursor = await repository.collect_metadata(sessions, limit=limit, after=after)
removed = await repository.finish_deleting(sessions, limit=limit)
orphans = await repository.collect_orphans(sessions, limit=limit)
print(f"Recovered {recovered}; marked {marked}; removed {removed}; orphans {orphans}.")
print(f"Next metadata scan cursor: {cursor or 'complete'}")
finally:
await engine.dispose()
store.close()
def main() -> None:
parser = argparse.ArgumentParser(description="Collect bounded old unreferenced artifacts")
parser.add_argument("--limit", type=int, default=100, choices=range(1, 101))
parser.add_argument("--after")
args = parser.parse_args()
try:
asyncio.run(run(args.limit, args.after))
except (ValueError, OSError, SQLAlchemyError):
parser.exit(1, "Artifact collection rejected; check storage and database configuration.\n")
if __name__ == "__main__":
main()
backend/src/kf_oj/infra/artifacts/models.py
-21
View File
@@ -1,21 +0,0 @@
"""Metadata ownership and lifecycle for immutable, digest-addressed bytes."""
from sqlalchemy import BigInteger, CheckConstraint, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
class Artifact(CreatedAt, Base):
__tablename__ = "artifacts"
__table_args__ = (
CheckConstraint("digest ~ '^[0-9a-f]{64}$'", name="digest"),
CheckConstraint("bytes BETWEEN 0 AND 67108864", name="bytes"),
CheckConstraint("state IN ('ready', 'deleting')", name="state"),
UniqueConstraint("digest", "state"),
)
digest: Mapped[str] = mapped_column(String(64), primary_key=True)
bytes: Mapped[int] = mapped_column(BigInteger)
state: Mapped[str] = mapped_column(String(8), server_default="ready")
backend/src/kf_oj/infra/artifacts/repository.py
-166
View File
@@ -1,166 +0,0 @@
"""Byte durability precedes metadata. Digest locks span caller reference transactions."""
import asyncio
from datetime import timedelta
from sqlalchemy import func, select, text, update
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from kf_oj.contracts.judging import ArtifactReference
from kf_oj.infra.artifacts.models import Artifact
from kf_oj.infra.artifacts.store import ArtifactError, ArtifactStore
async def lock_digest(session: AsyncSession, digest: str) -> None:
await session.execute(
text("SELECT pg_advisory_xact_lock(:key)"),
{"key": int(digest[:16], 16) & ((1 << 63) - 1)},
)
class ArtifactRepository:
def __init__(self, store: ArtifactStore):
self.store = store
async def put(
self, session: AsyncSession, data: bytes, expected: ArtifactReference | None = None
) -> ArtifactReference:
reference = self.store.reference(data)
if expected is not None and reference != expected:
raise ArtifactError("artifact digest or length mismatch")
await lock_digest(session, reference.digest)
existing = await session.get(Artifact, reference.digest)
if existing is not None and (
existing.state != "ready" or existing.bytes != reference.bytes
):
raise ArtifactError("artifact metadata unavailable")
await asyncio.to_thread(self.store.put, data, reference)
await session.execute(
insert(Artifact)
.values(digest=reference.digest, bytes=reference.bytes)
.on_conflict_do_nothing(index_elements=[Artifact.digest])
)
return reference
async def read(self, session: AsyncSession, reference: ArtifactReference) -> bytes:
metadata = await session.get(Artifact, reference.digest)
if metadata is None or metadata.state != "ready" or metadata.bytes != reference.bytes:
raise ArtifactError("artifact metadata unavailable")
return await asyncio.to_thread(self.store.read, reference)
async def put_file(
self, session: AsyncSession, descriptor: int, reference: ArtifactReference
) -> ArtifactReference:
await lock_digest(session, reference.digest)
existing = await session.get(Artifact, reference.digest)
if existing is not None and (
existing.state != "ready" or existing.bytes != reference.bytes
):
raise ArtifactError("artifact metadata unavailable")
task = asyncio.create_task(asyncio.to_thread(self.store.put_file, descriptor, reference))
try:
await asyncio.shield(task)
except BaseException:
await task
raise
await session.execute(
insert(Artifact)
.values(digest=reference.digest, bytes=reference.bytes)
.on_conflict_do_nothing(index_elements=[Artifact.digest])
)
return reference
async def check_metadata(self, session: AsyncSession, reference: ArtifactReference) -> None:
metadata = await session.get(Artifact, reference.digest)
if metadata is None or metadata.state != "ready" or metadata.bytes != reference.bytes:
raise ArtifactError("artifact metadata unavailable")
async def collect_metadata(
self,
sessions: async_sessionmaker,
*,
min_age_seconds: int = 86400,
limit: int = 100,
after: str | None = None,
) -> tuple[int, str | None]:
if min_age_seconds < 3600 or not 1 <= limit <= 100:
raise ArtifactError("invalid collection budget")
if after is not None:
ArtifactReference(digest=after, bytes=0)
async with sessions() as session:
query = select(Artifact.digest).where(
Artifact.state == "ready",
Artifact.created_at < func.now() - timedelta(seconds=min_age_seconds),
)
if after is not None:
query = query.where(Artifact.digest > after)
names = list(await session.scalars(query.order_by(Artifact.digest).limit(limit)))
marked = 0
for digest in names:
async with sessions.begin() as session:
try:
async with session.begin_nested():
changed = await self.mark_deleting(session, digest)
marked += int(changed)
except IntegrityError:
# Referenced rows are protected by their generated ready-state foreign key.
# No domain ORM imports or reverse dependencies are needed for reclamation.
pass
return marked, names[-1] if len(names) == limit else None
async def collect_orphans(
self, sessions: async_sessionmaker, *, min_age_seconds: int = 86400, limit: int = 100
) -> int:
# Only filesystem orphans are collected here. Referenced metadata is never deleted.
# A durable row cannot become an orphan while another transaction publishes it:
# every writer and collector holds the same transaction-scoped digest lock.
names = await asyncio.to_thread(
self.store.old_digests, min_age_seconds=min_age_seconds, limit=limit
)
collected = 0
for digest in names:
async with sessions.begin() as session:
await lock_digest(session, digest)
if await session.get(Artifact, digest) is None:
await asyncio.to_thread(self.store.remove, digest)
collected += 1
await asyncio.to_thread(
self.store.clean_temporary, min_age_seconds=min_age_seconds, limit=limit
)
return collected
async def mark_deleting(self, session: AsyncSession, digest: str) -> bool:
# FK(digest, 'ready') on every business reference rejects this transition if referenced.
await lock_digest(session, digest)
result = await session.scalar(
update(Artifact)
.where(Artifact.digest == digest, Artifact.state == "ready")
.values(state="deleting")
.returning(Artifact.digest)
)
return result is not None
async def finish_deleting(self, sessions: async_sessionmaker, *, limit: int = 100) -> int:
if not 1 <= limit <= 100:
raise ArtifactError("invalid collection budget")
async with sessions() as session:
names = list(
await session.scalars(
select(Artifact.digest)
.where(Artifact.state == "deleting")
.order_by(Artifact.digest)
.limit(limit)
)
)
count = 0
for digest in names:
async with sessions.begin() as session:
await lock_digest(session, digest)
metadata = await session.get(Artifact, digest)
if metadata is not None and metadata.state == "deleting":
await asyncio.to_thread(self.store.remove, digest)
await session.delete(metadata)
count += 1
return count
backend/src/kf_oj/infra/artifacts/staging.py
-208
View File
@@ -1,208 +0,0 @@
"""Owned, private, bounded import staging. A kernel lock survives parent/child handoff."""
import fcntl
import os
import stat
from contextlib import contextmanager
from pathlib import Path
from uuid import UUID
from kf_oj.contracts.admin_problems import MAX_ARCHIVE_FILES, STAGING_MAX_BYTES
from kf_oj.infra.artifacts.store import ArtifactError
class ImportStaging:
def __init__(self, root: Path):
root = root.absolute()
root.mkdir(parents=True, mode=0o700, exist_ok=True)
if root.resolve() != root:
raise ArtifactError("invalid staging root")
self.root = root
self.fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
self.directory(self.fd)
@staticmethod
def directory(fd):
s = os.fstat(fd)
if (
not stat.S_ISDIR(s.st_mode)
or s.st_uid != os.getuid()
or stat.S_IMODE(s.st_mode) != 0o700
):
raise ArtifactError("invalid staging directory")
def close(self):
os.close(self.fd)
@staticmethod
def names(descriptor, maximum):
result = []
with os.scandir(descriptor) as entries:
for entry in entries:
if len(result) >= maximum:
raise ArtifactError("staging file capacity exhausted")
result.append(entry.name)
return result
def operation_ids(self):
identifiers = []
for name in self.names(self.fd, 33):
if name == ".lock":
continue
try:
identifier = UUID(name)
if str(identifier) != name:
raise ValueError
except ValueError:
raise ArtifactError("unexpected staging entry") from None
identifiers.append(identifier)
return identifiers
@contextmanager
def exclusive(self):
lock = os.open(
".lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=self.fd
)
try:
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(os.fstat(lock))
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise ArtifactError("import capacity busy") from None
yield lock
finally:
os.close(lock)
def usage(self):
total, count = 0, 0
for name in self.names(self.fd, 33):
if name == ".lock":
continue
try:
if str(UUID(name)) != name:
raise ValueError
except ValueError:
raise ArtifactError("unexpected staging entry") from None
descriptor = self.operation(UUID(name))
try:
for entry in self.names(descriptor, 3):
count += 1
if count > 2 * (MAX_ARCHIVE_FILES + 2):
raise ArtifactError("staging file capacity exhausted")
if entry == "members":
child = self.members(descriptor)
try:
for member in self.names(child, MAX_ARCHIVE_FILES):
count += 1
info = os.stat(member, dir_fd=child, follow_symlinks=False)
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(info)
total += info.st_size
finally:
os.close(child)
elif entry in ("archive", "validated.json"):
info = os.stat(entry, dir_fd=descriptor, follow_symlinks=False)
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(info)
total += info.st_size
else:
raise ArtifactError("unexpected staging entry")
if count > 2 * (MAX_ARCHIVE_FILES + 2) or total > STAGING_MAX_BYTES:
raise ArtifactError("staging capacity exhausted")
finally:
os.close(descriptor)
return total
def operation(self, operation_id: UUID, *, create=False):
name = str(operation_id)
if create:
try:
os.mkdir(name, 0o700, dir_fd=self.fd)
except FileExistsError:
pass
descriptor = os.open(
name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=self.fd
)
try:
self.directory(descriptor)
except Exception:
os.close(descriptor)
raise
return descriptor
@staticmethod
def members(operation, *, create=False):
if create:
os.mkdir("members", 0o700, dir_fd=operation)
descriptor = os.open(
"members", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=operation
)
try:
ImportStaging.directory(descriptor)
except Exception:
os.close(descriptor)
raise
return descriptor
def remove(self, operation_id):
try:
descriptor = self.operation(operation_id)
except FileNotFoundError:
return
try:
names = self.names(descriptor, 3)
if len(names) > 3:
raise ArtifactError("unexpected staging entry")
for name in names:
if name == "members":
child = self.members(descriptor)
try:
entries = self.names(child, MAX_ARCHIVE_FILES)
if len(entries) > MAX_ARCHIVE_FILES:
raise ArtifactError("unexpected staging entry")
for member in entries:
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(
os.stat(member, dir_fd=child, follow_symlinks=False)
)
os.unlink(member, dir_fd=child)
finally:
os.close(child)
os.rmdir("members", dir_fd=descriptor)
elif name in ("archive", "validated.json"):
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(os.stat(name, dir_fd=descriptor, follow_symlinks=False))
os.unlink(name, dir_fd=descriptor)
else:
raise ArtifactError("unexpected staging entry")
finally:
os.close(descriptor)
os.rmdir(str(operation_id), dir_fd=self.fd)
os.fsync(self.fd)
def reset_validation(self, operation_id):
directory = self.operation(operation_id)
try:
names = self.names(directory, 3)
if "members" in names:
members = self.members(directory)
try:
for name in self.names(members, MAX_ARCHIVE_FILES):
from kf_oj.infra.artifacts.store import ArtifactStore
ArtifactStore._regular(os.stat(name, dir_fd=members, follow_symlinks=False))
os.unlink(name, dir_fd=members)
finally:
os.close(members)
os.rmdir("members", dir_fd=directory)
if "validated.json" in names:
os.unlink("validated.json", dir_fd=directory)
os.fsync(directory)
finally:
os.close(directory)
backend/src/kf_oj/infra/artifacts/store.py
-337
View File
@@ -1,337 +0,0 @@
"""Private immutable byte store. Authorization is applied before calling this adapter."""
import fcntl
import hashlib
import os
import re
import stat
import time
from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path
from uuid import uuid4
from kf_oj.contracts.judging import MAX_PACKAGE_BYTES, ArtifactReference
DIGEST = re.compile(r"[0-9a-f]{64}\Z")
TEMP = re.compile(r"\.tmp-[0-9a-f]{32}\Z")
class ArtifactError(ValueError):
"""Sanitized artifact failure; never includes paths or private byte content."""
class ArtifactStore:
def __init__(self, root: Path, *, max_bytes: int = 512 * 1024 * 1024, max_files: int = 4096):
if not 1 <= max_bytes <= 16 * 1024**3 or not 1 <= max_files <= 65536:
raise ArtifactError("invalid storage capacity")
root = root.absolute()
root.mkdir(mode=0o700, parents=True, exist_ok=True)
if root.resolve() != root:
raise ArtifactError("storage root must not contain links")
self._directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
self.max_bytes = max_bytes
self.max_files = max_files
info = os.fstat(self._directory)
if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
os.close(self._directory)
raise ArtifactError("storage root must be private and owned")
self._closed = False
def close(self) -> None:
if not self._closed:
os.close(self._directory)
self._closed = True
@contextmanager
def exclusive(self) -> Iterator[None]:
# Open a different description for every operation: flock also serializes threads.
descriptor = os.open(
".lock",
os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC,
0o600,
dir_fd=self._directory,
)
try:
self._regular(os.fstat(descriptor))
deadline = time.monotonic() + 5
while True:
try:
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
break
except BlockingIOError:
if time.monotonic() >= deadline:
raise ArtifactError("storage lock deadline exceeded") from None
time.sleep(0.01)
yield
finally:
os.close(descriptor)
@staticmethod
def _regular(info: os.stat_result) -> None:
if (
not stat.S_ISREG(info.st_mode)
or info.st_nlink != 1
or info.st_uid != os.getuid()
or stat.S_IMODE(info.st_mode) != 0o600
):
raise ArtifactError("invalid storage entry")
def _entries(self) -> list[tuple[str, os.stat_result]]:
entries = []
with os.scandir(self._directory) as directory:
for entry in directory:
if entry.name == ".lock":
continue
if not DIGEST.fullmatch(entry.name) and not TEMP.fullmatch(entry.name):
raise ArtifactError("unexpected storage entry")
info = entry.stat(follow_symlinks=False)
self._regular(info)
entries.append((entry.name, info))
if len(entries) > self.max_files:
raise ArtifactError("storage file capacity exceeded")
return entries
@staticmethod
def reference(data: bytes) -> ArtifactReference:
if type(data) is not bytes or len(data) > MAX_PACKAGE_BYTES:
raise ArtifactError("artifact size outside policy")
return ArtifactReference(digest=hashlib.sha256(data).hexdigest(), bytes=len(data))
def put(self, data: bytes, expected: ArtifactReference | None = None) -> ArtifactReference:
reference = self.reference(data)
if expected is not None and reference != expected:
raise ArtifactError("artifact digest or length mismatch")
with self.exclusive():
entries = self._entries()
if any(name == reference.digest for name, _ in entries):
self._read(reference)
return reference
if len(entries) + 1 > self.max_files or (
sum(info.st_size for _, info in entries) + len(data) > self.max_bytes
):
raise ArtifactError("storage capacity exhausted")
temporary = ".tmp-" + uuid4().hex
descriptor = os.open(
temporary,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
0o600,
dir_fd=self._directory,
)
try:
with os.fdopen(descriptor, "wb") as output:
output.write(data)
output.flush()
os.fsync(output.fileno())
# Locking covers every mutation. Never replace or repair an existing digest.
os.rename(
temporary,
reference.digest,
src_dir_fd=self._directory,
dst_dir_fd=self._directory,
)
os.fsync(self._directory)
finally:
try:
os.unlink(temporary, dir_fd=self._directory)
except FileNotFoundError:
pass
return reference
def _read(self, reference: ArtifactReference) -> bytes:
if (
type(reference.digest) is not str
or not DIGEST.fullmatch(reference.digest)
or type(reference.bytes) is not int
or not 0 <= reference.bytes <= MAX_PACKAGE_BYTES
):
raise ArtifactError("invalid artifact reference")
try:
descriptor = os.open(
reference.digest,
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
dir_fd=self._directory,
)
with os.fdopen(descriptor, "rb") as source:
info = os.fstat(source.fileno())
self._regular(info)
if info.st_size != reference.bytes:
raise ArtifactError("artifact digest or length mismatch")
data = source.read(reference.bytes + 1)
if self.reference(data) != reference:
raise ArtifactError("artifact digest or length mismatch")
return data
except OSError:
raise ArtifactError("artifact unavailable") from None
def read(self, reference: ArtifactReference) -> bytes:
with self.exclusive():
return self._read(reference)
def verify(self, reference: ArtifactReference) -> None:
"""Verify publication inputs with a 64 KiB buffer, without exposing hidden bytes."""
ArtifactReference.model_validate(reference)
with self.exclusive():
try:
descriptor = os.open(
reference.digest,
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
dir_fd=self._directory,
)
with os.fdopen(descriptor, "rb") as source:
before = os.fstat(source.fileno())
self._regular(before)
if before.st_size != reference.bytes:
raise ArtifactError("artifact digest or length mismatch")
digest = hashlib.sha256()
remaining = reference.bytes
while remaining:
data = source.read(min(65536, remaining))
if not data:
raise ArtifactError("artifact digest or length mismatch")
digest.update(data)
remaining -= len(data)
after = os.fstat(source.fileno())
if (
source.read(1)
or digest.hexdigest() != reference.digest
or any(
getattr(before, k) != getattr(after, k)
for k in (
"st_dev",
"st_ino",
"st_nlink",
"st_mode",
"st_uid",
"st_size",
"st_mtime_ns",
"st_ctime_ns",
)
)
):
raise ArtifactError("artifact input changed")
except OSError:
raise ArtifactError("artifact unavailable") from None
def put_file(self, source: int, expected: ArtifactReference) -> ArtifactReference:
"""Adopt a validated private FD without buffering a hidden testcase in memory."""
before = os.fstat(source)
self._regular(before)
if before.st_size != expected.bytes:
raise ArtifactError("artifact digest or length mismatch")
def unchanged(descriptor, original):
current = os.fstat(descriptor)
fields = (
"st_dev",
"st_ino",
"st_nlink",
"st_mode",
"st_uid",
"st_size",
"st_mtime_ns",
"st_ctime_ns",
)
if any(getattr(current, k) != getattr(original, k) for k in fields):
raise ArtifactError("artifact input changed")
def stream(descriptor, *, output=None):
digest = hashlib.sha256()
for offset in range(0, expected.bytes, 65536):
data = os.pread(descriptor, min(65536, expected.bytes - offset), offset)
if len(data) != min(65536, expected.bytes - offset):
raise ArtifactError("artifact digest or length mismatch")
digest.update(data)
if output is not None:
output.write(data)
if digest.hexdigest() != expected.digest:
raise ArtifactError("artifact digest or length mismatch")
with self.exclusive():
entries = self._entries()
if any(name == expected.digest for name, _ in entries):
existing = os.open(
expected.digest,
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
dir_fd=self._directory,
)
try:
self._regular(os.fstat(existing))
if os.fstat(existing).st_size != expected.bytes:
raise ArtifactError("artifact digest or length mismatch")
stream(source)
stream(existing)
finally:
os.close(existing)
else:
if (
len(entries) + 1 > self.max_files
or sum(s.st_size for _, s in entries) + expected.bytes > self.max_bytes
):
raise ArtifactError("storage capacity exhausted")
temporary = ".tmp-" + uuid4().hex
descriptor = os.open(
temporary,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
0o600,
dir_fd=self._directory,
)
try:
with os.fdopen(descriptor, "wb") as out:
stream(source, output=out)
out.flush()
os.fsync(out.fileno())
unchanged(source, before)
os.rename(
temporary,
expected.digest,
src_dir_fd=self._directory,
dst_dir_fd=self._directory,
)
os.fsync(self._directory)
finally:
try:
os.unlink(temporary, dir_fd=self._directory)
except FileNotFoundError:
pass
unchanged(source, before)
return expected
def remove(self, digest: str) -> None:
if not DIGEST.fullmatch(digest):
raise ArtifactError("invalid artifact identifier")
with self.exclusive():
try:
info = os.stat(digest, dir_fd=self._directory, follow_symlinks=False)
self._regular(info)
os.unlink(digest, dir_fd=self._directory)
os.fsync(self._directory)
except FileNotFoundError:
pass
def old_digests(self, *, min_age_seconds: int, limit: int) -> tuple[str, ...]:
if not 3600 <= min_age_seconds or not 1 <= limit <= 100:
raise ArtifactError("invalid collection budget")
with self.exclusive():
cutoff = time.time() - min_age_seconds
return tuple(
name
for name, info in self._entries()
if DIGEST.fullmatch(name) and info.st_mtime < cutoff
)[:limit]
def clean_temporary(self, *, min_age_seconds: int = 86400, limit: int = 100) -> int:
if min_age_seconds < 3600 or not 1 <= limit <= 100:
raise ArtifactError("invalid collection budget")
with self.exclusive():
cutoff = time.time() - min_age_seconds
names = [
name
for name, info in self._entries()
if TEMP.fullmatch(name) and info.st_mtime < cutoff
][:limit]
for name in names:
os.unlink(name, dir_fd=self._directory)
if names:
os.fsync(self._directory)
return len(names)
backend/src/kf_oj/infra/availability.py
-52
View File
@@ -1,52 +0,0 @@
"""Ordinary read-only view of the fixed protected Root admission record."""
import json
import os
import stat
from pathlib import Path
from kf_oj.contracts.availability import AVAILABILITY_PATH, AvailabilityRecord
GATE = Path(AVAILABILITY_PATH)
def protected_record() -> AvailabilityRecord:
for path in GATE.parents:
info = path.lstat()
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
raise ValueError("protected availability directory required")
fd = os.open(GATE, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
try:
info = os.fstat(fd)
if (
not stat.S_ISREG(info.st_mode)
or info.st_nlink != 1
or info.st_uid != 0
or info.st_gid != 0
or stat.S_IMODE(info.st_mode) != 0o644
or info.st_size > 4096
):
raise ValueError("protected availability file required")
with os.fdopen(fd, "rb", closefd=False) as source:
contents = source.read(4097)
if len(contents) > 4096:
raise ValueError("availability size bound")
finally:
os.close(fd)
def unique(pairs):
value = {}
for key, item in pairs:
if key in value:
raise ValueError("duplicate availability field")
value[key] = item
return value
return AvailabilityRecord.model_validate(json.loads(contents, object_pairs_hook=unique))
def current(release: str | None, batch: str | None) -> AvailabilityRecord:
value = protected_record()
if value.release_id != release or value.configuration_id != batch:
raise ValueError("availability is not this running API binding")
return value
backend/src/kf_oj/infra/coordinator.py
-30
View File
@@ -1,30 +0,0 @@
"""Bounded loopback control adapter; never owns evaluation database writes."""
from uuid import UUID
import httpx
class CoordinatorControl:
def __init__(self, url: str, token: str):
self.url, self.token = url.rstrip("/"), token
async def cancel(self, evaluation_id: UUID) -> bool:
try:
async with httpx.AsyncClient(
timeout=3, trust_env=False, follow_redirects=False
) as client:
# Response has no business payload. Do not buffer or log arbitrary remote bodies.
async with client.stream(
"POST",
self.url + "/internal/cancel",
headers={"Authorization": "Bearer " + self.token},
json={
"schema_version": 1,
"kind": "judge.evaluation",
"evaluation_id": str(evaluation_id),
},
) as response:
return response.status_code == 204
except httpx.HTTPError:
return False
backend/src/kf_oj/infra/db/columns.py
-12
View File
@@ -1,12 +0,0 @@
"""Shared column types; business schemas are registered by composition roots."""
from datetime import datetime
from sqlalchemy import DateTime, func
from sqlalchemy.orm import Mapped, mapped_column
class CreatedAt:
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), nullable=False
)
backend/src/kf_oj/infra/db/session.py
+1 -16
View File
@@ -6,22 +6,7 @@ from kf_oj.core.settings import Settings
def build_engine(settings: Settings) -> AsyncEngine:
if settings.database_url is None:
raise ValueError("KF_OJ_DATABASE_URL is required for database-backed operations")
return create_async_engine(
settings.database_url.get_secret_value(),
pool_pre_ping=True,
pool_size=5,
max_overflow=0,
pool_timeout=5,
connect_args={
"timeout": 5,
"command_timeout": 10,
"server_settings": {
"statement_timeout": "10000",
"lock_timeout": "5000",
"idle_in_transaction_session_timeout": "30000",
},
},
)
return create_async_engine(settings.database_url.get_secret_value(), pool_pre_ping=True)
def build_session_factory(engine: AsyncEngine) -> async_sessionmaker:
backend/src/kf_oj/infra/messaging/models.py
-31
View File
@@ -1,31 +0,0 @@
"""Transactional notifications; contains IDs, never source or hidden artifacts."""
from datetime import datetime
from uuid import UUID
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, String, func
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
class Outbox(CreatedAt, Base):
__tablename__ = "outbox"
__table_args__ = (
CheckConstraint("failures BETWEEN 0 AND 100", name="failure_budget"),
CheckConstraint("kind = 'judge.evaluation'", name="kind"),
Index("ix_outbox_delivery", "delivered_at", "next_attempt_at", "id"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
evaluation_id: Mapped[UUID] = mapped_column(ForeignKey("evaluations.id"), unique=True)
kind: Mapped[str] = mapped_column(String(32), server_default="judge.evaluation")
failures: Mapped[int] = mapped_column(server_default="0")
next_attempt_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
locked_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
lock_owner: Mapped[UUID | None]
delivered_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_error_code: Mapped[str | None] = mapped_column(String(32))
backend/src/kf_oj/infra/messaging/outbox_queries.py
-48
View File
@@ -1,48 +0,0 @@
"""Read-only bounded outbox counts, without broker or delivery dependencies."""
from sqlalchemy import text
class OutboxQueries:
def __init__(self, sessions):
self.sessions = sessions
async def status(self):
async with self.sessions() as session, session.begin():
await session.execute(text("SET TRANSACTION READ ONLY"))
await session.execute(text("SET LOCAL statement_timeout='1500'"))
await session.execute(text("SET LOCAL lock_timeout='500'"))
safe = await session.scalar(
text("""
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
OR rolreplication OR rolbypassrls)
AND pg_has_role(current_user,'kf_oj_platform','USAGE')
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
AND NOT has_column_privilege(current_user,'evaluations','state','UPDATE')
FROM pg_roles WHERE rolname=session_user
""")
)
if safe is not True:
raise ValueError("dedicated outbox projection role required")
row = (
(
await session.execute(
text("""
SELECT
(SELECT count(*) FROM (SELECT 1 FROM outbox
WHERE delivered_at IS NULL LIMIT 129) p) pending,
(SELECT count(*) FROM (SELECT 1 FROM outbox WHERE delivered_at IS NULL
AND locked_until>clock_timestamp() LIMIT 129) l) locked
""")
)
)
.mappings()
.one()
)
return {
"schema_version": 1,
"pending_capped": min(row["pending"], 128),
"locked_capped": min(row["locked"], 128),
"truncated": row["pending"] > 128 or row["locked"] > 128,
}
backend/src/kf_oj/infra/messaging/publisher.py
-187
View File
@@ -1,187 +0,0 @@
"""Finite transactional outbox delivery; confirmations are notifications, not grades."""
import asyncio
import logging
import signal
from uuid import uuid4
import aio_pika
from sqlalchemy import text
from kf_oj.contracts.judge import JudgeTaskNotification
from kf_oj.core.settings import Settings
from kf_oj.infra.db.session import build_engine, build_session_factory
from kf_oj.infra.messaging.outbox_queries import OutboxQueries
QUEUE = "kf.judge.evaluations.v1"
QUEUE_ARGUMENTS = {
"x-max-length": 64,
"x-max-length-bytes": 32768,
"x-overflow": "reject-publish",
"x-message-ttl": 600000,
}
class OutboxRepository:
def __init__(self, sessions):
self.sessions = sessions
async def status(self):
return await OutboxQueries(self.sessions).status()
async def claim(self):
owner = uuid4()
async with self.sessions.begin() as s:
row = (
await s.execute(
text("""WITH candidate AS (
SELECT id FROM outbox
WHERE delivered_at IS NULL AND next_attempt_at<=clock_timestamp()
AND (locked_until IS NULL OR locked_until<=clock_timestamp())
ORDER BY next_attempt_at,id FOR UPDATE SKIP LOCKED LIMIT 1)
UPDATE outbox o SET lock_owner=:owner,
locked_until=clock_timestamp()+interval '15 seconds'
FROM candidate c WHERE o.id=c.id RETURNING o.id,o.evaluation_id"""),
{"owner": owner},
)
).first()
return None if row is None else (row.id, row.evaluation_id, owner)
async def confirmed(self, item):
async with self.sessions.begin() as s:
result = await s.execute(
text("""UPDATE outbox SET delivered_at=clock_timestamp(),
locked_until=NULL,lock_owner=NULL,last_error_code=NULL
WHERE id=:id AND lock_owner=:owner AND locked_until>clock_timestamp()"""),
{"id": item[0], "owner": item[2]},
)
return result.rowcount == 1
async def failed(self, item):
async with self.sessions.begin() as s:
await s.execute(
text("""UPDATE outbox SET failures=least(failures+1,100),
next_attempt_at=clock_timestamp()+
least(power(2,least(failures+1,6)),60)*interval '1 second',
locked_until=NULL,lock_owner=NULL,last_error_code='broker_unavailable'
WHERE id=:id AND lock_owner=:owner AND locked_until>clock_timestamp()"""),
{"id": item[0], "owner": item[2]},
)
class ConfirmedBroker:
def __init__(self, url):
# Parent disabled does not suppress descendant loggers; set their inherited level.
for name in ("aio_pika", "aiormq", "pamqp"):
logging.getLogger(name).setLevel(logging.CRITICAL)
self.url = url
self.connection = None
self.channel = None
async def open(self):
async with asyncio.timeout(10):
await self._open()
async def _open(self):
# Use a finite connect, not a reconnect loop holding an outbox lock.
self.connection = await aio_pika.connect(self.url, timeout=5, heartbeat=30)
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
await self.channel.declare_queue(QUEUE, durable=True, arguments=QUEUE_ARGUMENTS, timeout=5)
async def publish(self, item):
# The complete operation ends before the database claim expires.
async with asyncio.timeout(12):
await self._publish(item)
async def _publish(self, item):
if self.channel is None or self.channel.is_closed:
await self.open()
body = (
JudgeTaskNotification(schema_version=1, kind="judge.evaluation", evaluation_id=item[1])
.model_dump_json()
.encode()
)
assert len(body) <= 512
confirmation = await self.channel.default_exchange.publish(
aio_pika.Message(
body,
content_type="application/json",
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=str(item[0]),
),
routing_key=QUEUE,
mandatory=True,
timeout=5,
)
from pamqp.commands import Basic
if not isinstance(confirmation, Basic.Ack):
raise RuntimeError("notification not confirmed")
async def close(self):
if self.connection is not None and not self.connection.is_closed:
try:
await asyncio.wait_for(self.connection.close(), timeout=2)
except Exception:
pass
self.connection = self.channel = None
async def run(settings: Settings, stop: asyncio.Event):
if settings.amqp_url is None:
raise ValueError("AMQP configuration required")
engine = build_engine(settings)
repository = OutboxRepository(build_session_factory(engine))
broker = ConfirmedBroker(settings.amqp_url.get_secret_value())
logger = logging.getLogger("kf_oj.outbox")
# Client exceptions may include connection URIs; suppress transport-library logging.
for name in ("aio_pika", "aiormq", "pamqp"):
logging.getLogger(name).disabled = True
try:
async with engine.connect() as c:
allowed = await c.scalar(
text("""SELECT NOT r.rolsuper AND NOT r.rolbypassrls
AND pg_has_role(current_user,'kf_oj_platform','MEMBER')
AND NOT has_column_privilege(current_user,'evaluations','state','UPDATE')
FROM pg_roles r WHERE r.rolname=current_user""")
)
if not allowed:
raise ValueError("dedicated platform database role required")
while not stop.is_set():
try:
item = await repository.claim()
if item is not None:
try:
await broker.publish(item)
await repository.confirmed(item)
except Exception:
await repository.failed(item)
await broker.close()
logger.warning("outbox broker unavailable")
except Exception:
logger.warning("outbox database unavailable")
try:
await asyncio.wait_for(stop.wait(), timeout=1)
except TimeoutError:
pass
finally:
await broker.close()
await engine.dispose()
def main():
async def entry():
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(sig, stop.set)
await run(Settings(), stop)
try:
asyncio.run(entry())
except Exception:
raise SystemExit("outbox unavailable") from None
if __name__ == "__main__":
main()
backend/src/kf_oj/modules/identity/README.md
+2 -56
View File
@@ -1,59 +1,5 @@
# identity
负责账号、登录凭证、会话与角色；向其他模块提供经过验证的用户身份和授权依据。
`public.py` 提供已验证主体，`service.py` 管理账号和可撤销会话，`repository.py` 封装持久化。
用户身份不能来自前端自报字段或模型参数。
本地管理命令为 `uv run --locked kf-accounts create <username>`。
创建口令由隐藏终端提示读取和确认，不接受命令行参数、环境中的默认口令或自助注册。
用户名为 3–24 个小写 ASCII 字母、数字或下划线；口令 12–128 字符，最多 512 UTF-8 字节。
密码摘要由新增的 `argon2-cffi` 依赖提供 Argon2id，固定 19 MiB/2 次/1 并行度。
参数依据 [OWASP](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html)
和 [argon2-cffi 文档](https://argon2-cffi.readthedocs.io/en/stable/parameters.html)。
登录接口为 `/api/v1/auth/login`，退出 `/logout`，当前主体 `/me`。会话使用随机 256-bit token，
数据库只保存 token / CSRF 摘要；默认 12 小时有效，单账号保留至多 10 个有效会话。
账号禁用立即拒绝登录和旧会话。请求只采用 ASGI 连接 peer，不读取客户端转发地址。
`0005_identity_authority` 为旧账号保留普通用户默认值，并加入管理员标记与服务端派生的授权版本。
启用状态或管理员标记实际变化时，同一事务递增授权版本并撤销旧会话；事务回滚也回滚撤销。
平台账号仅可插入普通账号列、更新启用状态，不能写管理员标记或授权版本；管理员的启用状态
需要独立本地运维身份。`kf_oj_identity_operator` 与 `kf_oj_problem_import` 为无登录的独立权限组，
口令与登录身份由运维另行配置，平台不继承运维组。导入组目前没有表权限，后续导入迁移只授予
队列与候选所需权限，不授予评测状态写入权。迁移不创建默认管理员。
保留授权或撤销历史后拒绝回退这次迁移。
`0006_authority_operations` 将运维组进一步收敛为固定函数的 EXECUTE 权限；不能直接读取
口令/会话摘要或修改账号/会话。函数由独立 NOLOGIN 角色持有，仅有授权所需列权限，固定
`search_path` 并逐项限定操作。所有账号写操作在行锁前更新同一个事务守卫；撤销/禁用/删除
最后一名启用管理员由数据库拒绝。重复授予/撤销不递增授权版本，实际变更仍原子撤销会话。
旧可重复读快照的并发变更产生序列化失败，调用方须结束整个事务后重新操作；不吞掉失败。
并发语义参见[PostgreSQL事务隔离](https://www.postgresql.org/docs/18/transaction-iso.html)，
固定函数的权限与搜索路径参见[CREATE FUNCTION](https://www.postgresql.org/docs/18/sql-createfunction.html)。
在单独的本地运维环境配置 `KF_OJ_IDENTITY_OPERATOR_DATABASE_URL`，使用具有
`kf_oj_identity_operator` 权限组的普通 LOGIN，禁止超级用户、建库/建角色/复制/绕过 RLS
以及平台/协调器/导入组成员。API、Worker 和常规服务不配置此凭据，CLI 不回退平台身份。
先显式创建普通账号，再运行 `uv run --locked kf-accounts bootstrap-admin <username>`。
没有启用管理员时只有明确 bootstrap 可首次授权；不创建默认账号或口令。已有管理员后使用
`grant-admin` / `revoke-admin` / `disable` / `enable`。恢复已禁用管理员使用 `enable`；恢复失去
全部管理员的受控数据库备份时可再次显式 bootstrap。最后一名启用管理员不能被撤销或禁用，
应先授予另一名账号。操作的锁等待最多 2 秒、SQL 最多 5 秒，CLI 只输出账号/能力/授权版本和
是否变化；错误不输出连接串、SQL 参数或驱动诊断。网页管理 API 与题包导入仍待后续实现。
当前主体与登录响应附带版本为 1 的 `capabilities`，其中维护能力和十进制授权版本来自当前
有效会话关联的账号；浏览器只展示这些信息，不能提交它们来获得授权。模块公开端口
`ProblemMaintenancePort.require_problem_maintenance` 在调用方写事务中重新检查账号、授权版本
与实时会话，并按账号→会话的顺序持有 SHARE 锁直到提交/回滚，防止验证之后撤权的竞态。
普通构造的 `Principal` 默认没有维护能力；会话摘要仅在服务端主体内传递，不进入 DTO 或 repr。
受控启动使用 `--no-proxy-headers`，避免服务器中间件预先替换 peer；反向代理后的用户共享
该连接预算，引入可信代理的独立限流策略之前不能直接打开客户端转发头信任。
每 peer 至多 10 次登录/15 分钟，预算通过真实数据库串行更新，失败尝试也消耗预算。
摘要计算最多同时 2 次，超出直接 429；限流表至多 4096 桶且每次有界回收过期桶。
有副作用的浏览器请求必须匹配 `KF_OJ_BROWSER_ORIGINS` 的显式来源；生产只接受 HTTPS 来源，
必须提供生产来源配置。会话 Cookie 为 HttpOnly / SameSite=Strict，生产 Secure；CSRF Cookie
供前端读出请求 token，在 `X-CSRF-Token` 携带，服务端核对其与当前会话绑定的摘要。
登录、主体和退出响应不缓存；请求校验错误不回显密码或输入值。API 层限制请求字节、块数和
读取期限。跨服务 Worker 凭证不使用用户 Cookie。
后续 `public.py` 暴露身份查询接口，`service.py` 编排用例，`repository.py` 封装持久化。
用户身份不能来自前端自报字段或模型参数。目前仅建立模块边界，未实现登录。
backend/src/kf_oj/modules/identity/authority.py
-83
View File
@@ -1,83 +0,0 @@
"""Local-only composition for fixed database authority operations."""
from dataclasses import dataclass
from typing import Literal
from uuid import UUID
from sqlalchemy import text
from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import async_sessionmaker
from kf_oj.modules.identity.passwords import USERNAME
AuthorityOperation = Literal["bootstrap", "grant", "revoke", "enable", "disable"]
@dataclass(frozen=True)
class AuthorityResult:
account_id: UUID
username: str
enabled: bool
is_administrator: bool
authorization_version: int
changed: bool
async def manage_authority(
sessions: async_sessionmaker, username: str, operation: AuthorityOperation
) -> AuthorityResult:
if not USERNAME.fullmatch(username) or operation not in (
"bootstrap",
"grant",
"revoke",
"enable",
"disable",
):
raise ValueError("invalid authority operation")
try:
async with sessions.begin() as session:
await session.execute(text("SET LOCAL statement_timeout='5s'"))
await session.execute(text("SET LOCAL lock_timeout='2s'"))
# The CLI must use a dedicated, unprivileged local operator login.
# Web/platform, import and coordinator credentials are never reused.
safe = await session.scalar(
text("""
SELECT NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication
OR rolbypassrls)
AND rolcanlogin AND pg_has_role(current_user,'kf_oj_identity_operator','USAGE')
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_problem_import','MEMBER')
FROM pg_roles WHERE rolname=current_user
""")
)
if not safe:
raise ValueError("a dedicated unprivileged identity operator login is required")
row = (
(
await session.execute(
text(
"SELECT * FROM public.kf_manage_account_authority(:username,:operation)"
),
{"username": username, "operation": operation},
)
)
.mappings()
.one()
)
return AuthorityResult(**row)
except DBAPIError as error:
code = getattr(error.orig, "sqlstate", None)
messages = {
"P0002": "account not found",
"23514": (
"authority change rejected; preserve an enabled administrator "
"and use explicit bootstrap"
),
"40001": "concurrent authority change rejected; retry the whole operation",
"55P03": "authority operation busy; retry later",
"57014": "authority operation timed out; verify current state before retrying",
"42501": "identity operator permissions unavailable",
}
# Never print a connection URL, SQL bind values or driver diagnostics.
raise ValueError(messages.get(code, "authority database operation unavailable")) from None
backend/src/kf_oj/modules/identity/cli.py
-79
View File
@@ -1,79 +0,0 @@
"""Controlled local account management; passwords are read from the terminal only."""
import argparse
import asyncio
import getpass
from sqlalchemy.exc import IntegrityError
from kf_oj.core.settings import Settings
from kf_oj.infra.db.session import build_engine, build_session_factory
from kf_oj.modules.identity.authority import manage_authority
from kf_oj.modules.identity.service import IdentityService
async def run(command: str, username: str, password: str | None = None) -> None:
settings = Settings()
authority_commands = {
"bootstrap-admin": "bootstrap",
"grant-admin": "grant",
"revoke-admin": "revoke",
"enable": "enable",
"disable": "disable",
}
if command != "create" and command not in authority_commands:
raise ValueError("unknown account command")
# Ordinary account creation uses the platform identity; authority management
# always requires a separately configured login and never falls back to it.
if command in authority_commands:
if settings.identity_operator_database_url is None:
raise ValueError("KF_OJ_IDENTITY_OPERATOR_DATABASE_URL is required")
if settings.identity_operator_database_url == settings.database_url:
raise ValueError("operator and platform database identities must be separate")
settings = settings.model_copy(
update={"database_url": settings.identity_operator_database_url}
)
engine = build_engine(settings)
try:
if command in authority_commands:
result = await manage_authority(
build_session_factory(engine), username, authority_commands[command]
)
print(
f"Account {result.username}: enabled={result.enabled}, "
f"administrator={result.is_administrator}, "
f"authorization_version={result.authorization_version}, changed={result.changed}."
)
return
service = IdentityService(build_session_factory(engine))
if command == "create":
assert password is not None
await service.create_account(username, password)
print("Account created.")
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description="Manage local OJ accounts")
parser.add_argument(
"command",
choices=("create", "disable", "enable", "bootstrap-admin", "grant-admin", "revoke-admin"),
)
parser.add_argument("username")
args = parser.parse_args()
password = None
if args.command == "create":
password = getpass.getpass("Password (12–128 characters): ")
if password != getpass.getpass("Repeat password: "):
parser.error("passwords differ")
try:
asyncio.run(run(args.command, args.username, password))
except IntegrityError:
parser.exit(1, "Account operation rejected by database constraints.\n")
except ValueError as error:
parser.exit(1, f"Account operation rejected: {error}.\n")
if __name__ == "__main__":
main()
backend/src/kf_oj/modules/identity/models.py
-84
View File
@@ -1,84 +0,0 @@
"""Identity-owned persistent account and revocable session records."""
from datetime import datetime
from uuid import UUID
from sqlalchemy import (
BigInteger,
Boolean,
CheckConstraint,
DateTime,
ForeignKey,
Index,
SmallInteger,
String,
Text,
text,
)
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
class AuthorityGuard(Base):
"""Migration-owned singleton; runtime roles have no access."""
__tablename__ = "identity_authority_guard"
__table_args__ = (
CheckConstraint("id = 1", name="singleton"),
CheckConstraint("generation >= 0", name="generation"),
)
id: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
generation: Mapped[int] = mapped_column(BigInteger, server_default="0")
class Account(CreatedAt, Base):
__tablename__ = "accounts"
__table_args__ = (
CheckConstraint("username ~ '^[a-z0-9_]{3,24}$'", name="canonical_username"),
CheckConstraint("char_length(password_hash) BETWEEN 40 AND 256", name="password_hash_size"),
CheckConstraint("authorization_version >= 1", name="authorization_version"),
Index(
"ix_accounts_active_administrators",
"id",
postgresql_where=text("is_administrator AND enabled"),
),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
username: Mapped[str] = mapped_column(String(24), unique=True)
password_hash: Mapped[str] = mapped_column(Text)
enabled: Mapped[bool] = mapped_column(Boolean, server_default="true")
is_administrator: Mapped[bool] = mapped_column(Boolean, server_default="false")
authorization_version: Mapped[int] = mapped_column(BigInteger, server_default="1")
class Session(CreatedAt, Base):
__tablename__ = "sessions"
__table_args__ = (
CheckConstraint("token_hash ~ '^[0-9a-f]{64}$'", name="token_digest"),
CheckConstraint("csrf_hash ~ '^[0-9a-f]{64}$'", name="csrf_digest"),
CheckConstraint("expires_at > created_at", name="expiry"),
Index("ix_sessions_account_expiry", "account_id", "expires_at"),
)
token_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
account_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
csrf_hash: Mapped[str] = mapped_column(String(64))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class LoginThrottle(Base):
__tablename__ = "login_throttles"
__table_args__ = (
CheckConstraint("bucket_hash ~ '^[0-9a-f]{64}$'", name="bucket_digest"),
CheckConstraint("attempts BETWEEN 1 AND 10", name="attempt_budget"),
Index("ix_login_throttles_reset", "reset_at"),
)
bucket_hash: Mapped[str] = mapped_column(String(64), primary_key=True)
attempts: Mapped[int]
reset_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
backend/src/kf_oj/modules/identity/passwords.py
-32
View File
@@ -1,32 +0,0 @@
"""Argon2id password verification with explicit bounded parameters."""
import re
from argon2 import PasswordHasher, Type
from argon2.exceptions import InvalidHashError, VerificationError
USERNAME = re.compile(r"[a-z0-9_]{3,24}\Z")
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1, type=Type.ID)
def validate_account(username: str, password: str) -> None:
if not USERNAME.fullmatch(username) or not 12 <= len(password) <= 128:
raise ValueError("username or password outside account policy")
if len(password.encode("utf-8")) > 512:
raise ValueError("password outside byte policy")
def hash_password(username: str, password: str) -> str:
validate_account(username, password)
return HASHER.hash(password)
def verify_password(stored: str, password: str) -> bool:
try:
# Stored hashes are owner-controlled, but don't allow corrupted parameters to allocate
# arbitrary memory. Account creation always uses these exact parameters.
if not stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$"):
return False
return HASHER.verify(stored, password)
except (VerificationError, InvalidHashError):
return False
backend/src/kf_oj/modules/identity/public.py
-38
View File
@@ -1,38 +0,0 @@
"""Only verified principals cross business module boundaries."""
from dataclasses import dataclass, field
from typing import Protocol
from uuid import UUID
@dataclass(frozen=True)
class Principal:
account_id: UUID
username: str
can_maintain_problems: bool = False
authorization_version: int = 1
session_token_hash: str | None = field(default=None, repr=False, compare=False)
class MaintenanceDenied(ValueError):
pass
class ProblemMaintenancePort(Protocol):
async def require_problem_maintenance(self, session, principal: Principal) -> None:
"""Recheck and lock live authority/session inside the caller's write transaction."""
...
class AuthenticationFailed(ValueError):
pass
class ReservedMaintenancePort(Protocol):
async def require_reserved_maintenance(self, session, actor_id: UUID, epoch: int) -> None:
"""Recheck/lock an immutable accepted receipt, with no login/session secret access."""
...
class LoginThrottled(ValueError):
pass
backend/src/kf_oj/modules/identity/repository.py
-97
View File
@@ -1,97 +0,0 @@
from datetime import datetime
from sqlalchemy import delete, func, select, text, update
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from kf_oj.modules.identity.models import Account, LoginThrottle, Session
from kf_oj.modules.identity.public import LoginThrottled
async def consume_login_budget(
session: AsyncSession, bucket: str, now: datetime, reset_at: datetime
) -> None:
# An upsert establishes the key before FOR UPDATE; requests share one DB-backed budget.
inserted = await session.scalar(
insert(LoginThrottle)
.values(bucket_hash=bucket, attempts=1, reset_at=reset_at)
.on_conflict_do_nothing(index_elements=[LoginThrottle.bucket_hash])
.returning(LoginThrottle.bucket_hash)
)
row = await session.scalar(
select(LoginThrottle).where(LoginThrottle.bucket_hash == bucket).with_for_update()
)
assert row is not None
# The initial upsert consumes this request only if there was no prior row.
# A separate returning flag avoids granting an extra request during concurrency.
if inserted is not None:
return
if row.reset_at <= now:
row.attempts, row.reset_at = 1, reset_at
elif row.attempts >= 10:
raise LoginThrottled("login request budget exhausted")
else:
row.attempts += 1
async def prune_throttles(session: AsyncSession, now: datetime, *, limit: int = 100) -> None:
await session.execute(text("SELECT pg_advisory_xact_lock(-740164777)"))
keys = select(LoginThrottle.bucket_hash).where(LoginThrottle.reset_at <= now)
keys = keys.order_by(LoginThrottle.reset_at).limit(limit).with_for_update(skip_locked=True)
await session.execute(delete(LoginThrottle).where(LoginThrottle.bucket_hash.in_(keys)))
async def check_throttle_capacity(session: AsyncSession, bucket: str) -> None:
if await session.get(LoginThrottle, bucket) is None:
if await session.scalar(select(func.count()).select_from(LoginThrottle)) >= 4096:
raise LoginThrottled("login throttle capacity exhausted")
async def account_by_username(session: AsyncSession, username: str) -> Account | None:
return await session.scalar(select(Account).where(Account.username == username))
async def lock_authority(session: AsyncSession, account_id) -> Account | None:
return await session.scalar(
select(Account)
.where(Account.id == account_id)
.with_for_update(read=True)
.execution_options(populate_existing=True)
)
async def current_session(session: AsyncSession, token_hash: str, *, lock_session: bool = False):
query = (
select(Session, Account)
.join(Account)
.where(
Session.token_hash == token_hash,
Session.revoked_at.is_(None),
Session.expires_at > func.clock_timestamp(),
Account.enabled.is_(True),
)
)
if lock_session:
query = query.with_for_update(read=True, of=Session).execution_options(
populate_existing=True
)
return (await session.execute(query)).one_or_none()
async def prune_sessions(session: AsyncSession, account_id) -> None:
await session.execute(
delete(Session).where(
Session.account_id == account_id,
(Session.expires_at <= func.now()) | Session.revoked_at.is_not(None),
)
)
# Keep at most nine older valid sessions before adding the new one.
old = select(Session.token_hash).where(Session.account_id == account_id)
old = old.order_by(Session.created_at.desc(), Session.token_hash).offset(9)
await session.execute(delete(Session).where(Session.token_hash.in_(old)))
async def revoke(session: AsyncSession, token_hash: str) -> None:
await session.execute(
update(Session).where(Session.token_hash == token_hash).values(revoked_at=func.now())
)
backend/src/kf_oj/modules/identity/service.py
-190
View File
@@ -1,190 +0,0 @@
import asyncio
import hashlib
import hmac
import re
import secrets
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from uuid import uuid4
from sqlalchemy import func, select, text, update
from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import async_sessionmaker
from kf_oj.modules.identity import repository
from kf_oj.modules.identity.models import Account, Session
from kf_oj.modules.identity.passwords import (
HASHER,
hash_password,
validate_account,
verify_password,
)
from kf_oj.modules.identity.public import (
AuthenticationFailed,
LoginThrottled,
MaintenanceDenied,
Principal,
)
TOKEN = re.compile(r"[0-9a-f]{64}\Z")
def token_digest(value: str) -> str:
return hashlib.sha256(value.encode("ascii")).hexdigest()
@dataclass(frozen=True)
class IssuedSession:
principal: Principal
expires_at: datetime
token: str = field(repr=False)
csrf_token: str = field(repr=False)
class IdentityService:
def __init__(self, sessions: async_sessionmaker, *, lifetime_seconds: int = 43200):
if not 300 <= lifetime_seconds <= 86400:
raise ValueError("invalid session lifetime")
self.sessions = sessions
self.lifetime_seconds = lifetime_seconds
self._hash_slots = asyncio.Semaphore(2)
self._dummy_hash = HASHER.hash(secrets.token_hex(32))
async def create_account(self, username: str, password: str) -> Principal:
hashed = await asyncio.to_thread(hash_password, username, password)
async with self.sessions.begin() as session:
account = Account(id=uuid4(), username=username, password_hash=hashed)
session.add(account)
return Principal(account.id, username)
async def disable_account(self, username: str) -> bool:
async with self.sessions.begin() as session:
account = await repository.account_by_username(session, username)
if account is None:
return False
account.enabled = False
await session.execute(
update(Session)
.where(Session.account_id == account.id)
.values(revoked_at=func.now())
)
return True
async def login(self, username: str, password: str, *, peer: str) -> IssuedSession:
validate_account(username, password)
if not peer or len(peer) > 128:
raise AuthenticationFailed("invalid login context")
# Fail fast instead of allowing an unbounded password-hash queue.
if self._hash_slots.locked():
raise LoginThrottled("password verification capacity exhausted")
async with self._hash_slots:
async with self.sessions.begin() as session:
now = await session.scalar(select(func.now()))
assert now is not None
await repository.prune_throttles(session, now)
# Direct peer only; caller must never substitute forwarded client headers.
bucket = hashlib.sha256(peer.encode("utf-8")).hexdigest()
await repository.check_throttle_capacity(session, bucket)
await repository.consume_login_budget(
session, bucket, now, now + timedelta(minutes=15)
)
async with self.sessions() as session:
account = await repository.account_by_username(session, username)
hashed = account.password_hash if account is not None else self._dummy_hash
correct = await asyncio.to_thread(verify_password, hashed, password)
if not correct or account is None or not account.enabled:
raise AuthenticationFailed("invalid credentials")
token, csrf = secrets.token_hex(32), secrets.token_hex(32)
async with self.sessions.begin() as session:
# Serialize with disabling the account and concurrent logins before issuing.
account = await session.scalar(
select(Account).where(Account.id == account.id).with_for_update()
)
if account is None or not account.enabled:
raise AuthenticationFailed("invalid credentials")
now = await session.scalar(select(func.now()))
expires = now + timedelta(seconds=self.lifetime_seconds)
await repository.prune_sessions(session, account.id)
session.add(
Session(
token_hash=token_digest(token),
account_id=account.id,
csrf_hash=token_digest(csrf),
expires_at=expires,
)
)
principal = self._principal(account, token_digest(token))
return IssuedSession(principal, expires, token, csrf)
async def authenticate(self, token: str | None, *, csrf_token: str | None = None) -> Principal:
if token is None or not TOKEN.fullmatch(token):
raise AuthenticationFailed("invalid session")
async with self.sessions() as session:
row = await repository.current_session(session, token_digest(token))
if row is None:
raise AuthenticationFailed("invalid session")
record, account = row
if csrf_token is not None:
if not TOKEN.fullmatch(csrf_token) or not hmac.compare_digest(
record.csrf_hash, token_digest(csrf_token)
):
raise AuthenticationFailed("invalid request token")
return self._principal(account, record.token_hash)
@staticmethod
def _principal(account: Account, session_token_hash: str) -> Principal:
return Principal(
account.id,
account.username,
account.is_administrator,
account.authorization_version,
session_token_hash,
)
async def require_problem_maintenance(self, session, principal: Principal) -> None:
if not session.in_transaction():
raise ValueError("maintenance authorization requires an active write transaction")
if (
not principal.can_maintain_problems
or principal.session_token_hash is None
or not TOKEN.fullmatch(principal.session_token_hash)
):
raise MaintenanceDenied("problem maintenance required")
# Account first, then session: authority changes use that same lock order.
# SHARE locks last until the caller commits/rolls back; a stale principal,
# explicit logout, revocation or expired session cannot authorize a write.
account = await repository.lock_authority(session, principal.account_id)
if (
account is None
or not account.enabled
or not account.is_administrator
or account.authorization_version != principal.authorization_version
):
raise MaintenanceDenied("problem maintenance required")
row = await repository.current_session(
session, principal.session_token_hash, lock_session=True
)
if row is None or row[1].id != account.id:
raise MaintenanceDenied("problem maintenance required")
async def logout(self, token: str, csrf_token: str) -> None:
await self.authenticate(token, csrf_token=csrf_token)
async with self.sessions.begin() as session:
await repository.revoke(session, token_digest(token))
class ReservedMaintenance:
"""Import worker uses only the fixed, narrow SQL authority port."""
async def require_reserved_maintenance(self, session, actor_id, epoch):
if not session.in_transaction() or type(epoch) is not int or epoch < 1:
raise MaintenanceDenied("invalid maintenance receipt")
try:
await session.execute(
text("SELECT public.kf_lock_reserved_maintenance(:actor,:epoch)"),
{"actor": actor_id, "epoch": epoch},
)
except DBAPIError as error:
if getattr(error.orig, "sqlstate", None) == "42501":
raise MaintenanceDenied("problem maintenance revoked") from None
raise
backend/src/kf_oj/modules/operations/README.md
+3 -15
View File
@@ -1,17 +1,5 @@
# operations
维护审计通过 `public.py` 的追加端口写入调用方事务；本模块不成为评测状态的写入入口。
上传、解析完成/失败、丢弃、发布、下架与恢复记录真实账号、授权版本、操作 UUID、对象与
数据库时间。发布回执和审计同时提交；审计失败使整个业务事务回滚，不确认半发布。
本机受控身份 CLI 使用已核验的窄权限数据库函数。授权变化由固定包装函数在同一事务
追加审计，主体类型为 operator，操作者为数据库 `session_user`，对象为实际变更账号；
不把授予/撤销冒充成该账号自己的操作。旧账号事件保留 account 主体类型。
管理员 `/api/v1/admin/maintenance-audit` 按时间和 UUID 的复合游标递减读取，每页至多
100项，可选择有限动作类型；只展示已提交事件。导入失败动作仍是失败，committed 表示
审计事务提交。响应不含口令、会话令牌、工件引用、内部路径或任意诊断正文。
应用数据库角色无审计 UPDATE/DELETE 权限，数据库触发器还阻止历史改写。
导入 Worker 只有追加权限，不能读取整个维护审计；Coordinator 无审计写入权限。
节点管理、部署控制及跨账号业务管理仍未开放。
负责平台侧节点、任务积压、重试原因和审计查询的管理视图。
对判题状态的操作必须调用协调器协议；观测模块不成为第二个状态写入入口。
当前只有目录边界，节点登记、指标聚合和管理员接口待实现。
backend/src/kf_oj/modules/operations/models.py
-39
View File
@@ -1,39 +0,0 @@
"""Append-only bounded maintenance events, independent of evaluation state."""
from uuid import UUID
from sqlalchemy import BigInteger, CheckConstraint, ForeignKey, Index, String, text
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
class MaintenanceEvent(CreatedAt, Base):
__tablename__ = "problem_maintenance_events"
__table_args__ = (
CheckConstraint(
"action IN ('upload_created','upload_received','import_ready',"
"'import_failed','import_discarded','import_expired','published',"
"'disabled','restored','authority_bootstrap','authority_grant',"
"'authority_revoke','authority_enable','authority_disable')",
name="action",
),
CheckConstraint("authorization_version > 0", name="authority"),
CheckConstraint(
"(actor_kind='account' AND actor_id IS NOT NULL AND operator_name IS NULL) "
"OR (actor_kind='operator' AND actor_id IS NULL AND operator_name IS NOT NULL)",
name="actor",
),
Index("ix_problem_maintenance_events_page", "created_at", "id"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
actor_id: Mapped[UUID | None] = mapped_column(ForeignKey("accounts.id"))
actor_kind: Mapped[str] = mapped_column(
String(8), default="account", server_default=text("'account'")
)
operator_name: Mapped[str | None] = mapped_column(String(63))
authorization_version: Mapped[int] = mapped_column(BigInteger)
operation_id: Mapped[UUID] = mapped_column()
action: Mapped[str] = mapped_column(String(24))
target_id: Mapped[UUID | None] = mapped_column()
backend/src/kf_oj/modules/operations/public.py
-28
View File
@@ -1,28 +0,0 @@
"""Accepted actor receipts and finite maintenance events; no private data/log text."""
from typing import Literal, Protocol
from uuid import UUID
Action = Literal[
"upload_created",
"upload_received",
"import_ready",
"import_failed",
"import_discarded",
"import_expired",
"published",
"disabled",
"restored",
]
class MaintenanceAuditPort(Protocol):
async def record(
self,
session,
actor_id: UUID,
authorization_version: int,
operation_id: UUID,
action: Action,
target_id: UUID | None = None,
) -> None: ...
backend/src/kf_oj/modules/operations/queries.py
-62
View File
@@ -1,62 +0,0 @@
"""Only authorized administrators read bounded append-only maintenance projections."""
from datetime import datetime
from uuid import UUID
from sqlalchemy import select, text, tuple_
from kf_oj.contracts.problem_maintenance import MaintenanceAuditEvent, MaintenanceAuditPage
from kf_oj.modules.identity.public import ProblemMaintenancePort
from kf_oj.modules.operations.models import MaintenanceEvent
class MaintenanceAuditQueries:
def __init__(self, sessions, identity: ProblemMaintenancePort):
self.sessions, self.identity = sessions, identity
async def page(self, principal, *, before=None, limit=20, action=None):
if not 1 <= limit <= 100:
raise ValueError("invalid page limit")
cursor = None
if before is not None:
stamp, identifier = before.split("|")
stamp = datetime.fromisoformat(stamp)
if stamp.tzinfo is None:
raise ValueError("invalid audit cursor")
cursor = stamp, UUID(identifier)
async with self.sessions.begin() as session:
await session.execute(text("SET LOCAL statement_timeout='5s'"))
await session.execute(text("SET LOCAL lock_timeout='2s'"))
await self.identity.require_problem_maintenance(session, principal)
q = select(MaintenanceEvent)
if cursor:
q = q.where(tuple_(MaintenanceEvent.created_at, MaintenanceEvent.id) < cursor)
if action:
q = q.where(MaintenanceEvent.action == action)
rows = list(
await session.scalars(
q.order_by(
MaintenanceEvent.created_at.desc(), MaintenanceEvent.id.desc()
).limit(limit + 1)
)
)
visible = rows[:limit]
return MaintenanceAuditPage(
items=tuple(
MaintenanceAuditEvent(
id=r.id,
created_at=r.created_at,
actor_kind=r.actor_kind,
actor_id=r.actor_id,
operator_name=r.operator_name,
authorization_version=str(r.authorization_version),
operation_id=r.operation_id,
action=r.action,
target_id=r.target_id,
)
for r in visible
),
next_before=f"{visible[-1].created_at.isoformat()}|{visible[-1].id}"
if len(rows) > limit
else None,
)
backend/src/kf_oj/modules/operations/service.py
-27
View File
@@ -1,27 +0,0 @@
from uuid import uuid4
from sqlalchemy import func, select
from kf_oj.modules.operations.models import MaintenanceEvent
from kf_oj.modules.operations.public import Action
class MaintenanceAudit:
async def record(
self, session, actor_id, authorization_version, operation_id, action: Action, target_id=None
):
if not session.in_transaction():
raise ValueError("audit requires a write transaction")
# Supplying the trusted DB timestamp avoids ORM INSERT ... RETURNING:
# the ordinary importer has append-only authority and cannot SELECT this table.
session.add(
MaintenanceEvent(
id=uuid4(),
created_at=await session.scalar(select(func.clock_timestamp())),
actor_id=actor_id,
authorization_version=authorization_version,
operation_id=operation_id,
action=action,
target_id=target_id,
)
)
backend/src/kf_oj/modules/problems/README.md
+2 -77
View File
@@ -1,80 +1,5 @@
# problems
负责题目逻辑身份、不可变版本、导入、标签、限制、发布状态与测试工件元数据。
跨模块通过 `public.py` 的不可变判题投影；大文件存取通过 `infra/artifacts` 的适配器。
公开题面/显式样例与隐藏输入/答案分别授权。公开 API 不返回工件摘要或存储路径。
`uv run --locked kf-problems <directory>` 导入并发布一个新版本。目录必须属于当前操作账号，
权限 0700；`problem.json` 遵循 `import_format.py`，只引用该目录下的单层普通文件名。
不解压归档、不跟随链接、不读取任意路径、不执行导入内容。默认清单可由 `--manifest` 指定。
要求配置 `KF_OJ_ARTIFACT_ROOT` 和可信 `KF_OJ_JUDGE_ADMISSION` JSON；同一准入配置应交付给节点，
不能由题目放宽。结构示例来自共享 `judge-admission.valid.json`，不是当前宿主隔离验收证明。
v1 清单固定 C17/C++20、工具链/镜像版本、精确字节比较、编译/运行/整个作业预算，并声明按序
测试点 `{key,input_file,expected_file}` 和最多十对公开样例 `{input_file,output_file}`。
题面以 `statement_file` 读取 UTF-8。每份样例至多 64 KiB，响应采用 base64 保留二进制数据。
同一 slug 受事务锁保护，每次发布递增 revision；发布后题面、限额、样例、测试点不可修改。
旧评测继续绑定旧版本；当前列表指向新版本。工件先持久化，再在同一事务建立引用和发布。
`GET /api/v1/problems` 以 slug 游标分页（`after` / `limit`，最多 100 项），
`GET /api/v1/problems/versions/{uuid}` 读取已发布版本。禁用逻辑题目时其版本对公开读取不可用。
内部判题投影仅供授权应用编排使用；外部 Worker 读取由 submissions 的有效租约端口控制。
v2 使用 `schema_version: 2` 与 `language_configurations`，每个配置包含 `language`、
完整 `toolchain` 身份、`compile_limits`、`run_limits` 和 `job_wall_time_us`。
可分别声明 `c17`、`cpp20`、`python313`、`java21`；不再同时提供 v1 顶层工具链或预算。
所有语言按可信节点配置准入，缺失能力/错误补丁版本/镜像摘要/过宽预算在发布前拒绝。
`judge.admission.example.json` 的镜像摘要为占位值，使用前替换为管理员制作镜像的实际摘要。
公开题面仅显示固定语言、版本、入口与预算，不公布宿主镜像路径和隐藏工件。
导入同一 slug 的 v2 版本保持原 v1 版本与提交不变。已有提交按其保存的协议版本解析，
不能重新序列化旧 canonical 数据来补入新字段。Java 使用 Main.java 与默认包 Main；
Python 使用 main.py。每个新版本的语言集合独立，浏览器不得为旧版本补开新语言。
Web题包导入是独立的待发布流程：`/api/v1/admin/problem-imports` 使用真实维护会话，
写入还要求明确Origin与绑定CSRF。先以稳定操作UUID、ZIP摘要和精确字节数建立容量预约，
再向该操作的 `/archive` 流式PUT；同键重试仍验证实际内容，不会确认异内容。
列表、详情和丢弃只开放给该操作的管理员本人，响应仅含公开题面、样例、语言预算与有限错误。
`ready` 尚未发布，不会加入学生题库，也不改变既有题目版本。
Web格式固定单层ZIP的stored/deflate，清单名为`problem.json`；公开文件与隐藏用例文件
必须显式分别声明且不得混用。文件名只接受有界ASCII单层普通名；未知成员、链接/设备、
加密、嵌套归档、额外前缀/尾随内容、损坏CRC与虚假大小被拒。旧受控目录CLI格式保持原义。
压缩与实际解压各64 MiB、最多534文件，上传最多4096帧/每帧1 MiB/30秒；
解析期限60秒。staging总预约256 MiB，每次128 MiB，单全局上传/解析槽位，
本人最多10个/全局32个活跃候选，操作24小时到期，解析死亡最多领取两次。
真实限额由管理capabilities返回；缺少配置或新鲜Worker资源证明时明确不可用。
`uv run --locked kf-problem-import-worker` 是普通用户、独立数据库登录的组合入口。
配置`KF_OJ_IMPORT_STAGING_ROOT`、`KF_OJ_ARTIFACT_ROOT`、可信准入和
`KF_OJ_IMPORT_WORKER_CGROUP`；staging与工件目录必须归服务用户所有且权限0700。
服务不会sudo、执行上传内容/学生源码、联网安装依赖或读写评测状态。启动及每次解析
读回128 MiB/swap0/PIDs16/CPU不超过一核及文件/FD/core上限，错误身份/无界环境拒绝服务。
解析子进程继承同一个内核文件锁，父进程死亡不能让GC在存活子进程仍持有文件时释放容量。
租约与fence防止旧结果写入；完整工件持久化、候选和引用原子落库后才返回ready。
丢弃/失败/到期先清理私有暂存字节再释放预约，保留不可变操作记录与追加式有限审计。
资源约束服务模板见`deploy/problem-import-worker.service.example`。API和Worker需可访问
同一普通用户的私有目录；数据库采用各自最小角色，导入Worker只通过固定身份公开端口
验证已受理的维护授权。PrivateNetwork模板要求Unix PostgreSQL socket；实际部署前由运维
设置外部私有配置、专用登录与目录，不包含默认口令，也不代表本轮已经生产部署。
管理员发布使用 `/api/v1/admin/problem-publications`，绑定独立操作 UUID、本人导入操作、
候选 UUID 与预期活动版本（首次发布为 null）。`publication.py` 为 CLI 与 Web 共用的
准入与不可变版本写入路径；Web 在事务内再次校验实时维护权限、候选摘要、工件实际字节
和可信节点配置。CLI 保留本机受控入口与原有导入格式，不借用网页会话。
成功维护回执只追加保存，UUID 同内容重试返回原快照，不重复创建 revision 或审计。
UUID 绑定不同内容或活动版本变化返回409；前端应核对最新状态后建立新的操作凭据。
题目状态写入同样要求预期活动版本及 enabled；下架不修改历史版本或删除任何提交。
新提交持有逻辑题目的共享锁直到提交事务结束，下架/恢复持有排他锁，二者明确排序。
管理列表按 slug、版本历史按递减 revision 分页，每页至多100条，仅返回公开元数据。
候选仍通过本人导入操作读取；维护回执仅本人可读。隐藏测试字节、摘要和宿主路径不进入
这些投影。所有管理读取也经实时维护权限检查，缺失发布配置明确503。
`for_accepted_execution` 仅在 submissions 已核对有效租约、源码归属及不可变尝试快照后
调用，读取已发布原版本时允许逻辑题目已下架。它没有外部 UUID 直读端点，不放宽未发布
版本或无有效租约的访问。公开读取与新提交仍检查 enabled。
跨模块访问通过后续 `public.py`；大文件存取通过 `infra/artifacts` 的适配器。
隐藏数据与公开样例分别授权。当前未创建题库表或导入接口。
backend/src/kf_oj/modules/problems/candidate_validation.py
-74
View File
@@ -1,74 +0,0 @@
"""Pure data validation shared by the bounded child and fenced candidate transaction."""
import hashlib
import json
from dataclasses import dataclass
from uuid import uuid5
from kf_oj.contracts.admin_problems import CandidatePreview, LegacyImportBudget
from kf_oj.contracts.judging import ArtifactReference, TestCase
from kf_oj.contracts.problems import PublicLanguageConfiguration
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport
from kf_oj.modules.problems.publication import admit_version
@dataclass(frozen=True)
class ValidatedCandidate:
manifest: dict
preview: CandidatePreview
references: dict[str, ArtifactReference]
@property
def digest(self):
return hashlib.sha256(
json.dumps(
self.manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=True
).encode()
).hexdigest()
def validate_candidate(manifest_data, preview_data, references, policy) -> ValidatedCandidate:
manifest = ImportEnvelope.model_validate(manifest_data).root
preview = CandidatePreview.model_validate(preview_data)
refs = {k: ArtifactReference.model_validate(v) for k, v in references.items()}
legacy = isinstance(manifest, ProblemImport)
if (
preview.slug != manifest.slug
or preview.title != manifest.title
or preview.case_count != len(manifest.cases)
or preview.judging_schema_version != manifest.schema_version
or tuple(v.value for v in preview.languages) != tuple(v.value for v in manifest.languages)
):
raise ValueError("invalid candidate projection")
expected_budget = (
LegacyImportBudget(
compile_limits=manifest.compile_limits,
run_limits=manifest.run_limits,
job_wall_time_us=manifest.job_wall_time_us,
)
if legacy
else None
)
configs = (
()
if legacy
else tuple(
PublicLanguageConfiguration.from_configuration(c)
for c in manifest.language_configurations
)
)
if preview.legacy_budget != expected_budget or preview.language_configurations != configs:
raise ValueError("invalid candidate budget")
names = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
if set(refs) != names:
raise ValueError("invalid hidden references")
cases = tuple(
TestCase(
case_id=uuid5(preview.candidate_id, c.key),
input=refs[c.input_file],
expected=refs[c.expected_file],
)
for c in manifest.cases
)
admit_version(policy, manifest, preview.candidate_id, preview.candidate_id, cases)
return ValidatedCandidate(manifest.model_dump(mode="json"), preview, refs)
backend/src/kf_oj/modules/problems/cli.py
-49
View File
@@ -1,49 +0,0 @@
"""Local, controlled import of ordinary files; no archive extraction or program execution."""
import argparse
import asyncio
from pathlib import Path
from sqlalchemy.exc import SQLAlchemyError
from kf_oj.core.settings import Settings
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.infra.artifacts.store import ArtifactStore
from kf_oj.infra.db.session import build_engine, build_session_factory
from kf_oj.modules.problems.service import ProblemService
async def run(root: Path, manifest: str) -> None:
settings = Settings()
if settings.artifact_root is None or settings.judge_admission is None:
raise ValueError("artifact root and trusted judge admission required")
store = ArtifactStore(
settings.artifact_root,
max_bytes=settings.artifact_max_bytes,
max_files=settings.artifact_max_files,
)
engine = build_engine(settings)
try:
service = ProblemService(
build_session_factory(engine), ArtifactRepository(store), settings.judge_admission
)
version_id = await service.import_directory(root, manifest)
print(f"Problem version published: {version_id}")
finally:
await engine.dispose()
store.close()
def main() -> None:
parser = argparse.ArgumentParser(description="Import immutable problem version")
parser.add_argument("directory", type=Path)
parser.add_argument("--manifest", default="problem.json")
args = parser.parse_args()
try:
asyncio.run(run(args.directory, args.manifest))
except (ValueError, OSError, SQLAlchemyError):
parser.exit(1, "Problem import rejected; check controlled files, budgets and database.\n")
if __name__ == "__main__":
main()
backend/src/kf_oj/modules/problems/import_format.py
-86
View File
@@ -1,86 +0,0 @@
"""Local operator format. Filenames never cross into the judge execution protocol."""
from typing import Annotated, Literal, Self
from pydantic import Field, RootModel, model_validator
from kf_oj.contracts import judging_v2 as v2
from kf_oj.contracts.judging import (
MAX_CASES,
MAX_JOB_WALL_US,
Label,
Language,
RunLimits,
Toolchain,
Version,
WireModel,
)
FileName = Annotated[str, Field(strict=True, pattern=r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$")]
class ImportCase(WireModel):
key: Label
input_file: FileName
expected_file: FileName
class ImportSample(WireModel):
input_file: FileName
output_file: FileName
class ProblemImport(WireModel):
schema_version: Version
slug: Label
title: Annotated[str, Field(strict=True, min_length=1, max_length=160)]
statement_file: FileName
languages: Annotated[tuple[Language, ...], Field(min_length=1, max_length=2)]
toolchain: Toolchain
comparison: Literal["exact_bytes"]
compile_limits: RunLimits
run_limits: RunLimits
job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=MAX_JOB_WALL_US)]
cases: Annotated[tuple[ImportCase, ...], Field(min_length=1, max_length=MAX_CASES)]
samples: Annotated[tuple[ImportSample, ...], Field(max_length=10)]
@model_validator(mode="after")
def unique(self) -> Self:
if len(set(self.languages)) != len(self.languages):
raise ValueError("duplicate language")
if len({case.key for case in self.cases}) != len(self.cases):
raise ValueError("duplicate testcase key")
if len({(c.input_file, c.expected_file) for c in self.cases}) != len(self.cases):
raise ValueError("duplicate testcase reference")
return self
class ProblemImportV2(WireModel):
schema_version: v2.Version
slug: Label
title: Annotated[str, Field(strict=True, min_length=1, max_length=160)]
statement_file: FileName
language_configurations: Annotated[
tuple[v2.LanguageConfiguration, ...], Field(min_length=1, max_length=4)
]
comparison: Literal["exact_bytes"]
cases: Annotated[tuple[ImportCase, ...], Field(min_length=1, max_length=MAX_CASES)]
samples: Annotated[tuple[ImportSample, ...], Field(max_length=10)]
@property
def languages(self) -> tuple[v2.Language, ...]:
return tuple(config.language for config in self.language_configurations)
@model_validator(mode="after")
def unique(self) -> Self:
if len(set(self.languages)) != len(self.languages):
raise ValueError("duplicate language configuration")
if len({case.key for case in self.cases}) != len(self.cases):
raise ValueError("duplicate testcase key")
if len({(case.input_file, case.expected_file) for case in self.cases}) != len(self.cases):
raise ValueError("duplicate testcase reference")
return self
class ImportEnvelope(RootModel[ProblemImport | ProblemImportV2]):
pass
backend/src/kf_oj/modules/problems/import_models.py
-121
View File
@@ -1,121 +0,0 @@
"""Private durable imports and immutable candidates; no grading-state write authority."""
from datetime import datetime
from uuid import UUID
from sqlalchemy import (
BigInteger,
CheckConstraint,
Computed,
DateTime,
ForeignKey,
ForeignKeyConstraint,
Index,
Integer,
SmallInteger,
String,
UniqueConstraint,
text,
)
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
from kf_oj.modules.problems.models import ProblemVersion
class ImportPool(Base):
__tablename__ = "problem_import_pool"
__table_args__ = (
CheckConstraint("id = 1", name="singleton"),
CheckConstraint("reserved_bytes BETWEEN 0 AND 268435456", name="capacity"),
)
id: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="0")
worker_ready_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
worker_policy_digest: Mapped[str | None] = mapped_column(String(64))
class ProblemImportOperation(CreatedAt, Base):
__tablename__ = "problem_import_operations"
__table_args__ = (
CheckConstraint(
"state IN ('uploading','uploaded','validating','ready','failed',"
"'published','discarded','expired')",
name="state",
),
CheckConstraint("archive_digest ~ '^[0-9a-f]{64}$'", name="digest"),
CheckConstraint("archive_bytes BETWEEN 1 AND 67108864", name="bytes"),
CheckConstraint("authorization_version > 0", name="authority"),
CheckConstraint("attempts BETWEEN 0 AND 2 AND fence >= 0", name="attempts"),
CheckConstraint("reserved_bytes IN (0,134217728)", name="reservation"),
CheckConstraint("expires_at > created_at AND updated_at >= created_at", name="times"),
Index("ix_problem_import_owner", "owner_id", "id"),
Index("ix_problem_import_queue", "state", "created_at"),
Index("ix_problem_import_expiry", "expires_at"),
Index("ix_problem_import_reservations", "id", postgresql_where=text("reserved_bytes>0")),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
owner_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
authorization_version: Mapped[int] = mapped_column(BigInteger)
archive_digest: Mapped[str] = mapped_column(String(64))
archive_bytes: Mapped[int] = mapped_column(BigInteger)
state: Mapped[str] = mapped_column(String(16), server_default="uploading")
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
attempts: Mapped[int] = mapped_column(Integer, server_default="0")
fence: Mapped[int] = mapped_column(BigInteger, server_default="0")
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="134217728")
failure_code: Mapped[str | None] = mapped_column(String(32))
candidate_id: Mapped[UUID | None] = mapped_column(
ForeignKey("problem_candidates.id", use_alter=True, name="fk_problem_import_candidate")
)
published_version_id: Mapped[UUID | None] = mapped_column(ForeignKey(ProblemVersion.id))
class ProblemCandidate(CreatedAt, Base):
__tablename__ = "problem_candidates"
__table_args__ = (
UniqueConstraint("operation_id"),
CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="digest"),
CheckConstraint("case_count BETWEEN 1 AND 256", name="cases"),
CheckConstraint("octet_length(preview::text) <= 4194304", name="preview"),
CheckConstraint("octet_length(manifest::text) <= 262144", name="manifest"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
operation_id: Mapped[UUID] = mapped_column(ForeignKey("problem_import_operations.id"))
manifest_digest: Mapped[str] = mapped_column(String(64))
manifest: Mapped[dict] = mapped_column(JSONB)
preview: Mapped[dict] = mapped_column(JSONB)
case_count: Mapped[int] = mapped_column(Integer)
class CandidateCase(Base):
__tablename__ = "problem_candidate_cases"
__table_args__ = (
CheckConstraint("position BETWEEN 0 AND 255", name="position"),
CheckConstraint(
"input_bytes BETWEEN 0 AND 67108864 AND expected_bytes BETWEEN 0 AND 67108864",
name="bytes",
),
ForeignKeyConstraint(
["input_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
),
ForeignKeyConstraint(
["expected_digest", "artifact_state"],
["artifacts.digest", "artifacts.state"],
name="fk_candidate_cases_expected_artifacts",
),
)
candidate_id: Mapped[UUID] = mapped_column(
ForeignKey("problem_candidates.id"), primary_key=True
)
position: Mapped[int] = mapped_column(Integer, primary_key=True)
key: Mapped[str] = mapped_column(String(64))
input_digest: Mapped[str] = mapped_column(String(64))
input_bytes: Mapped[int] = mapped_column(BigInteger)
expected_digest: Mapped[str] = mapped_column(String(64))
expected_bytes: Mapped[int] = mapped_column(BigInteger)
artifact_state: Mapped[str] = mapped_column(String(8), Computed("'ready'", persisted=True))
backend/src/kf_oj/modules/problems/import_policy.py
-11
View File
@@ -1,11 +0,0 @@
"""Shared import availability error and exact policy identity; no lifecycle imports."""
import hashlib
class ImportUnavailable(ValueError):
pass
def policy_digest(policy):
return hashlib.sha256(policy.model_dump_json().encode()).hexdigest()
backend/src/kf_oj/modules/problems/import_worker.py
-430
View File
@@ -1,430 +0,0 @@
"""Fenced ordinary-user import consumption. No student execution or evaluation DB access."""
import asyncio
import json
import os
import resource
import sys
from datetime import timedelta
from pathlib import Path
from uuid import uuid4
from sqlalchemy import delete, func, or_, select, text
from sqlalchemy.exc import DBAPIError
from kf_oj.contracts.admin_problems import MAX_IMPORT_ATTEMPTS, VALIDATION_DEADLINE_SECONDS
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.identity.public import MaintenanceDenied
from kf_oj.modules.problems.candidate_validation import validate_candidate
from kf_oj.modules.problems.import_models import (
CandidateCase,
ImportPool,
ProblemCandidate,
)
from kf_oj.modules.problems.import_models import (
ProblemImportOperation as Operation,
)
from kf_oj.modules.problems.imports import ACTIVE, ImportUnavailable, ProblemImports, policy_digest
def verify_worker_limits(configured_group: Path):
if os.getuid() == 0:
raise ImportUnavailable("import worker must be ordinary")
actual = next(
line.split(":", 2)[2]
for line in Path("/proc/self/cgroup").read_text().splitlines()
if line.startswith("0::")
)
group = Path("/sys/fs/cgroup") / actual.lstrip("/")
if group.resolve() != configured_group.absolute().resolve():
raise ImportUnavailable("import worker cgroup mismatch")
caps = {
n: (group / n).read_text().strip()
for n in ("memory.max", "memory.swap.max", "pids.max", "cpu.max")
}
if (
caps["memory.max"] == "max"
or not 0 < int(caps["memory.max"]) <= 134217728
or caps["memory.swap.max"] != "0"
or caps["pids.max"] == "max"
or not 0 < int(caps["pids.max"]) <= 16
):
raise ImportUnavailable("import worker resource limits unavailable")
quota, period = caps["cpu.max"].split()
if quota == "max" or int(quota) > int(period):
raise ImportUnavailable("import worker CPU limit unavailable")
for name, maximum in (
(resource.RLIMIT_FSIZE, 67108864),
(resource.RLIMIT_NOFILE, 64),
(resource.RLIMIT_CORE, 0),
):
limits = resource.getrlimit(name)
if any(v == resource.RLIM_INFINITY or v > maximum for v in limits):
raise ImportUnavailable("import worker process limits unavailable")
return caps
class ImportWorker:
def __init__(self, sessions, staging, artifacts, authority, audit, policy, cgroup):
(
self.sessions,
self.staging,
self.artifacts,
self.authority,
self.audit,
self.policy,
self.cgroup,
) = sessions, staging, artifacts, authority, audit, policy, cgroup
async def heartbeat(self):
verify_worker_limits(self.cgroup)
async with self.sessions.begin() as session:
safe = await session.scalar(
text("""
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
OR rolreplication OR rolbypassrls)
AND pg_has_role(current_user,'kf_oj_problem_import','USAGE')
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
FROM pg_roles WHERE rolname=current_user
""")
)
if not safe:
raise ImportUnavailable("dedicated unprivileged import login required")
pool = await session.get(ImportPool, 1, with_for_update=True)
if pool is None:
raise ImportUnavailable("import pool unavailable")
pool.worker_ready_until = await session.scalar(
select(func.clock_timestamp())
) + timedelta(seconds=90)
pool.worker_policy_digest = policy_digest(self.policy)
async def _release(self, operation_id):
await asyncio.to_thread(self.staging.remove, operation_id)
async with self.sessions.begin() as session:
pool = await ProblemImports.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
pool.reserved_bytes -= op.reserved_bytes
op.reserved_bytes = 0
op.lease_until = None
async def collect(self):
# Caller owns the root flock, so a live upload/parser cannot be unlinked or
# have its reservation reused, even if its DB lease expired while suspended.
async with self.sessions() as session:
now = await session.scalar(select(func.clock_timestamp()))
names = list(
await session.scalars(
select(Operation.id)
.where(
or_(
Operation.reserved_bytes > 0,
Operation.state.in_(ACTIVE) & (Operation.expires_at <= now),
)
)
.order_by(Operation.updated_at, Operation.id)
.limit(32)
)
)
garbage = list(
await session.scalars(
select(ProblemCandidate.operation_id)
.join(Operation, Operation.id == ProblemCandidate.operation_id)
.where(Operation.state.in_(("discarded", "expired", "failed")))
.limit(32)
)
)
names = list(dict.fromkeys(names + garbage))[:32]
for identifier in names:
async with self.sessions.begin() as session:
await ProblemImports.pool(session)
op = await session.get(Operation, identifier, with_for_update=True)
now = await session.scalar(select(func.clock_timestamp()))
if op.state in ACTIVE and op.expires_at <= now:
op.state = "expired"
op.failure_code = "expired"
op.updated_at = now
op.candidate_id = None
op.fence += 1
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "import_expired"
)
elif op.state == "uploading" and op.lease_until and op.lease_until <= now:
op.state = "failed"
op.failure_code = "upload_interrupted"
op.updated_at = now
op.fence += 1
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "import_failed"
)
elif op.state == "validating" and op.lease_until and op.lease_until <= now:
if op.attempts < MAX_IMPORT_ATTEMPTS:
op.state = "uploaded"
op.failure_code = None
op.lease_until = None
op.fence += 1
op.updated_at = now
await asyncio.to_thread(self.staging.reset_validation, op.id)
else:
op.state = "failed"
op.failure_code = "validation_unavailable"
op.updated_at = now
op.fence += 1
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "import_failed"
)
if op.state in ("discarded", "expired", "failed"):
op.candidate_id = None
await session.flush()
candidate = await session.scalar(
select(ProblemCandidate).where(ProblemCandidate.operation_id == op.id)
)
if candidate:
await session.execute(
delete(CandidateCase).where(CandidateCase.candidate_id == candidate.id)
)
await session.delete(candidate)
release = (
op.state in ("ready", "published", "discarded", "expired", "failed")
and op.reserved_bytes > 0
)
if release:
await self._release(identifier)
# Uncommitted staging objects after a failed transaction are removed only under
# the same kernel lock and only from this service's private, bounded namespace.
for identifier in self.staging.operation_ids():
async with self.sessions() as session:
exists = await session.get(Operation, identifier)
if exists is None:
await asyncio.to_thread(self.staging.remove, identifier)
async def once(self):
await self.heartbeat()
try:
with self.staging.exclusive() as lock:
await self.collect()
async with self.sessions.begin() as session:
await ProblemImports.pool(session)
now = await session.scalar(select(func.clock_timestamp()))
busy = await session.scalar(
select(func.count())
.select_from(Operation)
.where(
Operation.state.in_(("uploading", "validating")),
Operation.lease_until > now,
)
)
if busy:
return False
op = await session.scalar(
select(Operation)
.where(
Operation.state == "uploaded",
Operation.attempts < MAX_IMPORT_ATTEMPTS,
Operation.expires_at > now,
)
.order_by(Operation.created_at, Operation.id)
.limit(1)
.with_for_update()
)
if op is None:
return False
op.attempts += 1
op.fence += 1
op.state = "validating"
op.updated_at = now
op.lease_until = now + timedelta(seconds=VALIDATION_DEADLINE_SECONDS + 15)
identifier, fence, size, digest, actor, epoch = (
op.id,
op.fence,
op.archive_bytes,
op.archive_digest,
op.owner_id,
op.authorization_version,
)
try:
async with self.sessions.begin() as session:
await self.authority.require_reserved_maintenance(session, actor, epoch)
candidate_id = uuid4()
result = await self.parse(identifier, size, digest, candidate_id, lock)
if "failure_code" in result:
await self.fail(identifier, fence, result["failure_code"])
return True
candidate = validate_candidate(
result["manifest"], result["preview"], result["references"], self.policy
)
if (
candidate.digest != result["manifest_digest"]
or candidate.preview.candidate_id != candidate_id
):
raise ValueError("invalid candidate receipt")
directory = self.staging.operation(identifier)
members = self.staging.members(directory)
try:
async with self.sessions.begin() as session:
await self.authority.require_reserved_maintenance(session, actor, epoch)
await ProblemImports.pool(session)
op = await session.get(Operation, identifier, with_for_update=True)
now = await session.scalar(select(func.clock_timestamp()))
if (
op.state != "validating"
or op.fence != fence
or op.lease_until <= now
or op.expires_at <= now
):
raise ValueError("stale candidate receipt")
for name, reference in sorted(
candidate.references.items(), key=lambda item: item[1].digest
):
fd = os.open(
name,
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
dir_fd=members,
)
try:
await self.artifacts.put_file(session, fd, reference)
finally:
os.close(fd)
session.add(
ProblemCandidate(
id=candidate_id,
operation_id=identifier,
manifest_digest=candidate.digest,
manifest=candidate.manifest,
preview=candidate.preview.model_dump(mode="json"),
case_count=candidate.preview.case_count,
)
)
await session.flush()
for position, case in enumerate(candidate.manifest["cases"]):
input_ref, answer_ref = (
candidate.references[case["input_file"]],
candidate.references[case["expected_file"]],
)
session.add(
CandidateCase(
candidate_id=candidate_id,
position=position,
key=case["key"],
input_digest=input_ref.digest,
input_bytes=input_ref.bytes,
expected_digest=answer_ref.digest,
expected_bytes=answer_ref.bytes,
)
)
await session.flush()
# Adoption must not turn an expired lease into a ready candidate.
completed_at = await session.scalar(select(func.clock_timestamp()))
if op.lease_until <= completed_at or op.expires_at <= completed_at:
raise ValueError("candidate completion lease expired")
op.state = "ready"
op.candidate_id = candidate_id
op.updated_at = completed_at
op.lease_until = None
await self.audit.record(
session, actor, epoch, identifier, "import_ready", candidate_id
)
finally:
os.close(members)
os.close(directory)
await self._release(identifier)
except MaintenanceDenied:
await self.fail(identifier, fence, "authority_revoked")
except (ArtifactError, OSError):
await self.fail(identifier, fence, "storage_unavailable")
except (ValueError, DBAPIError):
await self.fail(identifier, fence, "validation_unavailable")
return True
except ArtifactError:
return False # Another live writer holds the one global slot.
async def fail(self, identifier, fence, code):
from kf_oj.contracts.admin_problems import ImportFailure
code = ImportFailure(code).value
async with self.sessions.begin() as session:
await ProblemImports.pool(session)
op = await session.get(Operation, identifier, with_for_update=True)
if op.state != "validating" or op.fence != fence:
return
op.state = "failed"
op.failure_code = code
op.updated_at = await session.scalar(select(func.clock_timestamp()))
op.lease_until = None
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "import_failed"
)
await self._release(identifier)
async def parse(self, identifier, size, digest, candidate, lock):
verify_worker_limits(self.cgroup)
directory = self.staging.operation(identifier)
archive = members = None
try:
archive = os.open(
"archive",
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
dir_fd=directory,
)
members = self.staging.members(directory, create=True)
# Preserve only the trusted runtime import roots, without inheriting DB/
# service credentials. Curated runtimes may keep dependencies outside site-packages.
env = {
"PATH": str(Path(sys.executable).parent),
"PYTHONPATH": os.pathsep.join(
dict.fromkeys(
[str(Path(__file__).parents[3])]
+ [p for p in os.environ.get("PYTHONPATH", "").split(os.pathsep) if p]
)
),
"PYTHONUNBUFFERED": "1",
"PYTHONDONTWRITEBYTECODE": "1",
"PYTHONNOUSERSITE": "1",
"LC_ALL": "C.UTF-8",
}
# Minimal environment may require the trusted dynamic-loader path in a curated VM.
if "LD_LIBRARY_PATH" in os.environ:
env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"]
child = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"kf_oj.workers._archive_child",
str(archive),
str(members),
str(lock),
str(size),
digest,
str(candidate),
self.policy.model_dump_json(),
env=env,
pass_fds=(archive, members, lock),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
try:
async with asyncio.timeout(VALIDATION_DEADLINE_SECONDS + 5):
output, _ = await child.communicate()
except BaseException:
child.kill()
await child.wait()
raise
if child.returncode != 0 or not 0 < len(output) <= 4 * 1024**2:
return {
"failure_code": "validation_deadline"
if child.returncode == -14
else "validation_unavailable"
}
result = json.loads(output)
if not isinstance(result, dict):
raise ValueError("invalid child result")
return result
except TimeoutError:
return {"failure_code": "validation_deadline"}
finally:
if members is not None:
os.close(members)
if archive is not None:
os.close(archive)
os.close(directory)
backend/src/kf_oj/modules/problems/imports.py
-432
View File
@@ -1,432 +0,0 @@
"""Authenticated durable import lifecycle; filesystem work remains behind private adapters."""
import asyncio
import hashlib
import os
from contextlib import asynccontextmanager
from datetime import timedelta
from uuid import UUID
from sqlalchemy import func, select, text
from kf_oj.contracts.admin_problems import (
IMPORT_LIFETIME_SECONDS,
MAX_OWNER_CANDIDATES,
MAX_TOTAL_CANDIDATES,
MAX_UPLOAD_CHUNK_BYTES,
MAX_UPLOAD_CHUNKS,
STAGING_MAX_BYTES,
STAGING_RESERVATION_BYTES,
UPLOAD_DEADLINE_SECONDS,
CandidatePreview,
CreateProblemImport,
ProblemImportCapabilities,
ProblemImportDetail,
ProblemImportPage,
ProblemImportSummary,
)
from kf_oj.infra.artifacts.staging import ImportStaging
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.identity.public import MaintenanceDenied, ProblemMaintenancePort
from kf_oj.modules.operations.public import MaintenanceAuditPort
from kf_oj.modules.problems.import_models import ImportPool, ProblemCandidate
from kf_oj.modules.problems.import_models import ProblemImportOperation as Operation
from kf_oj.modules.problems.import_policy import ImportUnavailable as ImportUnavailable
from kf_oj.modules.problems.import_policy import policy_digest as policy_digest
ACTIVE = ("uploading", "uploaded", "validating", "ready")
class ImportConflict(ValueError):
pass
class ImportBadDigest(ImportConflict):
pass
class ImportMissing(ValueError):
pass
def summary(operation) -> ProblemImportSummary:
return ProblemImportSummary(
operation_id=operation.id,
state=operation.state,
expected_archive_digest=operation.archive_digest,
archive_bytes=operation.archive_bytes,
created_at=operation.created_at,
expires_at=operation.expires_at,
updated_at=operation.updated_at,
failure_code=operation.failure_code,
candidate_id=operation.candidate_id,
published_version_id=operation.published_version_id,
)
class ProblemImports:
def __init__(
self,
sessions,
staging: ImportStaging,
identity: ProblemMaintenancePort,
audit: MaintenanceAuditPort,
policy,
):
self.sessions, self.staging, self.identity, self.audit, self.policy = (
sessions,
staging,
identity,
audit,
policy,
)
@asynccontextmanager
async def write(self, principal):
async with self.sessions.begin() as session:
await session.execute(text("SET LOCAL statement_timeout='5s'"))
await session.execute(text("SET LOCAL lock_timeout='2s'"))
await self.identity.require_problem_maintenance(session, principal)
yield session
@staticmethod
async def pool(session):
pool = await session.get(ImportPool, 1, with_for_update=True)
if pool is None:
raise ImportUnavailable("import capacity unavailable")
actual = await session.scalar(
select(func.coalesce(func.sum(Operation.reserved_bytes), 0)).where(
Operation.reserved_bytes > 0
)
)
if pool.reserved_bytes != actual:
raise ImportUnavailable("import capacity inconsistent")
return pool
async def capabilities(self):
async with self.sessions() as session:
pool = await session.get(ImportPool, 1)
now = await session.scalar(select(func.clock_timestamp()))
available = bool(
pool
and pool.worker_ready_until
and pool.worker_ready_until > now
and pool.worker_policy_digest == policy_digest(self.policy)
)
return ProblemImportCapabilities(
uploads_available=available, validation_available=available
)
async def create(self, principal, request: CreateProblemImport):
# Immutable receipt retries need no filesystem slot, including while a parser/GC
# owns the root lock. Missing receipts still enter the locked creation path below.
async with self.write(principal) as session:
existing = await session.get(Operation, request.operation_id)
if existing:
if (
existing.owner_id != principal.account_id
or existing.archive_digest != request.archive_digest
or existing.archive_bytes != request.archive_bytes
):
raise ImportConflict("operation key already bound")
return summary(existing)
try:
with self.staging.exclusive():
await asyncio.to_thread(self.staging.usage)
async with self.write(principal) as session:
pool = await self.pool(session)
existing = await session.get(
Operation, request.operation_id, with_for_update=True
)
if existing:
if (
existing.owner_id != principal.account_id
or existing.archive_digest != request.archive_digest
or existing.archive_bytes != request.archive_bytes
):
raise ImportConflict("operation key already bound")
return summary(existing)
now = await session.scalar(select(func.clock_timestamp()))
if (
not pool.worker_ready_until
or pool.worker_ready_until <= now
or pool.worker_policy_digest != policy_digest(self.policy)
):
raise ImportUnavailable("import validation unavailable")
global_count = await session.scalar(
select(func.count())
.select_from(Operation)
.where(Operation.state.in_(ACTIVE))
)
owner_count = await session.scalar(
select(func.count())
.select_from(Operation)
.where(
Operation.state.in_(ACTIVE), Operation.owner_id == principal.account_id
)
)
busy = await session.scalar(
select(func.count())
.select_from(Operation)
.where(
Operation.state.in_(("uploading", "validating")),
Operation.lease_until > now,
)
)
if (
global_count >= MAX_TOTAL_CANDIDATES
or owner_count >= MAX_OWNER_CANDIDATES
or busy
or pool.reserved_bytes + STAGING_RESERVATION_BYTES > STAGING_MAX_BYTES
):
raise ImportUnavailable("import capacity exhausted")
operation = Operation(
id=request.operation_id,
owner_id=principal.account_id,
authorization_version=principal.authorization_version,
archive_digest=request.archive_digest,
archive_bytes=request.archive_bytes,
state="uploading",
created_at=now,
updated_at=now,
expires_at=now + timedelta(seconds=IMPORT_LIFETIME_SECONDS),
lease_until=now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15),
attempts=0,
fence=0,
reserved_bytes=STAGING_RESERVATION_BYTES,
)
session.add(operation)
pool.reserved_bytes += STAGING_RESERVATION_BYTES
await self.audit.record(
session,
principal.account_id,
principal.authorization_version,
operation.id,
"upload_created",
)
await session.flush()
return summary(operation)
except ArtifactError:
raise ImportUnavailable("import storage unavailable") from None
async def detail(self, principal, operation_id):
# Recheck live authority even when a principal was injected by an internal caller.
async with self.write(principal) as session:
op = await session.get(Operation, operation_id)
if op is None or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
candidate = (
await session.get(ProblemCandidate, op.candidate_id) if op.candidate_id else None
)
return ProblemImportDetail(
**summary(op).model_dump(),
candidate=CandidatePreview.model_validate(candidate.preview) if candidate else None,
)
async def list(self, principal, after=None, limit=20):
if not 1 <= limit <= 100:
raise ValueError("invalid page limit")
async with self.write(principal) as session:
query = select(Operation).where(Operation.owner_id == principal.account_id)
if after:
query = query.where(Operation.id > after)
rows = list(await session.scalars(query.order_by(Operation.id).limit(limit + 1)))
return ProblemImportPage(
items=tuple(summary(o) for o in rows[:limit]),
next_after=rows[limit - 1].id if len(rows) > limit else None,
)
async def upload(self, principal, operation_id: UUID, chunks, content_length: int | None):
# Reject missing/foreign receipts before consulting shared filesystem capacity.
# The locked transaction below still rechecks authority and ownership before writes.
async with self.write(principal) as session:
op = await session.get(Operation, operation_id)
if op is None or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
if content_length is not None and content_length != op.archive_bytes:
raise ImportConflict("archive length mismatch")
descriptor = operation_fd = None
fence = None
received = 0
chunk_count = 0
digest = hashlib.sha256()
try:
with self.staging.exclusive():
async with self.write(principal) as session:
await self.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
if not op or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
accepted = op.state in ("uploaded", "validating", "ready", "published")
now = await session.scalar(select(func.clock_timestamp()))
if (
not accepted
and (
op.state != "uploading"
or op.lease_until is None
or op.lease_until <= now
)
) or op.expires_at <= now:
raise ImportConflict("upload no longer available")
if content_length is not None and content_length != op.archive_bytes:
raise ImportConflict("archive length mismatch")
if not accepted:
op.fence += 1
fence = op.fence
op.lease_until = now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15)
expected_bytes, expected_digest = op.archive_bytes, op.archive_digest
if not accepted:
operation_fd = self.staging.operation(operation_id, create=True)
descriptor = os.open(
"archive",
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
0o600,
dir_fd=operation_fd,
)
async with asyncio.timeout(UPLOAD_DEADLINE_SECONDS):
async for chunk in chunks:
chunk_count += 1
received += len(chunk)
if (
type(chunk) is not bytes
or len(chunk) > MAX_UPLOAD_CHUNK_BYTES
or chunk_count > MAX_UPLOAD_CHUNKS
or received > expected_bytes
):
raise ImportConflict("archive bytes outside policy")
digest.update(chunk)
if accepted:
continue
# Cancellation waits for the finite writer outside the event loop.
task = asyncio.create_task(asyncio.to_thread(write_all, descriptor, chunk))
try:
await asyncio.shield(task)
except BaseException:
await task
raise
if received != expected_bytes or digest.hexdigest() != expected_digest:
raise ImportBadDigest("archive digest or length mismatch")
if accepted:
async with self.write(principal) as session:
op = await session.get(Operation, operation_id)
if not op or op.state not in (
"uploaded",
"validating",
"ready",
"published",
):
raise ImportConflict("import receipt no longer available")
return summary(op)
await asyncio.to_thread(os.fsync, descriptor)
os.close(descriptor)
descriptor = None
os.fsync(operation_fd)
async with self.write(principal) as session:
await self.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
now = await session.scalar(select(func.clock_timestamp()))
if op.state != "uploading" or op.fence != fence or op.lease_until <= now:
raise ImportConflict("upload receipt expired")
op.state = "uploaded"
op.lease_until = None
op.updated_at = now
await self.audit.record(
session, op.owner_id, op.authorization_version, op.id, "upload_received"
)
result = summary(op)
return result
except BaseException as error:
# Failure bookkeeping cannot authorize a new business write. It only fences this
# accepted upload and releases capacity after physical bytes have been removed.
if descriptor is not None:
os.close(descriptor)
descriptor = None
if operation_fd is not None:
os.close(operation_fd)
operation_fd = None
if fence is not None:
try:
with self.staging.exclusive():
await asyncio.to_thread(self.staging.remove, operation_id)
async with self.sessions.begin() as session:
pool = await self.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
if op and op.state == "uploading" and op.fence == fence:
op.state = "failed"
op.failure_code = (
"upload_deadline"
if isinstance(error, TimeoutError)
else "digest_mismatch"
if isinstance(error, ImportBadDigest)
else "authority_revoked"
if isinstance(error, MaintenanceDenied)
else "storage_unavailable"
if isinstance(error, OSError)
else "upload_interrupted"
)
op.updated_at = await session.scalar(select(func.clock_timestamp()))
op.lease_until = None
pool.reserved_bytes -= op.reserved_bytes
op.reserved_bytes = 0
await self.audit.record(
session,
op.owner_id,
op.authorization_version,
op.id,
"import_failed",
)
except (ArtifactError, OSError):
pass # Durable reservation retained for fenced GC.
raise
finally:
if descriptor is not None:
os.close(descriptor)
if operation_fd is not None:
os.close(operation_fd)
async def discard(self, principal, operation_id):
async with self.write(principal) as session:
op = await session.get(Operation, operation_id)
if op is None or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
if op.state == "discarded":
return summary(op)
try:
with self.staging.exclusive():
async with self.write(principal) as session:
pool = await self.pool(session)
op = await session.get(Operation, operation_id, with_for_update=True)
if not op or op.owner_id != principal.account_id:
raise ImportMissing("import unavailable")
if op.state == "published":
raise ImportConflict("published import cannot be discarded")
if op.state == "discarded":
return summary(op)
await asyncio.to_thread(self.staging.remove, operation_id)
op.state = "discarded"
op.candidate_id = None
op.failure_code = None
op.lease_until = None
op.fence += 1
op.updated_at = await session.scalar(select(func.clock_timestamp()))
pool.reserved_bytes -= op.reserved_bytes
op.reserved_bytes = 0
await self.audit.record(
session,
principal.account_id,
principal.authorization_version,
op.id,
"import_discarded",
)
return summary(op)
except ArtifactError:
raise ImportUnavailable("import storage unavailable") from None
def write_all(descriptor, data):
remaining = memoryview(data)
while remaining:
size = os.write(descriptor, remaining)
if size <= 0:
raise OSError("short staging write")
remaining = remaining[size:]
backend/src/kf_oj/modules/problems/local_files.py
-48
View File
@@ -1,48 +0,0 @@
import os
import re
import stat
from pathlib import Path
class LocalImportFiles:
def __init__(self, root: Path):
root = root.absolute()
if root.resolve() != root:
raise ValueError("import directory must not contain links")
self.descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
info = os.fstat(self.descriptor)
if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
self.close()
raise ValueError("import directory must be private and owned")
def close(self):
os.close(self.descriptor)
def read(self, filename: str, maximum: int) -> bytes:
if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}", filename):
raise ValueError("invalid local filename")
try:
descriptor = os.open(
filename, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=self.descriptor
)
with os.fdopen(descriptor, "rb") as stream:
before = os.fstat(stream.fileno())
if (
not stat.S_ISREG(before.st_mode)
or before.st_nlink != 1
or before.st_uid != os.getuid()
or before.st_size > maximum
):
raise ValueError("invalid local input file")
data = stream.read(maximum + 1)
after = os.fstat(stream.fileno())
if (
len(data) > maximum
or before.st_size != len(data)
or before.st_mtime_ns != after.st_mtime_ns
or before.st_ctime_ns != after.st_ctime_ns
):
raise ValueError("local input changed during import")
return data
except OSError:
raise ValueError("local input unavailable") from None
backend/src/kf_oj/modules/problems/maintenance.py
-324
View File
@@ -1,324 +0,0 @@
"""Administrator transactions; no evaluation writes or private data in projections."""
import asyncio
import hashlib
import json
from contextlib import asynccontextmanager
from uuid import uuid4, uuid5
from sqlalchemy import func, select, text
from kf_oj.contracts.admin_problems import CandidatePreview
from kf_oj.contracts.judging import ArtifactReference, TestCase
from kf_oj.contracts.problem_maintenance import (
MaintainedProblem,
MaintainedProblemPage,
MaintainedVersion,
MaintainedVersionPage,
MaintenanceReceipt,
)
from kf_oj.modules.identity.public import ProblemMaintenancePort
from kf_oj.modules.operations.public import MaintenanceAuditPort
from kf_oj.modules.problems import repository
from kf_oj.modules.problems.candidate_validation import validate_candidate
from kf_oj.modules.problems.import_format import ImportEnvelope
from kf_oj.modules.problems.import_models import (
CandidateCase,
ProblemCandidate,
ProblemImportOperation,
)
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
from kf_oj.modules.problems.maintenance_models import ProblemMaintenanceOperation
from kf_oj.modules.problems.models import Problem, ProblemCase, ProblemVersion
from kf_oj.modules.problems.publication import admit_version, publish_version
from kf_oj.modules.problems.service import ProblemService
def project_problem(problem, version):
return MaintainedProblem(
id=problem.id,
slug=problem.slug,
enabled=problem.enabled,
active_version_id=problem.active_version_id,
revision=version.revision if version else 0,
title=version.title if version else None,
created_at=problem.created_at,
)
class ProblemMaintenance:
def __init__(
self,
sessions,
identity: ProblemMaintenancePort,
audit: MaintenanceAuditPort,
artifacts=None,
policy=None,
):
self.sessions, self.identity, self.audit = sessions, identity, audit
self.artifacts, self.policy = artifacts, policy
self._publishing = asyncio.Semaphore(2)
@asynccontextmanager
async def transaction(self, principal):
async with self.sessions.begin() as session:
await session.execute(text("SET LOCAL statement_timeout='5s'"))
await session.execute(text("SET LOCAL lock_timeout='2s'"))
await self.identity.require_problem_maintenance(session, principal)
yield session
async def page(self, principal, *, after=None, limit=20, enabled=None):
if not 1 <= limit <= 100:
raise ValueError("invalid page limit")
async with self.transaction(principal) as session:
q = select(Problem, ProblemVersion).outerjoin(
ProblemVersion, Problem.active_version_id == ProblemVersion.id
)
if after is not None:
q = q.where(Problem.slug > after)
if enabled is not None:
q = q.where(Problem.enabled == enabled)
rows = list(await session.execute(q.order_by(Problem.slug).limit(limit + 1)))
return MaintainedProblemPage(
items=tuple(project_problem(*r) for r in rows[:limit]),
next_after=rows[limit - 1][0].slug if len(rows) > limit else None,
)
async def by_slug(self, principal, slug):
async with self.transaction(principal) as session:
row = (
await session.execute(
select(Problem, ProblemVersion)
.outerjoin(ProblemVersion, Problem.active_version_id == ProblemVersion.id)
.where(Problem.slug == slug)
)
).one_or_none()
if row is None:
raise ImportMissing("problem unavailable")
return project_problem(*row)
async def detail(self, principal, problem_id):
async with self.transaction(principal) as session:
problem = await session.get(Problem, problem_id)
if problem is None:
raise ImportMissing("problem unavailable")
version = (
await session.get(ProblemVersion, problem.active_version_id)
if problem.active_version_id
else None
)
return project_problem(problem, version)
async def versions(self, principal, problem_id, *, before=None, limit=20):
if not 1 <= limit <= 100 or (before is not None and before < 1):
raise ValueError("invalid page limit")
async with self.transaction(principal) as session:
problem = await session.get(Problem, problem_id)
if problem is None:
raise ImportMissing("problem unavailable")
count = (
select(func.count())
.where(ProblemCase.version_id == ProblemVersion.id)
.scalar_subquery()
)
q = select(ProblemVersion, count).where(
ProblemVersion.problem_id == problem_id, ProblemVersion.published_at.is_not(None)
)
if before is not None:
q = q.where(ProblemVersion.revision < before)
rows = list(
await session.execute(q.order_by(ProblemVersion.revision.desc()).limit(limit + 1))
)
return MaintainedVersionPage(
items=tuple(
MaintainedVersion(
**ProblemService._summary(problem, v).model_dump(),
published_at=v.published_at,
case_count=n,
)
for v, n in rows[:limit]
),
next_before=rows[limit - 1][0].revision if len(rows) > limit else None,
)
async def receipt(self, principal, operation_id):
async with self.transaction(principal) as session:
row = await session.get(ProblemMaintenanceOperation, operation_id)
if row is None or row.owner_id != principal.account_id:
raise ImportMissing("operation unavailable")
return MaintenanceReceipt.model_validate(row.receipt)
async def _replay(self, session, principal, action, body, target):
digest = hashlib.sha256(
json.dumps(
{"action": action, "target": str(target), "request": body.model_dump(mode="json")},
sort_keys=True,
separators=(",", ":"),
).encode()
).hexdigest()
key = int.from_bytes(hashlib.sha256(body.operation_id.bytes).digest()[:4], signed=True)
await session.execute(text("SELECT pg_advisory_xact_lock(1262907470,:key)"), {"key": key})
row = await session.get(ProblemMaintenanceOperation, body.operation_id)
if row is not None:
if row.owner_id != principal.account_id:
raise ImportMissing("operation unavailable")
if row.action != action or row.request_digest != digest:
raise ImportConflict("operation UUID already bound")
return digest, MaintenanceReceipt.model_validate(row.receipt)
return digest, None
async def _record(
self, session, principal, request, action, digest, problem, version, published=None
):
receipt = MaintenanceReceipt(
operation_id=request.operation_id,
action=action,
completed_at=await session.scalar(select(func.clock_timestamp())),
problem=project_problem(problem, version),
published_version_id=published,
)
session.add(
ProblemMaintenanceOperation(
id=request.operation_id,
owner_id=principal.account_id,
authorization_version=principal.authorization_version,
action=action,
request_digest=digest,
receipt=receipt.model_dump(mode="json"),
)
)
await self.audit.record(
session,
principal.account_id,
principal.authorization_version,
request.operation_id,
{"publish": "published", "disable": "disabled", "restore": "restored"}[action],
published if action == "publish" else problem.id,
)
await session.flush()
return receipt
async def publish(self, principal, request):
if self.artifacts is None or self.policy is None or self._publishing.locked():
raise ImportUnavailable("publication unavailable")
async with self._publishing:
async with self.transaction(principal) as session:
digest, replay = await self._replay(
session, principal, "publish", request, request.import_operation_id
)
if replay:
return replay
op = await session.get(
ProblemImportOperation, request.import_operation_id, with_for_update=True
)
if op is None or op.owner_id != principal.account_id:
raise ImportMissing("candidate unavailable")
if (
op.state != "ready"
or op.candidate_id != request.candidate_id
or op.authorization_version != principal.authorization_version
or op.expires_at <= await session.scalar(select(func.clock_timestamp()))
):
raise ImportConflict("candidate no longer publishable")
candidate = await session.get(ProblemCandidate, op.candidate_id)
if candidate is None or candidate.operation_id != op.id:
raise ImportUnavailable("candidate unavailable")
manifest = ImportEnvelope.model_validate(candidate.manifest).root
preview = CandidatePreview.model_validate(candidate.preview)
rows = list(
await session.scalars(
select(CandidateCase)
.where(CandidateCase.candidate_id == candidate.id)
.order_by(CandidateCase.position)
)
)
if len(rows) != len(manifest.cases) or len(rows) != candidate.case_count:
raise ImportUnavailable("candidate references unavailable")
refs = {}
for position, (row, c) in enumerate(zip(rows, manifest.cases, strict=True)):
if row.position != position or row.key != c.key:
raise ImportUnavailable("candidate references unavailable")
for name, ref in (
(
c.input_file,
ArtifactReference(digest=row.input_digest, bytes=row.input_bytes),
),
(
c.expected_file,
ArtifactReference(digest=row.expected_digest, bytes=row.expected_bytes),
),
):
if name in refs and refs[name] != ref:
raise ImportUnavailable("candidate references unavailable")
refs[name] = ref
validated = validate_candidate(
candidate.manifest, candidate.preview, refs, self.policy
)
if (
validated.digest != candidate.manifest_digest
or preview.candidate_id != candidate.id
):
raise ImportUnavailable("candidate integrity unavailable")
await repository.lock_slug(session, manifest.slug)
problem = await repository.lock_problem(session, manifest.slug)
actual = problem.active_version_id if problem else None
if actual != request.expected_active_version_id or (
problem and not problem.enabled
):
raise ImportConflict("active version or availability changed")
if problem is None:
problem = Problem(id=uuid4(), slug=manifest.slug)
session.add(problem)
await session.flush()
version_id = uuid4()
cases = tuple(
TestCase(
case_id=uuid5(version_id, c.key),
input=refs[c.input_file],
expected=refs[c.expected_file],
)
for c in manifest.cases
)
admit_version(self.policy, manifest, version_id, problem.id, cases)
# Candidate FKs protect all referenced ready metadata from GC. The
# private adapter additionally re-hashes disk bytes without buffering them.
for ref in sorted(
{r.digest: r for r in refs.values()}.values(), key=lambda r: r.digest
):
await self.artifacts.check_metadata(session, ref)
task = asyncio.create_task(asyncio.to_thread(self.artifacts.store.verify, ref))
try:
await asyncio.shield(task)
except BaseException:
await task
raise
version = await publish_version(
session, problem, manifest, preview, version_id, cases
)
op.state, op.published_version_id = "published", version.id
op.updated_at = await session.scalar(select(func.clock_timestamp()))
return await self._record(
session, principal, request, "publish", digest, problem, version, version.id
)
async def availability(self, principal, problem_id, request, *, enabled):
action = "restore" if enabled else "disable"
async with self.transaction(principal) as session:
digest, replay = await self._replay(session, principal, action, request, problem_id)
if replay:
return replay
problem = await session.get(Problem, problem_id, with_for_update=True)
if problem is None:
raise ImportMissing("problem unavailable")
if (
problem.active_version_id != request.expected_active_version_id
or problem.enabled != request.expected_enabled
or problem.enabled == enabled
):
raise ImportConflict("active version or availability changed")
version = await session.get(ProblemVersion, problem.active_version_id)
if version is None or version.published_at is None:
raise ImportUnavailable("published version unavailable")
problem.enabled = enabled
await session.flush()
return await self._record(session, principal, request, action, digest, problem, version)
backend/src/kf_oj/modules/problems/maintenance_models.py
-26
View File
@@ -1,26 +0,0 @@
"""Append-only actor-bound receipts; replay never re-applies a published operation."""
from uuid import UUID
from sqlalchemy import BigInteger, CheckConstraint, ForeignKey, String
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
class ProblemMaintenanceOperation(CreatedAt, Base):
__tablename__ = "problem_maintenance_operations"
__table_args__ = (
CheckConstraint("action IN ('publish','disable','restore')", name="action"),
CheckConstraint("request_digest ~ '^[0-9a-f]{64}$'", name="digest"),
CheckConstraint("authorization_version > 0", name="authority"),
CheckConstraint("octet_length(receipt::text) <= 4096", name="receipt"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
owner_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
authorization_version: Mapped[int] = mapped_column(BigInteger)
action: Mapped[str] = mapped_column(String(16))
request_digest: Mapped[str] = mapped_column(String(64))
receipt: Mapped[dict] = mapped_column(JSONB)
backend/src/kf_oj/modules/problems/models.py
-110
View File
@@ -1,110 +0,0 @@
"""Immutable published versions, ordered cases, and mutable public availability."""
from datetime import datetime
from uuid import UUID
from sqlalchemy import (
BigInteger,
Boolean,
CheckConstraint,
Computed,
DateTime,
ForeignKey,
ForeignKeyConstraint,
Index,
Integer,
SmallInteger,
String,
Text,
UniqueConstraint,
)
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
class Problem(CreatedAt, Base):
__tablename__ = "problems"
__table_args__ = (
CheckConstraint("slug ~ '^[a-z0-9][a-z0-9_-]{0,63}$'", name="slug"),
ForeignKeyConstraint(
["active_version_id", "id"],
["problem_versions.id", "problem_versions.problem_id"],
name="fk_problems_active_version",
use_alter=True,
deferrable=True,
initially="DEFERRED",
),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
slug: Mapped[str] = mapped_column(String(64), unique=True)
active_version_id: Mapped[UUID | None]
enabled: Mapped[bool] = mapped_column(Boolean, server_default="true")
class ProblemVersion(CreatedAt, Base):
__tablename__ = "problem_versions"
__table_args__ = (
UniqueConstraint("id", "problem_id"),
UniqueConstraint("problem_id", "revision"),
CheckConstraint("revision > 0", name="revision"),
CheckConstraint("octet_length(statement) BETWEEN 1 AND 262144", name="statement_bytes"),
CheckConstraint("char_length(title) BETWEEN 1 AND 160", name="title_size"),
CheckConstraint("comparison = 'exact_bytes'", name="comparison"),
CheckConstraint("job_wall_time_us BETWEEN 1000 AND 3600000000", name="job_wall"),
CheckConstraint("judging_schema_version IN (1,2)", name="judging_schema"),
CheckConstraint(
"(judging_schema_version = 1 AND language_configurations IS NULL "
"AND toolchain IS NOT NULL AND compile_limits IS NOT NULL "
"AND run_limits IS NOT NULL AND job_wall_time_us IS NOT NULL) OR "
"(judging_schema_version = 2 AND language_configurations IS NOT NULL "
"AND toolchain IS NULL AND compile_limits IS NULL "
"AND run_limits IS NULL AND job_wall_time_us IS NULL "
"AND public.kf_valid_language_configurations(languages, language_configurations))",
name="language_configurations",
),
Index("ix_problem_versions_public", "problem_id", "published_at"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
problem_id: Mapped[UUID] = mapped_column(ForeignKey("problems.id"))
revision: Mapped[int] = mapped_column(Integer)
title: Mapped[str] = mapped_column(String(160))
statement: Mapped[str] = mapped_column(Text)
languages: Mapped[list[str]] = mapped_column(JSONB)
judging_schema_version: Mapped[int] = mapped_column(SmallInteger, server_default="1")
language_configurations: Mapped[list[dict] | None] = mapped_column(JSONB(none_as_null=True))
toolchain: Mapped[dict | None] = mapped_column(JSONB(none_as_null=True))
compile_limits: Mapped[dict | None] = mapped_column(JSONB(none_as_null=True))
run_limits: Mapped[dict | None] = mapped_column(JSONB(none_as_null=True))
job_wall_time_us: Mapped[int | None] = mapped_column(BigInteger)
comparison: Mapped[str] = mapped_column(String(16), server_default="exact_bytes")
public_samples: Mapped[list[dict]] = mapped_column(JSONB)
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class ProblemCase(Base):
__tablename__ = "problem_cases"
__table_args__ = (
UniqueConstraint("id", "version_id"),
UniqueConstraint("version_id", "position"),
CheckConstraint("position BETWEEN 0 AND 255", name="position"),
ForeignKeyConstraint(
["input_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
),
ForeignKeyConstraint(
["expected_digest", "artifact_state"],
["artifacts.digest", "artifacts.state"],
name="fk_problem_cases_expected_digest_artifacts",
),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
version_id: Mapped[UUID] = mapped_column(ForeignKey("problem_versions.id"))
position: Mapped[int]
input_digest: Mapped[str] = mapped_column(String(64))
expected_digest: Mapped[str] = mapped_column(String(64))
artifact_state: Mapped[str] = mapped_column(String(8), Computed("'ready'", persisted=True))
backend/src/kf_oj/modules/problems/operations.py
-64
View File
@@ -1,64 +0,0 @@
"""Read-only import projection for the ordinary operator CLI; no private archive data."""
from sqlalchemy import text
from kf_oj.modules.problems.import_models import ImportPool
from kf_oj.modules.problems.import_policy import ImportUnavailable, policy_digest
class ImportOperationQueries:
def __init__(self, sessions, policy):
self.sessions, self.policy = sessions, policy
async def status(self) -> dict:
async with self.sessions() as session, session.begin():
await session.execute(text("SET TRANSACTION READ ONLY"))
await session.execute(text("SET LOCAL statement_timeout='1500'"))
await session.execute(text("SET LOCAL lock_timeout='500'"))
authorized = await session.scalar(
text("""
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
OR rolreplication OR rolbypassrls)
AND pg_has_role(current_user,'kf_oj_problem_import','USAGE')
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','MEMBER')
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
FROM pg_roles WHERE rolname=session_user
""")
)
if authorized is not True:
raise ImportUnavailable("dedicated import observation role required")
pool = await session.get(ImportPool, 1)
row = (
(
await session.execute(
text("""
SELECT clock_timestamp() now,
(SELECT count(*) FROM (SELECT 1 FROM problem_import_operations
WHERE state='uploaded' LIMIT 129) q) uploaded,
(SELECT count(*) FROM (SELECT 1 FROM problem_import_operations
WHERE state='validating' LIMIT 129) q) validating,
(SELECT coalesce(sum(reserved_bytes),0) FROM (
SELECT reserved_bytes FROM problem_import_operations
WHERE reserved_bytes>0 LIMIT 129) q) actual_reserved
""")
)
)
.mappings()
.one()
)
if pool is None or pool.reserved_bytes != row["actual_reserved"]:
raise ImportUnavailable("import reservation consistency unknown")
return {
"schema_version": 1,
"worker_ready": bool(
pool.worker_ready_until
and pool.worker_ready_until > row["now"]
and pool.worker_policy_digest == policy_digest(self.policy)
),
"reserved_bytes": pool.reserved_bytes,
"uploaded_capped": min(row["uploaded"], 128),
"validating_capped": min(row["validating"], 128),
"truncated": row["uploaded"] > 128 or row["validating"] > 128,
}
backend/src/kf_oj/modules/problems/public.py
-84
View File
@@ -1,84 +0,0 @@
"""Authorized immutable version projection; hidden references stay out of public DTOs."""
from dataclasses import dataclass
from typing import Protocol
from uuid import UUID
from kf_oj.contracts import judging as v1
from kf_oj.contracts import judging_v2 as v2
from kf_oj.contracts.judging_versions import Manifest
class ProblemUnavailable(ValueError):
pass
@dataclass(frozen=True)
class JudgingVersion:
version_id: UUID
problem_id: UUID
languages: tuple[v1.Language | v2.Language, ...]
toolchain: v1.Toolchain | None
compile_limits: v1.RunLimits | None
run_limits: v1.RunLimits | None
job_wall_time_us: int | None
cases: tuple[v1.TestCase, ...]
judging_schema_version: int = 1
language_configurations: tuple[v2.LanguageConfiguration, ...] = ()
def manifest(
self,
evaluation_id: UUID,
submission_id: UUID,
language: v1.Language | v2.Language,
source: v1.ArtifactReference,
) -> Manifest:
"""Build only from the immutable authorized version, preserving v1 wire bytes."""
if language not in self.languages:
raise ProblemUnavailable("language unavailable")
fields = dict(
schema_version=self.judging_schema_version,
kind="judge.manifest",
evaluation_id=evaluation_id,
submission_id=submission_id,
problem_version_id=self.version_id,
language=language,
source=source,
cases=self.cases,
comparison="exact_bytes",
)
if self.judging_schema_version == 1:
if self.language_configurations:
raise ProblemUnavailable("inconsistent legacy version")
return v1.JobManifest(
**fields,
toolchain=self.toolchain,
compile_limits=self.compile_limits,
run_limits=self.run_limits,
job_wall_time_us=self.job_wall_time_us,
)
if self.judging_schema_version != 2 or any(
field is not None
for field in (
self.toolchain,
self.compile_limits,
self.run_limits,
self.job_wall_time_us,
)
):
raise ProblemUnavailable("unsupported judging configuration")
if tuple(c.language for c in self.language_configurations) != self.languages:
raise ProblemUnavailable("inconsistent language configurations")
configuration = next(c for c in self.language_configurations if c.language == language)
return v2.JobManifest(
**fields,
**configuration.model_dump(exclude={"language"}),
)
class ProblemPort(Protocol):
async def for_submission(self, session, version_id: UUID) -> JudgingVersion: ...
async def for_accepted_execution(self, session, version_id: UUID) -> JudgingVersion:
"""Require an already-validated live execution lease; allow a disabled published version."""
...
backend/src/kf_oj/modules/problems/publication.py
-82
View File
@@ -1,82 +0,0 @@
"""One admission and immutable publication path for local and web imports."""
from uuid import uuid4
from sqlalchemy import func, select
from kf_oj.contracts import judging_v2 as v2
from kf_oj.contracts.judging import MAX_SOURCE_BYTES, ArtifactReference
from kf_oj.modules.problems import repository
from kf_oj.modules.problems.import_format import ProblemImport
from kf_oj.modules.problems.models import ProblemCase, ProblemVersion
from kf_oj.modules.problems.public import JudgingVersion, ProblemUnavailable
def admit_version(policy, manifest, version_id, problem_id, cases):
legacy = isinstance(manifest, ProblemImport)
projection = JudgingVersion(
version_id,
problem_id,
manifest.languages,
manifest.toolchain if legacy else None,
manifest.compile_limits if legacy else None,
manifest.run_limits if legacy else None,
manifest.job_wall_time_us if legacy else None,
cases,
manifest.schema_version,
() if legacy else manifest.language_configurations,
)
for language in manifest.languages:
v2.admit_manifest(
policy,
projection.manifest(
uuid4(),
uuid4(),
language,
ArtifactReference(digest="0" * 64, bytes=MAX_SOURCE_BYTES),
),
1,
)
return projection
async def publish_version(session, problem, manifest, preview, version_id, cases):
"""Caller holds the logical slug/row locks and has made all artifacts durable."""
if not problem.enabled:
raise ProblemUnavailable("problem disabled")
legacy = isinstance(manifest, ProblemImport)
version = ProblemVersion(
id=version_id,
problem_id=problem.id,
revision=await repository.next_revision(session, problem.id),
title=manifest.title,
statement=preview.statement,
languages=[lang.value for lang in manifest.languages],
judging_schema_version=manifest.schema_version,
language_configurations=None
if legacy
else [c.model_dump(mode="json") for c in manifest.language_configurations],
toolchain=manifest.toolchain.model_dump(mode="json") if legacy else None,
compile_limits=manifest.compile_limits.model_dump(mode="json") if legacy else None,
run_limits=manifest.run_limits.model_dump(mode="json") if legacy else None,
job_wall_time_us=manifest.job_wall_time_us if legacy else None,
comparison=manifest.comparison,
public_samples=[s.model_dump(mode="json") for s in preview.samples],
)
session.add(version)
await session.flush()
for position, case in enumerate(cases):
session.add(
ProblemCase(
id=case.case_id,
version_id=version_id,
position=position,
input_digest=case.input.digest,
expected_digest=case.expected.digest,
)
)
await session.flush()
version.published_at = await session.scalar(select(func.now()))
problem.active_version_id = version_id
await session.flush()
return version
backend/src/kf_oj/modules/problems/repository.py
-74
View File
@@ -1,74 +0,0 @@
import hashlib
from uuid import UUID
from sqlalchemy import func, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from kf_oj.modules.problems.models import Problem, ProblemCase, ProblemVersion
async def lock_slug(session: AsyncSession, slug: str) -> None:
# Two-key namespace is disjoint from the one-key artifact advisory locks.
key = int.from_bytes(hashlib.sha256(slug.encode()).digest()[:4], signed=True)
await session.execute(text("SELECT pg_advisory_xact_lock(1262907468,:key)"), {"key": key})
async def lock_problem(session: AsyncSession, slug: str) -> Problem | None:
return await session.scalar(select(Problem).where(Problem.slug == slug).with_for_update())
async def next_revision(session: AsyncSession, problem_id: UUID) -> int:
latest = await session.scalar(
select(func.max(ProblemVersion.revision)).where(ProblemVersion.problem_id == problem_id)
)
return (latest or 0) + 1
async def published_version(session: AsyncSession, version_id: UUID, *, admission=False):
query = (
select(ProblemVersion, Problem)
.join(Problem, Problem.id == ProblemVersion.problem_id)
.where(ProblemVersion.id == version_id, ProblemVersion.published_at.is_not(None))
)
if admission:
# A new submission commits while this shared problem lock is held. A
# concurrent disable either precedes admission or follows that commit.
query = query.where(Problem.enabled.is_(True)).with_for_update(read=True, of=Problem)
return (await session.execute(query)).one_or_none()
async def public_version(session: AsyncSession, version_id: UUID, *, admission=False):
if admission:
return await published_version(session, version_id, admission=True)
return (
await session.execute(
select(ProblemVersion, Problem)
.join(Problem, Problem.id == ProblemVersion.problem_id)
.where(
ProblemVersion.id == version_id,
ProblemVersion.published_at.is_not(None),
Problem.enabled.is_(True),
)
)
).one_or_none()
async def ordered_cases(session: AsyncSession, version_id: UUID):
return list(
await session.scalars(
select(ProblemCase)
.where(ProblemCase.version_id == version_id)
.order_by(ProblemCase.position)
)
)
async def list_public(session: AsyncSession, after: str | None, limit: int):
query = (
select(Problem, ProblemVersion)
.join(ProblemVersion, ProblemVersion.id == Problem.active_version_id)
.where(Problem.enabled.is_(True), ProblemVersion.published_at.is_not(None))
)
if after is not None:
query = query.where(Problem.slug > after)
return list(await session.execute(query.order_by(Problem.slug).limit(limit)))
backend/src/kf_oj/modules/problems/service.py
-228
View File
@@ -1,228 +0,0 @@
import base64
from pathlib import Path
from types import SimpleNamespace
from uuid import UUID, uuid4, uuid5
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker
from kf_oj.contracts import judging_v2 as v2
from kf_oj.contracts.judging import (
MAX_PACKAGE_BYTES,
MAX_SOURCE_BYTES,
ArtifactReference,
Language,
RunLimits,
TestCase,
Toolchain,
decode_wire,
)
from kf_oj.contracts.problems import (
ProblemDetail,
ProblemPage,
ProblemSummary,
PublicLanguageConfiguration,
PublicSample,
)
from kf_oj.infra.artifacts.models import Artifact
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.modules.problems import repository
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport
from kf_oj.modules.problems.local_files import LocalImportFiles
from kf_oj.modules.problems.models import Problem
from kf_oj.modules.problems.public import JudgingVersion, ProblemUnavailable
from kf_oj.modules.problems.publication import admit_version, publish_version
class ProblemService:
def __init__(
self,
sessions: async_sessionmaker,
artifacts: ArtifactRepository | None = None,
policy: v2.NodeAdmission | None = None,
):
self.sessions, self.artifacts, self.policy = sessions, artifacts, policy
async def import_directory(self, root: Path, manifest_file: str = "problem.json") -> UUID:
if self.artifacts is None or self.policy is None:
raise ValueError("artifact store and trusted node policy required for import")
files = LocalImportFiles(root)
try:
manifest = decode_wire(ImportEnvelope, files.read(manifest_file, 256 * 1024)).root
stdout_limit = (
manifest.run_limits.stdout_bytes
if isinstance(manifest, ProblemImport)
else min(c.run_limits.stdout_bytes for c in manifest.language_configurations)
)
statement = files.read(manifest.statement_file, 256 * 1024).decode("utf-8")
if not statement or "\x00" in statement or "\x00" in manifest.title:
raise ValueError("invalid public text")
# Read the complete bounded package before writing anything. Public samples are
# separately declared, at most 64 KiB each, and never inferred from hidden tests.
remaining = MAX_PACKAGE_BYTES - MAX_SOURCE_BYTES
payloads = []
for case in manifest.cases:
data = files.read(case.input_file, remaining)
remaining -= len(data)
answer = files.read(case.expected_file, min(remaining, stdout_limit))
remaining -= len(answer)
payloads.append((data, answer))
samples = tuple(
PublicSample(
input=base64.b64encode(files.read(s.input_file, 65536)).decode("ascii"),
output=base64.b64encode(files.read(s.output_file, 65536)).decode("ascii"),
)
for s in manifest.samples
)
finally:
files.close()
version_id = uuid4()
cases = tuple(
TestCase(
case_id=uuid5(version_id, case.key),
input=self.artifacts.store.reference(data),
expected=self.artifacts.store.reference(answer),
)
for case, (data, answer) in zip(manifest.cases, payloads, strict=True)
)
admit_version(self.policy, manifest, version_id, uuid4(), cases)
async with self.sessions.begin() as session:
# Lock existing and not-yet-existing logical slugs before assigning revisions.
await repository.lock_slug(session, manifest.slug)
problem = await repository.lock_problem(session, manifest.slug)
if problem is None:
problem = Problem(id=uuid4(), slug=manifest.slug)
session.add(problem)
await session.flush()
if not problem.enabled:
raise ProblemUnavailable("problem disabled")
# Consistent digest lock order avoids deadlocks between overlapping imports/jobs.
unique = {
ref.digest: (data, ref)
for case, (input_bytes, answer_bytes) in zip(cases, payloads, strict=True)
for data, ref in [(input_bytes, case.input), (answer_bytes, case.expected)]
}
for digest in sorted(unique):
data, reference = unique[digest]
await self.artifacts.put(session, data, reference)
await publish_version(
session,
problem,
manifest,
SimpleNamespace(statement=statement, samples=samples),
version_id,
cases,
)
return version_id
@staticmethod
def _summary(problem, version) -> ProblemSummary:
return ProblemSummary(
id=problem.id,
slug=problem.slug,
version_id=version.id,
revision=version.revision,
title=version.title,
languages=tuple(version.languages),
)
async def list(self, *, after: str | None = None, limit: int = 20) -> ProblemPage:
if not 1 <= limit <= 100:
raise ValueError("invalid page limit")
async with self.sessions() as session:
rows = await repository.list_public(session, after, limit + 1)
visible = rows[:limit]
return ProblemPage(
items=tuple(self._summary(*row) for row in visible),
next_after=visible[-1][0].slug if len(rows) > limit else None,
)
async def detail(self, version_id: UUID) -> ProblemDetail:
async with self.sessions() as session:
row = await repository.public_version(session, version_id)
if row is None:
raise ProblemUnavailable("problem version unavailable")
version, problem = row
return ProblemDetail(
**self._summary(problem, version).model_dump(),
statement=version.statement,
samples=tuple(PublicSample.model_validate(s) for s in version.public_samples),
run_limits=(
RunLimits.model_validate(version.run_limits)
if version.judging_schema_version == 1
else None
),
language_configurations=tuple(
PublicLanguageConfiguration.from_configuration(
v2.LanguageConfiguration.model_validate(c)
)
for c in version.language_configurations or ()
),
)
async def for_submission(self, session, version_id: UUID) -> JudgingVersion:
if self.artifacts is None:
raise ProblemUnavailable("artifact service unavailable")
row = await repository.public_version(session, version_id, admission=True)
return await self._judging_version(session, row)
async def for_accepted_execution(self, session, version_id: UUID) -> JudgingVersion:
# Only WorkerAccess calls this after validating the current lease and its
# immutable submission/attempt binding; this is never a public UUID lookup.
row = await repository.published_version(session, version_id)
return await self._judging_version(session, row)
async def _judging_version(self, session, row):
if row is None:
raise ProblemUnavailable("problem version unavailable")
version, problem = row
cases = await repository.ordered_cases(session, version.id)
refs = []
digests = {d for case in cases for d in (case.input_digest, case.expected_digest)}
records = {
a.digest: a
for a in await session.scalars(select(Artifact).where(Artifact.digest.in_(digests)))
}
for case in cases:
input_record = records.get(case.input_digest)
answer_record = records.get(case.expected_digest)
if (
not input_record
or not answer_record
or input_record.state != "ready"
or answer_record.state != "ready"
):
raise ProblemUnavailable("problem artifacts unavailable")
refs.append(
TestCase(
case_id=case.id,
input=ArtifactReference(digest=case.input_digest, bytes=input_record.bytes),
expected=ArtifactReference(
digest=case.expected_digest, bytes=answer_record.bytes
),
)
)
return JudgingVersion(
version.id,
problem.id,
tuple(
(Language if version.judging_schema_version == 1 else v2.Language)(v)
for v in version.languages
),
Toolchain.model_validate(version.toolchain)
if version.judging_schema_version == 1
else None,
RunLimits.model_validate(version.compile_limits)
if version.judging_schema_version == 1
else None,
RunLimits.model_validate(version.run_limits)
if version.judging_schema_version == 1
else None,
version.job_wall_time_us,
tuple(refs),
version.judging_schema_version,
tuple(
v2.LanguageConfiguration.model_validate(c)
for c in version.language_configurations or ()
),
)
backend/src/kf_oj/modules/problems/web_format.py
-31
View File
@@ -1,31 +0,0 @@
"""Stricter Web package declarations; the accepted local CLI format remains unchanged."""
from dataclasses import dataclass
from kf_oj.contracts.judging import decode_wire
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport, ProblemImportV2
ARCHIVE_SUFFIXES = (".zip", ".jar", ".war", ".tar", ".tgz", ".gz", ".bz2", ".xz", ".7z", ".rar")
@dataclass(frozen=True)
class WebDeclaration:
manifest: ProblemImport | ProblemImportV2
members: frozenset[str]
def decode_web_declaration(data: bytes) -> WebDeclaration:
# decode_wire rejects duplicate JSON keys, unknown fields and noncanonical
# protocol values before either local paths or node policy can be considered.
manifest = decode_wire(ImportEnvelope, data).root
hidden = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
samples = {name for s in manifest.samples for name in (s.input_file, s.output_file)}
metadata = {"problem.json", manifest.statement_file}
if len(metadata) != 2 or hidden & (samples | metadata) or samples & metadata:
raise ValueError("public and hidden package declarations must be separate")
members = frozenset(metadata | hidden | samples)
if len({name.lower() for name in members}) != len(members):
raise ValueError("package names must be case-insensitively unique")
if any(name.lower().endswith(ARCHIVE_SUFFIXES) for name in members):
raise ValueError("nested archive declarations are unsupported")
return WebDeclaration(manifest, members)
backend/src/kf_oj/modules/submissions/README.md
+1 -45
View File
@@ -3,48 +3,4 @@
负责提交幂等请求、不可变源码引用、创建初始评测、历史查询与学生可见结果。
创建提交、评测和待投递事件必须处于同一事务。
协调器拥有领取后的执行状态与最终结果写入权；本模块不能绕过协议覆盖终态。
`queries.py` 提供已验证主体的提交历史、详情与私有源码读取，其他账号不可读取；
历史按创建时间/UUID降序游标翻页，一次至多100条。读取完整成绩时严格核对报告契约和
评测绑定，仅投影允许公开的诊断/计量/测试点序号。取消通过注入的 Coordinator 控制
端口接受意图，不更新执行状态；202与清理完成的终态不同。
`worker_access.py` 核对当前 evaluation / attempt / Worker / fencing、未撤销及未过期凭证、
固定 submission 与版本及任务清单摘要；只允许读取该清单声明的工件。
内部 API 使用 WorkerLease bearer 和 X-Worker-ID / X-Fencing-Version，不使用用户会话授权。
允许状态为 claimed / compiling / running；取消请求、终态和旧 attempt 均拒绝。
字节准备后再次核对当前租约；最多同时两个有限读取，超过直接 429，不排无界等待队列。
任务清单为 UTF-8 JSON、对象键递归排序、无额外空白，并以这些确切字节的 SHA-256 标识。
Python/Rust `manifest_bytes` 与共享字节样例保持一致；Coordinator 必须用同一序列化定义存快照。
读取凭证与客户端提交的身份分开：身份和归属从实际数据库绑定获得，不接受客户端自报 user ID。
`service.py` 按账号与幂等UUID绑定版本/语言/源码摘要，持久化源码后同一事务创建
submission、queued evaluation 和 outbox；同键同内容复用既有提交，不同内容409。
待评测固定总量64、个人8，取消清理中也占用容量；配置或容量不足明确拒绝。
Coordinator 通过正式 claim 创建短期领取权，Worker 不写业务数据库。
真实服务链与安全条件必须在独立 VM 验收；数据库 fixture 的租约或合成报告只能
验证控制规则、授权和投影，不能作为真实程序执行/停止证明。
`performance.py` 提供本人提交的只读匿名聚合。Coordinator 在受验证最终 AC 的同一事务
写入不可变 `evaluation_performance`；平台只有 SELECT 权限。比较组绑定不可变版本、
语言、精确镜像/工具链、编译与运行预算，以及运维核验后配置的测量环境纪元。
没有原纪元的旧尝试不会通过当前配置被补造可信性，仍保留个人历史与正常成绩。
单条受控回填与纪元变更规则见[协调器说明](../../../../../judge/crates/coordinator/README.md)。
历史可按 `problem_version_id` 与可选 wire `language` 过滤，始终带当前账号条件。
`GET /api/v1/submissions/{submission_id}/performance` 每次先检查会话和所有权；
匿名拒绝、其他主体与不存在记录同为不可用，私有响应为 `no-store`。
缓存命中仍查归属。每进程最多一个聚合查询，不排等待队列；SQL 期限两秒、请求总期限
三秒、锁等待至多半秒。聚合 DTO 缓存三十秒且最多128项，新 AC 的索引水位改变即失效。
缓存不保存逐条样本数组，不向浏览器发送其他账号的身份、源码或逐条用量。
每指标在同一个只读可重复读快照内，排除当前提交，取最近最多10,000条非缺测的其他
有效 AC，响应带快照、最早样本及截断标志。少于五条时仅显示本次真实用量。
CPU 为测试点运行 CPU 微秒之和，内存为测试点运行峰值字节的最大值，不包括编译。
用量与桶边界以十进制字符串传输；零合法、缺测不补零。严格优于数按原整数的 `>`
计算，同值独立计数。响应至多24个连续桶，只有最后上界包含端点；浏览器仅格式化单位
和绘制坐标，比例以服务端严格计数除以样本数，不按人数统计。
下架只阻止新提交与公开题目读取。Worker 的版本读取通过 `ProblemPort.for_accepted_execution`，
在本模块验证当前 Coordinator 租约/尝试/源码绑定之后才调用；生成并比较原 canonical
快照，再次核对租约后才释放字节。无有效租约、外来工件、未发布版本和其他提交仍拒绝。
本人历史和源码读取保留所有权检查，不依赖题目当前 enabled 状态。
后续 `public.py` 提供授权后的学习证据入口。当前未实现提交接口。
backend/src/kf_oj/modules/submissions/models.py
-233
View File
@@ -1,233 +0,0 @@
"""Platform creates initial rows; Coordinator exclusively advances execution."""
from datetime import datetime
from uuid import UUID
from sqlalchemy import (
BigInteger,
Boolean,
CheckConstraint,
Computed,
DateTime,
ForeignKey,
ForeignKeyConstraint,
Index,
String,
UniqueConstraint,
func,
text,
)
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from kf_oj.infra.db.base import Base
from kf_oj.infra.db.columns import CreatedAt
class Submission(CreatedAt, Base):
__tablename__ = "submissions"
__table_args__ = (
UniqueConstraint("account_id", "idempotency_key"),
UniqueConstraint("id", "problem_version_id"),
CheckConstraint("language IN ('c17','cpp20','python313','java21')", name="language"),
CheckConstraint("request_digest ~ '^[0-9a-f]{64}$'", name="request_digest"),
ForeignKeyConstraint(
["problem_version_id", "problem_id"],
["problem_versions.id", "problem_versions.problem_id"],
),
ForeignKeyConstraint(
["source_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
),
Index("ix_submissions_account_history", "account_id", "created_at", "id"),
Index(
"ix_submissions_owned_problem_history",
"account_id",
"problem_version_id",
text("created_at DESC"),
text("id DESC"),
),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
account_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
problem_id: Mapped[UUID]
problem_version_id: Mapped[UUID]
language: Mapped[str] = mapped_column(String(16))
source_digest: Mapped[str] = mapped_column(String(64))
artifact_state: Mapped[str] = mapped_column(String(8), Computed("'ready'", persisted=True))
idempotency_key: Mapped[UUID]
request_digest: Mapped[str] = mapped_column(String(64))
class Evaluation(CreatedAt, Base):
__tablename__ = "evaluations"
__table_args__ = (
UniqueConstraint("id", "problem_version_id"),
ForeignKeyConstraint(
["submission_id", "problem_version_id"],
["submissions.id", "submissions.problem_version_id"],
),
ForeignKeyConstraint(
["current_attempt_id", "id"],
["judge_attempts.id", "judge_attempts.evaluation_id"],
name="fk_evaluations_current_attempt",
use_alter=True,
deferrable=True,
initially="DEFERRED",
),
CheckConstraint(
"state IN ('queued','claimed','compiling','running','cancelling',"
"'finished','cancelled','failed')",
name="state",
),
CheckConstraint("fencing_version >= 0", name="fence"),
CheckConstraint("max_attempts BETWEEN 1 AND 5", name="max_attempts"),
CheckConstraint("attempts_count BETWEEN 0 AND max_attempts", name="attempts_count"),
CheckConstraint(
"state NOT IN ('claimed','compiling','running','cancelling') "
"OR current_attempt_id IS NOT NULL",
name="active_attempt",
),
Index("ix_evaluations_recovery", "state", "not_before", "id"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
submission_id: Mapped[UUID] = mapped_column(unique=True)
problem_version_id: Mapped[UUID]
state: Mapped[str] = mapped_column(String(16), server_default="queued")
fencing_version: Mapped[int] = mapped_column(BigInteger, server_default="0")
attempts_count: Mapped[int] = mapped_column(server_default="0")
max_attempts: Mapped[int] = mapped_column(server_default="3")
current_attempt_id: Mapped[UUID | None]
not_before: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
cancel_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
verdict: Mapped[str | None] = mapped_column(String(32))
failure_code: Mapped[str | None] = mapped_column(String(32))
final_report: Mapped[dict | None] = mapped_column(JSONB)
final_report_digest: Mapped[str | None] = mapped_column(String(64))
class JudgeAttempt(CreatedAt, Base):
__tablename__ = "judge_attempts"
__table_args__ = (
UniqueConstraint("id", "evaluation_id"),
UniqueConstraint("evaluation_id", "number", name="uq_judge_attempts_evaluation_number"),
UniqueConstraint(
"evaluation_id", "fencing_version", name="uq_judge_attempts_evaluation_fence"
),
CheckConstraint("number BETWEEN 1 AND 5", name="number"),
CheckConstraint("fencing_version > 0", name="fence"),
CheckConstraint("lease_token_hash ~ '^[0-9a-f]{64}$'", name="token_digest"),
CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="manifest_digest"),
CheckConstraint(
"measurement_epoch IS NULL OR measurement_epoch ~ '^[0-9a-f]{64}$'",
name="measurement_epoch",
),
CheckConstraint("sequence >= 0", name="sequence"),
Index("ix_judge_attempts_expiry", "expires_at", "evaluation_id"),
)
id: Mapped[UUID] = mapped_column(primary_key=True)
evaluation_id: Mapped[UUID] = mapped_column(ForeignKey("evaluations.id"))
worker_id: Mapped[UUID]
number: Mapped[int]
fencing_version: Mapped[int] = mapped_column(BigInteger)
lease_token_hash: Mapped[str] = mapped_column(String(64))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
manifest: Mapped[dict] = mapped_column(JSONB)
manifest_digest: Mapped[str] = mapped_column(String(64))
sequence: Mapped[int] = mapped_column(BigInteger, server_default="0")
stage: Mapped[str | None] = mapped_column(String(16))
case_index: Mapped[int | None]
report: Mapped[dict | None] = mapped_column(JSONB)
report_digest: Mapped[str | None] = mapped_column(String(64))
cleanup_complete: Mapped[bool | None] = mapped_column(Boolean)
measurement_epoch: Mapped[str | None] = mapped_column(String(64))
class EvaluationPerformance(Base):
"""Coordinator-only immutable final projection; the platform only reads it."""
__tablename__ = "evaluation_performance"
__table_args__ = (
ForeignKeyConstraint(
["evaluation_id", "problem_version_id"],
["evaluations.id", "evaluations.problem_version_id"],
),
ForeignKeyConstraint(
["submission_id", "problem_version_id"],
["submissions.id", "submissions.problem_version_id"],
),
ForeignKeyConstraint(
["attempt_id", "evaluation_id"], ["judge_attempts.id", "judge_attempts.evaluation_id"]
),
CheckConstraint("language IN ('c17','cpp20','python313','java21')", name="language"),
CheckConstraint("comparison_group ~ '^[0-9a-f]{64}$'", name="comparison_group"),
CheckConstraint("measurement_epoch ~ '^[0-9a-f]{64}$'", name="measurement_epoch"),
CheckConstraint("report_digest ~ '^[0-9a-f]{64}$'", name="report_digest"),
CheckConstraint("cpu_time_us IS NULL OR cpu_time_us >= 0", name="cpu"),
CheckConstraint("memory_peak_bytes IS NULL OR memory_peak_bytes >= 0", name="memory"),
Index(
"ix_performance_group_watermark",
"comparison_group",
text("finished_at DESC"),
text("evaluation_id DESC"),
),
Index(
"ix_performance_cpu_time_us_cohort",
"comparison_group",
text("finished_at DESC"),
text("evaluation_id DESC"),
postgresql_where=text("cpu_time_us IS NOT NULL"),
),
Index(
"ix_performance_memory_peak_bytes_cohort",
"comparison_group",
text("finished_at DESC"),
text("evaluation_id DESC"),
postgresql_where=text("memory_peak_bytes IS NOT NULL"),
),
)
evaluation_id: Mapped[UUID] = mapped_column(primary_key=True)
submission_id: Mapped[UUID] = mapped_column(unique=True)
attempt_id: Mapped[UUID] = mapped_column(unique=True)
problem_version_id: Mapped[UUID]
language: Mapped[str] = mapped_column(String(16))
comparison_group: Mapped[str] = mapped_column(String(64))
measurement_epoch: Mapped[str] = mapped_column(String(64))
report_digest: Mapped[str] = mapped_column(String(64))
finished_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
cpu_time_us: Mapped[int | None] = mapped_column(BigInteger)
memory_peak_bytes: Mapped[int | None] = mapped_column(BigInteger)
class CaseResult(Base):
__tablename__ = "case_results"
__table_args__ = (
UniqueConstraint("attempt_id", "position"),
ForeignKeyConstraint(
["case_id", "problem_version_id"], ["problem_cases.id", "problem_cases.version_id"]
),
ForeignKeyConstraint(
["evaluation_id", "problem_version_id"],
["evaluations.id", "evaluations.problem_version_id"],
),
ForeignKeyConstraint(
["attempt_id", "evaluation_id"], ["judge_attempts.id", "judge_attempts.evaluation_id"]
),
CheckConstraint("position BETWEEN 0 AND 255", name="position"),
CheckConstraint("cpu_time_us IS NULL OR cpu_time_us >= 0", name="cpu"),
CheckConstraint("memory_peak_bytes IS NULL OR memory_peak_bytes >= 0", name="memory"),
)
attempt_id: Mapped[UUID] = mapped_column(primary_key=True)
case_id: Mapped[UUID] = mapped_column(primary_key=True)
evaluation_id: Mapped[UUID]
problem_version_id: Mapped[UUID]
position: Mapped[int]
verdict: Mapped[str] = mapped_column(String(32))
cpu_time_us: Mapped[int | None] = mapped_column(BigInteger)
memory_peak_bytes: Mapped[int | None] = mapped_column(BigInteger)
backend/src/kf_oj/modules/submissions/performance.py
-217
View File
@@ -1,217 +0,0 @@
"""Read-only, bounded anonymous cohorts from Coordinator's immutable projection."""
import asyncio
import json
import time
from collections import OrderedDict
from uuid import UUID
from sqlalchemy import select, text
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import async_sessionmaker
from kf_oj.contracts.judging import I64_MAX
from kf_oj.contracts.judging_versions import decode_report
from kf_oj.contracts.submission_performance import (
MAX_PERFORMANCE_SAMPLES,
MIN_PERFORMANCE_SAMPLES,
PerformanceBucket,
PerformanceMetric,
SubmissionPerformance,
)
from kf_oj.modules.identity.public import Principal
from kf_oj.modules.submissions.models import Evaluation, EvaluationPerformance, Submission
from kf_oj.modules.submissions.public import SubmissionUnavailable
CACHE_TTL_SECONDS = 30
CACHE_CAPACITY = 128
HISTOGRAM_BUCKETS = 24
def metric(unit, current, rows=(), *, reason=None) -> PerformanceMetric:
"""Integer arithmetic throughout; conversion for display belongs to the client."""
truncated = len(rows) > MAX_PERFORMANCE_SAMPLES
rows = rows[:MAX_PERFORMANCE_SAMPLES]
count = len(rows)
base = dict(
unit=unit,
current=None if current is None else str(current),
sample_count=count,
window_truncated=truncated,
oldest_sample_at=rows[-1][1] if rows else None,
)
if reason is None:
reason = (
"measurement_missing"
if current is None
else ("insufficient_samples" if count < MIN_PERFORMANCE_SAMPLES else None)
)
if reason is not None:
return PerformanceMetric(
**base,
available=False,
reason=reason,
strictly_slower_count=None,
tied_count=None,
buckets=(),
)
values = [row[0] for row in rows]
end = max(1, current, max(values))
step = (end + HISTOGRAM_BUCKETS - 1) // HISTOGRAM_BUCKETS
# Last upper bound includes the maximum, including I64_MAX. No overflow/padding.
boundaries = list(range(0, end, step)) + [end]
counts = [0] * (len(boundaries) - 1)
for value in values:
counts[min(value // step, len(counts) - 1)] += 1
return PerformanceMetric(
**base,
available=True,
reason=None,
strictly_slower_count=sum(value > current for value in values),
tied_count=sum(value == current for value in values),
buckets=tuple(
PerformanceBucket(
start=str(start),
end=str(stop),
upper_inclusive=index == len(counts) - 1,
count=counts[index],
)
for index, (start, stop) in enumerate(zip(boundaries, boundaries[1:], strict=False))
),
)
class SubmissionPerformanceQueries:
def __init__(self, sessions: async_sessionmaker):
self.sessions = sessions
# No queued aggregate workload. At most one bounded in-memory row cohort per process.
self._slot = asyncio.Lock()
# Cache contains small immutable aggregate DTOs, never 10,000-row arrays.
self._cache: OrderedDict[tuple, tuple[float, SubmissionPerformance]] = OrderedDict()
async def detail(self, principal: Principal, submission_id: UUID) -> SubmissionPerformance:
if self._slot.locked():
raise ValueError("performance query capacity unavailable")
async with self._slot:
try:
async with asyncio.timeout(3), self.sessions() as session, session.begin():
await session.execute(
text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
)
await session.execute(text("SET LOCAL statement_timeout='2s'"))
await session.execute(text("SET LOCAL lock_timeout='500ms'"))
# Ownership checked even on a cache hit. Session authentication is upstream.
row = (
await session.execute(
select(Submission, Evaluation, EvaluationPerformance)
.join(Evaluation, Evaluation.submission_id == Submission.id)
.outerjoin(
EvaluationPerformance,
EvaluationPerformance.evaluation_id == Evaluation.id,
)
.where(
Submission.id == submission_id,
Submission.account_id == principal.account_id,
)
)
).one_or_none()
if row is None:
raise SubmissionUnavailable("submission unavailable")
submission, evaluation, projection = row
as_of = await session.scalar(text("SELECT clock_timestamp()"))
accepted = (
evaluation.state == "finished"
and evaluation.verdict == "accepted"
and evaluation.failure_code is None
)
if projection is not None and (
not accepted
or projection.attempt_id != evaluation.current_attempt_id
or projection.report_digest != evaluation.final_report_digest
):
raise ValueError("stored performance binding invalid")
cpu = memory = None
reason = None if accepted else "not_accepted"
if projection is not None:
cpu, memory = projection.cpu_time_us, projection.memory_peak_bytes
elif accepted:
# A single owned legacy report, never a full-table JSON scan or promotion.
report = decode_report(
json.dumps(evaluation.final_report, ensure_ascii=False).encode()
)
if (
report.binding.evaluation_id != evaluation.id
or report.binding.attempt_id != evaluation.current_attempt_id
or report.verdict.value != "accepted"
or not report.cleanup_complete
):
raise ValueError("stored result binding invalid")
cpu, memory = report.run_cpu_time_us, report.run_memory_peak_bytes
reason = "environment_unverified"
if any(v is not None and not 0 <= v <= I64_MAX for v in (cpu, memory)):
raise ValueError("stored measurement out of range")
base = dict(
submission_id=submission.id,
problem_version_id=submission.problem_version_id,
language=submission.language,
as_of=as_of,
comparison_group=None
if projection is None
else projection.comparison_group,
)
if projection is None:
return SubmissionPerformance(
**base,
cpu=metric("cpu_microseconds", cpu, reason=reason),
memory=metric("bytes", memory, reason=reason),
)
group = projection.comparison_group
watermark = (
await session.execute(
select(
EvaluationPerformance.finished_at,
EvaluationPerformance.evaluation_id,
)
.where(EvaluationPerformance.comparison_group == group)
.order_by(
EvaluationPerformance.finished_at.desc(),
EvaluationPerformance.evaluation_id.desc(),
)
.limit(1)
)
).one()
key = (submission.id, projection.report_digest, group, tuple(watermark))
cached = self._cache.get(key)
now = time.monotonic()
if cached is not None and now - cached[0] < CACHE_TTL_SECONDS:
self._cache.move_to_end(key)
return cached[1]
metrics = []
for column, unit, current in [
(EvaluationPerformance.cpu_time_us, "cpu_microseconds", cpu),
(EvaluationPerformance.memory_peak_bytes, "bytes", memory),
]:
rows = (
await session.execute(
select(column, EvaluationPerformance.finished_at)
.where(
EvaluationPerformance.comparison_group == group,
EvaluationPerformance.evaluation_id != evaluation.id,
column.is_not(None),
)
.order_by(
EvaluationPerformance.finished_at.desc(),
EvaluationPerformance.evaluation_id.desc(),
)
.limit(MAX_PERFORMANCE_SAMPLES + 1)
)
).all()
metrics.append(metric(unit, current, rows))
result = SubmissionPerformance(**base, cpu=metrics[0], memory=metrics[1])
self._cache[key] = (now, result)
self._cache.move_to_end(key)
while len(self._cache) > CACHE_CAPACITY:
self._cache.popitem(last=False)
return result
except (SQLAlchemyError, TimeoutError) as error:
raise ValueError("performance database unavailable") from error
backend/src/kf_oj/modules/submissions/public.py
-30
View File
@@ -1,30 +0,0 @@
from typing import Protocol
from uuid import UUID
class CancelUnavailable(ValueError):
pass
class CancellationPort(Protocol):
async def cancel(self, evaluation_id: UUID) -> bool: ...
class SubmissionUnavailable(ValueError):
pass
class WorkerAccessDenied(ValueError):
"""Never reveal whether the evaluation, attempt or artifact exists."""
class WorkerReadBusy(ValueError):
pass
class IdempotencyConflict(ValueError):
pass
class SubmissionBusy(ValueError):
pass
backend/src/kf_oj/modules/submissions/queries.py
-184
View File
@@ -1,184 +0,0 @@
import base64
import json
from datetime import datetime
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.ext.asyncio import async_sessionmaker
from kf_oj.contracts.judging import MAX_SOURCE_BYTES, ArtifactReference
from kf_oj.contracts.judging_versions import decode_report
from kf_oj.contracts.submissions import (
CancellationAccepted,
PublicCaseResult,
PublicJudgement,
SubmissionDetail,
SubmissionPage,
SubmissionSummary,
)
from kf_oj.infra.artifacts.models import Artifact
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.identity.public import Principal
from kf_oj.modules.submissions.models import Evaluation, Submission
from kf_oj.modules.submissions.public import (
CancellationPort,
CancelUnavailable,
SubmissionUnavailable,
)
def cursor_encode(submission: Submission) -> str:
return (
base64.urlsafe_b64encode(
f"{submission.created_at.isoformat()}|{submission.id}".encode("ascii")
)
.decode("ascii")
.rstrip("=")
)
def cursor_decode(value: str) -> tuple[datetime, UUID]:
try:
if not 1 <= len(value) <= 128:
raise ValueError
date, identifier = (
base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True)
.decode("ascii")
.split("|")
)
timestamp = datetime.fromisoformat(date)
if timestamp.utcoffset() is None:
raise ValueError
return timestamp, UUID(identifier)
except (ValueError, UnicodeError):
raise ValueError("invalid history cursor") from None
def summary(submission: Submission, evaluation: Evaluation) -> SubmissionSummary:
terminal = evaluation.state in {"finished", "failed", "cancelled"}
return SubmissionSummary(
submission_id=submission.id,
evaluation_id=evaluation.id,
problem_version_id=submission.problem_version_id,
language=submission.language,
state=evaluation.state,
created_at=submission.created_at,
updated_at=evaluation.updated_at,
attempts=evaluation.attempts_count,
verdict=evaluation.verdict if terminal else None,
failure_code=evaluation.failure_code,
)
class SubmissionQueries:
def __init__(
self,
sessions: async_sessionmaker,
artifacts: ArtifactRepository,
cancellation: CancellationPort | None = None,
):
self.sessions, self.artifacts = sessions, artifacts
self.cancellation = cancellation
def owned(self, principal: Principal, submission_id: UUID):
return (
select(Submission, Evaluation)
.join(Evaluation, Evaluation.submission_id == Submission.id)
.where(Submission.id == submission_id, Submission.account_id == principal.account_id)
)
async def page(
self,
principal: Principal,
after: str | None,
limit: int,
problem_version_id: UUID | None = None,
language: str | None = None,
) -> SubmissionPage:
if not 1 <= limit <= 100:
raise ValueError("invalid history limit")
query = (
select(Submission, Evaluation)
.join(Evaluation, Evaluation.submission_id == Submission.id)
.where(Submission.account_id == principal.account_id)
.order_by(Submission.created_at.desc(), Submission.id.desc())
.limit(limit + 1)
)
if problem_version_id is not None:
query = query.where(Submission.problem_version_id == problem_version_id)
if language is not None:
query = query.where(Submission.language == language)
if after is not None:
query = query.where(tuple_(Submission.created_at, Submission.id) < cursor_decode(after))
async with self.sessions() as session:
rows = (await session.execute(query)).all()
return SubmissionPage(
items=tuple(summary(s, e) for s, e in rows[:limit]),
next_after=cursor_encode(rows[limit - 1][0]) if len(rows) > limit else None,
)
async def detail(self, principal: Principal, submission_id: UUID) -> SubmissionDetail:
async with self.sessions() as session:
row = (await session.execute(self.owned(principal, submission_id))).one_or_none()
if row is None:
raise SubmissionUnavailable("submission unavailable")
submission, evaluation = row
result = None
if evaluation.state in {"finished", "failed", "cancelled"} and evaluation.final_report:
# A single joined snapshot, strict validation and explicit projection prevent leaking
# internal bindings, artifacts, hidden case identifiers or partial attempt reports.
report = decode_report(
json.dumps(
evaluation.final_report, ensure_ascii=False, separators=(",", ":")
).encode()
)
if str(report.binding.evaluation_id) != str(evaluation.id):
raise ValueError("stored result binding invalid")
if report.verdict.value != evaluation.verdict:
raise ValueError("stored result verdict invalid")
result = PublicJudgement(
compilation_status=report.compilation.status,
diagnostics=report.compilation.diagnostics,
diagnostics_truncated=report.compilation.diagnostics_truncated,
cpu_time_us=report.run_cpu_time_us,
memory_peak_bytes=report.run_memory_peak_bytes,
cleanup_complete=report.cleanup_complete,
cases=tuple(
PublicCaseResult(position=i, **c.model_dump(exclude={"case_id"}))
for i, c in enumerate(report.cases)
),
)
return SubmissionDetail(**summary(submission, evaluation).model_dump(), result=result)
async def cancel(self, principal: Principal, submission_id: UUID) -> CancellationAccepted:
async with self.sessions() as session:
row = (await session.execute(self.owned(principal, submission_id))).one_or_none()
if row is None:
raise SubmissionUnavailable("submission unavailable")
submission, evaluation = row
if self.cancellation is None or not await self.cancellation.cancel(evaluation.id):
raise CancelUnavailable("cancellation service unavailable")
return CancellationAccepted(submission_id=submission.id, evaluation_id=evaluation.id)
async def owned_source(self, principal: Principal, submission_id: UUID) -> bytes:
async with self.sessions() as session:
submission = await session.scalar(
select(Submission).where(
Submission.id == submission_id, Submission.account_id == principal.account_id
)
)
if submission is None:
raise SubmissionUnavailable("submission unavailable")
metadata = await session.get(Artifact, submission.source_digest)
if (
metadata is None
or metadata.state != "ready"
or not 1 <= metadata.bytes <= MAX_SOURCE_BYTES
):
raise SubmissionUnavailable("submission source unavailable")
reference = ArtifactReference(digest=metadata.digest, bytes=metadata.bytes)
try:
return await self.artifacts.read(session, reference)
except ArtifactError:
raise SubmissionUnavailable("submission source unavailable") from None
backend/src/kf_oj/modules/submissions/repository.py
-27
View File
@@ -1,27 +0,0 @@
from sqlalchemy import func, select
from kf_oj.contracts.judging import LeaseBinding
from kf_oj.modules.submissions.models import Evaluation, JudgeAttempt, Submission
async def leased_submission(session, binding: LeaseBinding, token_hash: str):
return (
await session.execute(
select(Evaluation, JudgeAttempt, Submission)
.join(JudgeAttempt, JudgeAttempt.id == Evaluation.current_attempt_id)
.join(Submission, Submission.id == Evaluation.submission_id)
.where(
Evaluation.id == binding.evaluation_id,
Evaluation.state.in_(("claimed", "compiling", "running")),
Evaluation.cancel_requested_at.is_(None),
Evaluation.fencing_version == binding.fencing_version,
JudgeAttempt.id == binding.attempt_id,
JudgeAttempt.evaluation_id == Evaluation.id,
JudgeAttempt.worker_id == binding.worker_id,
JudgeAttempt.fencing_version == binding.fencing_version,
JudgeAttempt.lease_token_hash == token_hash,
JudgeAttempt.expires_at > func.clock_timestamp(),
JudgeAttempt.revoked_at.is_(None),
)
)
).one_or_none()
backend/src/kf_oj/modules/submissions/service.py
-123
View File
@@ -1,123 +0,0 @@
"""Only initial queued evaluation/outbox writes; execution belongs to Coordinator."""
import hashlib
import json
from uuid import uuid4
from sqlalchemy import func, select, text
from kf_oj.contracts.judging_v2 import admit_manifest
from kf_oj.contracts.submissions import CreateSubmission, SubmissionCreated
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.infra.messaging.models import Outbox
from kf_oj.modules.identity.public import Principal
from kf_oj.modules.problems.public import ProblemPort, ProblemUnavailable
from kf_oj.modules.submissions.models import Evaluation, Submission
from kf_oj.modules.submissions.public import (
IdempotencyConflict,
SubmissionBusy,
SubmissionUnavailable,
)
class SubmissionService:
def __init__(self, sessions, problems: ProblemPort, artifacts: ArtifactRepository, admission):
self.sessions, self.problems = sessions, problems
self.artifacts, self.admission = artifacts, admission
@staticmethod
def _response(submission, evaluation):
return SubmissionCreated(
submission_id=submission.id,
evaluation_id=evaluation.id,
problem_version_id=submission.problem_version_id,
language=submission.language,
state=evaluation.state,
created_at=submission.created_at,
)
async def create(self, principal: Principal, request: CreateSubmission) -> SubmissionCreated:
# Validate even trusted non-HTTP callers. No path or argv is accepted.
request = CreateSubmission.model_validate(
request.model_dump(exclude={"language"}) | {"language": request.language}
)
source = request.source.encode("utf-8")
request_digest = hashlib.sha256(
json.dumps(
[
str(request.problem_version_id),
request.language.value,
hashlib.sha256(source).hexdigest(),
],
separators=(",", ":"),
).encode()
).hexdigest()
if self.admission is None:
raise SubmissionUnavailable("judge admission unavailable")
async with self.sessions.begin() as session:
# Bounded pending capacity and per-account idempotency serialize across API processes.
# Two-key namespace is separate from artifact/slug advisory locks.
await session.execute(text("SELECT pg_advisory_xact_lock(1262907470, 0)"))
existing = (
await session.execute(
select(Submission, Evaluation)
.join(Evaluation, Evaluation.submission_id == Submission.id)
.where(
Submission.account_id == principal.account_id,
Submission.idempotency_key == request.idempotency_key,
)
)
).one_or_none()
if existing is not None:
if existing[0].request_digest != request_digest:
raise IdempotencyConflict("idempotency key belongs to another request")
return self._response(*existing)
pending = ("queued", "claimed", "compiling", "running", "cancelling")
total = await session.scalar(
select(func.count()).select_from(Evaluation).where(Evaluation.state.in_(pending))
)
owned = await session.scalar(
select(func.count())
.select_from(Submission)
.join(Evaluation, Evaluation.submission_id == Submission.id)
.where(Submission.account_id == principal.account_id, Evaluation.state.in_(pending))
)
if total >= 64 or owned >= 8:
raise SubmissionBusy("pending submission capacity exhausted")
try:
version = await self.problems.for_submission(session, request.problem_version_id)
except ProblemUnavailable:
raise SubmissionUnavailable("problem version unavailable") from None
if request.language not in version.languages:
raise SubmissionUnavailable("language unavailable")
# Bytes fsync before metadata/ref transaction. Rollback may leave collectable bytes,
# but cannot leave an evaluation that refers to non-durable bytes.
reference = await self.artifacts.put(session, source)
submission_id, evaluation_id = uuid4(), uuid4()
manifest = version.manifest(evaluation_id, submission_id, request.language, reference)
admit_manifest(self.admission, manifest, 1)
submission = Submission(
id=submission_id,
account_id=principal.account_id,
problem_id=version.problem_id,
problem_version_id=version.version_id,
language=request.language.value,
source_digest=reference.digest,
idempotency_key=request.idempotency_key,
request_digest=request_digest,
)
session.add(submission)
await session.flush()
await session.execute(
Evaluation.__table__.insert().values(
id=evaluation_id,
submission_id=submission_id,
problem_version_id=version.version_id,
)
)
session.add(Outbox(id=uuid4(), evaluation_id=evaluation_id))
await session.flush()
evaluation = await session.get(Evaluation, evaluation_id)
response = self._response(submission, evaluation)
# The transaction commit must succeed before any success is returned.
return response
backend/src/kf_oj/modules/submissions/worker_access.py
-110
View File
@@ -1,110 +0,0 @@
import asyncio
import hashlib
import hmac
import json
import re
from contextlib import asynccontextmanager
from sqlalchemy.ext.asyncio import async_sessionmaker
from kf_oj.contracts.judging import LeaseBinding
from kf_oj.contracts.judging_versions import (
Manifest,
decode_manifest,
manifest_bytes,
)
from kf_oj.infra.artifacts.repository import ArtifactRepository
from kf_oj.infra.artifacts.store import ArtifactError
from kf_oj.modules.problems.public import ProblemPort, ProblemUnavailable
from kf_oj.modules.submissions import repository
from kf_oj.modules.submissions.public import WorkerAccessDenied, WorkerReadBusy
class WorkerAccess:
def __init__(
self, sessions: async_sessionmaker, problems: ProblemPort, artifacts: ArtifactRepository
):
self.sessions, self.problems, self.artifacts = sessions, problems, artifacts
self._reads = asyncio.Semaphore(2)
@asynccontextmanager
async def _read_slot(self):
if self._reads.locked():
raise WorkerReadBusy("worker read capacity exhausted")
async with self._reads:
yield
@staticmethod
def _token_hash(token: str) -> str:
if not re.fullmatch(r"[0-9a-f]{64}", token):
raise WorkerAccessDenied("worker grant rejected")
return hashlib.sha256(token.encode("ascii")).hexdigest()
async def _manifest(self, session, binding: LeaseBinding, token: str) -> Manifest:
row = await repository.leased_submission(session, binding, self._token_hash(token))
if row is None:
raise WorkerAccessDenied("worker grant rejected")
evaluation, attempt, submission = row
# Use a sanitized decoder at this trust boundary. JSONB snapshots are immutable for
# runtime roles; use canonical bytes for both retrieval and digest verification.
try:
manifest = decode_manifest(json.dumps(attempt.manifest).encode())
if (
manifest.evaluation_id != evaluation.id
or manifest.submission_id != submission.id
or manifest.problem_version_id != evaluation.problem_version_id
or manifest.problem_version_id != submission.problem_version_id
or manifest.language.value != submission.language
or manifest.source.digest != submission.source_digest
or not hmac.compare_digest(
hashlib.sha256(manifest_bytes(manifest)).hexdigest(), attempt.manifest_digest
)
):
raise WorkerAccessDenied("worker snapshot rejected")
version = await self.problems.for_accepted_execution(
session, manifest.problem_version_id
)
await self.artifacts.check_metadata(session, manifest.source)
expected = version.manifest(
manifest.evaluation_id, manifest.submission_id, manifest.language, manifest.source
)
if version.problem_id != submission.problem_id or manifest_bytes(
manifest
) != manifest_bytes(expected):
raise WorkerAccessDenied("worker snapshot rejected")
return manifest
except (ValueError, ProblemUnavailable):
raise WorkerAccessDenied("worker snapshot rejected") from None
async def manifest(self, binding: LeaseBinding, token: str) -> bytes:
async with self._read_slot(), self.sessions() as session:
manifest = await self._manifest(session, binding, token)
payload = manifest_bytes(manifest)
# READ COMMITTED: recheck after all preparation and just before releasing bytes.
if (
await repository.leased_submission(session, binding, self._token_hash(token))
is None
):
raise WorkerAccessDenied("worker grant rejected")
return payload
async def artifact(self, binding: LeaseBinding, token: str, digest: str) -> bytes:
async with self._read_slot(), self.sessions() as session:
manifest = await self._manifest(session, binding, token)
references = (manifest.source,) + tuple(
reference for case in manifest.cases for reference in (case.input, case.expected)
)
reference = next((ref for ref in references if ref.digest == digest), None)
if reference is None:
raise WorkerAccessDenied("artifact grant rejected")
try:
payload = await self.artifacts.read(session, reference)
except ArtifactError:
raise WorkerAccessDenied("artifact unavailable") from None
if (
await repository.leased_submission(session, binding, self._token_hash(token))
is None
):
raise WorkerAccessDenied("worker grant rejected")
return payload

Some files were not shown because too many files have changed in this diff Show More