Compare commits
146
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f5e9a10363 | ||
|
|
8bde33804d | ||
|
|
c820d4b637 | ||
|
|
cde9fab2d2 | ||
|
|
b3b0a5262a | ||
|
|
4fc06ee233 | ||
|
|
6e20b53013 | ||
|
|
4e5f491952 | ||
|
|
a436c659e9 | ||
|
|
60a0dad94f | ||
|
|
70007a36b9 | ||
|
|
81b5258808 | ||
|
|
8d7f651ddc | ||
|
|
9f4a8c9a68 | ||
|
|
db834dfc73 | ||
|
|
39a7b2902a | ||
|
|
08bac1ad84 | ||
|
|
1b712aae2c | ||
|
|
45360f72dd | ||
|
|
dbbb495b42 | ||
|
|
0575bb4286 | ||
|
|
944ac0a03b | ||
|
|
c54d989fc7 | ||
|
|
278337dd46 | ||
|
|
0cd94a20ec | ||
|
|
cc8f4217db | ||
|
|
7650a7e7b6 | ||
|
|
d66d42d021 | ||
|
|
cf43ae6c75 | ||
|
|
b7c586cebb | ||
|
|
8e8015d3f8 | ||
|
|
14d931a8d7 | ||
|
|
9ea709af92 | ||
|
|
ebe71db564 | ||
|
|
0a27704fd4 | ||
|
|
ee9ebd6d24 | ||
|
|
02c1dadf67 | ||
|
|
95a6b0dc1e | ||
|
|
ecce953fbb | ||
|
|
6071994695 | ||
|
|
3b8d9a2e71 | ||
|
|
c60ccf6858 | ||
|
|
d30043b99b | ||
|
|
75f781ee87 | ||
|
|
b181ca6bd6 | ||
|
|
09266df638 | ||
|
|
2cae75920d | ||
|
|
e159576340 | ||
|
|
721c293382 | ||
|
|
04e5fac311 | ||
|
|
1bd1746ef2 | ||
|
|
72828c06e6 | ||
|
|
b1e2c5a948 | ||
|
|
cd9f2e17f7 | ||
|
|
41d0e3d195 | ||
|
|
2023ca5de9 | ||
|
|
d3ef4f03aa | ||
|
|
f269a39bbe | ||
|
|
c7b260adc2 | ||
|
|
e742c3543d | ||
|
|
16bcce3f04 | ||
|
|
83d3bf8309 | ||
|
|
ef0ea71d0a | ||
|
|
f39a051d9f | ||
|
|
5ff88047ff | ||
|
|
be22dd9c12 | ||
|
|
11a2b6643e | ||
|
|
d5cb46278d | ||
|
|
81c674a17e | ||
|
|
dbacbddf64 | ||
|
|
11bfb3ff6f | ||
|
|
a9b59e127f | ||
|
|
e95a25892f | ||
|
|
52adb7eab9 | ||
|
|
f5f99c9cae | ||
|
|
70f4c78905 | ||
|
|
2e201ed7fb | ||
|
|
13c548454b | ||
|
|
b575462c95 | ||
|
|
110a4952a8 | ||
|
|
4f2513c93f | ||
|
|
9b3e49c909 | ||
|
|
8e6133a8d0 | ||
|
|
83e235567f | ||
|
|
57f5819dfd | ||
|
|
bb14f2616b | ||
|
|
ad7eebee54 | ||
|
|
325ed08ed2 | ||
|
|
71918399c9 | ||
|
|
e94b6fc8ef | ||
|
|
fd1ad64692 | ||
|
|
480db0e74f | ||
|
|
40b9a69e5a | ||
|
|
3a9d5e47d4 | ||
|
|
033d31d420 | ||
|
|
a55de23fd6 | ||
|
|
743d6ef429 | ||
|
|
5c7146d245 | ||
|
|
09af862957 | ||
|
|
5061084d4c | ||
|
|
fdc4446206 | ||
|
|
68b5f36c9b | ||
|
|
474406702f | ||
|
|
7145fb3fa4 | ||
|
|
3cde198358 | ||
|
|
5d127ad4db | ||
|
|
53f9e55fd8 | ||
|
|
eef732e2ad | ||
|
|
e34ad78329 | ||
|
|
4b02cb2c84 | ||
|
|
41c3334568 | ||
|
|
6733d4e435 | ||
|
|
f576756359 | ||
|
|
445affb63d | ||
|
|
3536ef1fbb | ||
|
|
676781223d | ||
|
|
2f860a17c9 | ||
|
|
835df6b7e7 | ||
|
|
5366418881 | ||
|
|
3152d59056 | ||
|
|
234ab23155 | ||
|
|
a7b2324a3e | ||
|
|
3ed5b72459 | ||
|
|
8143a7f665 | ||
|
|
08f2db616f | ||
|
|
c8cac766c7 | ||
|
|
bc5cd12056 | ||
|
|
218fa38f8e | ||
|
|
fce7522267 | ||
|
|
3cc90e4573 | ||
|
|
5df3c370aa | ||
|
|
029585739c | ||
|
|
47f535ea79 | ||
|
|
67c3bc55ba | ||
|
|
d20721313e | ||
|
|
9d960c2a9b | ||
|
|
507b15b64a | ||
|
|
643a8d342e | ||
|
|
802e019699 | ||
|
|
2f1166954a | ||
|
|
38ee332a3e | ||
|
|
9394a914f8 | ||
|
|
1dd6d1f971 | ||
|
|
53caafc157 | ||
|
|
4b8f255180 | ||
|
|
a5a7e40308 |
@@ -0,0 +1,205 @@
|
||||
name: Trusted engineering checks
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [feat/single-node-delivery]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: kf-onlinejudge-trusted-engineering
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
check:
|
||||
if: github.server_url == 'https://github.com' && github.repository == 'KiriAky107/KF-OnlineJudge'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
defaults:
|
||||
run:
|
||||
shell: bash --noprofile --norc -euo pipefail {0}
|
||||
env:
|
||||
CI: "true"
|
||||
RUSTUP_TOOLCHAIN: 1.99.0
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
CARGO_NET_RETRY: "2"
|
||||
UV_PYTHON: 3.13.16
|
||||
UV_NO_PROGRESS: "1"
|
||||
UV_HTTP_TIMEOUT: "30"
|
||||
UV_HTTP_RETRIES: "2"
|
||||
steps:
|
||||
- name: Check out the pushed commit
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
|
||||
timeout-minutes: 2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 1
|
||||
fetch-tags: false
|
||||
submodules: false
|
||||
lfs: false
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
|
||||
timeout-minutes: 3
|
||||
with:
|
||||
node-version: 26.11.1
|
||||
check-latest: false
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
|
||||
timeout-minutes: 3
|
||||
with:
|
||||
version: 0.12.24
|
||||
python-version: 3.13.16
|
||||
enable-cache: false
|
||||
|
||||
- name: Install locked toolchains and dependencies
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
test "$GITHUB_SERVER_URL" = https://github.com
|
||||
test "$GITHUB_REPOSITORY" = KiriAky107/KF-OnlineJudge
|
||||
test "$GITHUB_EVENT_NAME" = push
|
||||
test "$GITHUB_REF" = refs/heads/feat/single-node-delivery
|
||||
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
|
||||
uv python install 3.13.16
|
||||
rustup toolchain install 1.99.0 --profile minimal --component rustfmt --component clippy
|
||||
npm install --global --ignore-scripts pnpm@11.28.4
|
||||
make install
|
||||
|
||||
- name: Check toolchain versions and CI inputs
|
||||
timeout-minutes: 1
|
||||
run: |
|
||||
test "$(node --version)" = v26.11.1
|
||||
test "$(pnpm --version)" = 11.28.4
|
||||
test "$(rustc --version | cut -d ' ' -f 2)" = 1.99.0
|
||||
test "$(uv --version | cut -d ' ' -f 2)" = 0.12.24
|
||||
backend/.venv/bin/python - <<'PYCODE'
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
assert sys.version_info[:3] == (3, 13, 16)
|
||||
assert not any(name.startswith(('KF_', 'PGPASSWORD', 'PGSERVICE', 'DATABASE_URL', 'AMQP_URL'))
|
||||
for name in os.environ), 'CI must not carry node credentials or VM flags'
|
||||
assert not subprocess.check_output(['git', 'ls-files', '--', 'docs/', '.localplan/'])
|
||||
PYCODE
|
||||
|
||||
- name: Check architecture and generated contracts
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
uv run --project backend --locked python scripts/check_architecture.py
|
||||
make contracts
|
||||
git diff --exit-code -- contracts frontend/src/api/generated/schema.d.ts
|
||||
|
||||
- name: Check Backend and run ordinary tests
|
||||
timeout-minutes: 5
|
||||
working-directory: backend
|
||||
run: |
|
||||
uv run --locked ruff check . ../scripts
|
||||
uv run --locked ruff format --check . ../scripts
|
||||
uv run --locked pytest \
|
||||
tests/test_contracts.py \
|
||||
tests/test_judging_contracts.py \
|
||||
tests/test_judging_v2_contracts.py \
|
||||
tests/test_submission_performance_contract.py \
|
||||
tests/test_admin_import_contracts.py \
|
||||
tests/test_availability.py \
|
||||
tests/test_dependency_errors.py \
|
||||
tests/test_production_logging.py \
|
||||
tests/test_delivery.py \
|
||||
tests/test_delivery_io.py \
|
||||
tests/test_delivery_proxy_startup.py \
|
||||
tests/test_delivery_validation_work.py \
|
||||
tests/test_delivery_bytecode.py \
|
||||
tests/test_delivery_backup.py \
|
||||
tests/test_delivery_restore_io.py \
|
||||
tests/test_restore_io_contracts.py \
|
||||
tests/test_restore_native_transport.py \
|
||||
tests/test_operator_diagnostics.py \
|
||||
tests/test_outbox_startup_diagnostics.py \
|
||||
tests/test_delivery_role_projection.py \
|
||||
tests/test_operator_imports.py \
|
||||
tests/test_health.py \
|
||||
tests/test_identity.py \
|
||||
tests/test_coordinator_control.py \
|
||||
--junitxml="$RUNNER_TEMP/kf-engineering-backend.xml"
|
||||
uv run --locked python -m kf_oj.contracts.export --check
|
||||
|
||||
- name: Check Rust and run library and contract tests
|
||||
timeout-minutes: 8
|
||||
working-directory: judge
|
||||
run: |
|
||||
cargo fmt --all -- --check
|
||||
cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||
cargo test --workspace --lib --locked
|
||||
cargo test --package kf-contracts --locked
|
||||
cargo test --package kf-sandbox --bin kf-restore-io --locked
|
||||
|
||||
- name: Check and build Frontend
|
||||
timeout-minutes: 5
|
||||
run: make check-frontend
|
||||
|
||||
- name: Parse development Compose configuration
|
||||
timeout-minutes: 1
|
||||
run: make infra-config
|
||||
|
||||
- name: Collect public contracts and engineering report
|
||||
timeout-minutes: 1
|
||||
run: |
|
||||
backend/.venv/bin/python - <<'PYCODE'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
source = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()
|
||||
assert source == os.environ['GITHUB_SHA']
|
||||
destination = Path(os.environ['RUNNER_TEMP']) / 'kf-engineering-artifacts'
|
||||
destination.mkdir(mode=0o700)
|
||||
names = subprocess.check_output(['git', 'ls-files', '-z', '--',
|
||||
'contracts/openapi/platform.v1.json',
|
||||
'contracts/schemas/*.json']).decode().split('\0')[:-1]
|
||||
assert 1 <= len(names) <= 64
|
||||
files, total = [], 0
|
||||
for name in names:
|
||||
path = Path(name)
|
||||
assert not any(part.is_symlink() for part in (path, *path.parents))
|
||||
assert path.is_file() and path.stat().st_size <= 2 * 1024**2
|
||||
data = path.read_bytes()
|
||||
json.loads(data)
|
||||
total += len(data)
|
||||
assert total <= 8 * 1024**2
|
||||
output = destination / name
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_bytes(data)
|
||||
files.append({'path': name, 'bytes': len(data),
|
||||
'sha256': hashlib.sha256(data).hexdigest()})
|
||||
junit = ET.parse(Path(os.environ['RUNNER_TEMP']) / 'kf-engineering-backend.xml')
|
||||
suites = list(junit.getroot().iter('testsuite'))
|
||||
backend = {name: sum(int(suite.attrib.get(name, 0)) for suite in suites)
|
||||
for name in ('tests', 'failures', 'errors', 'skipped')}
|
||||
assert backend['failures'] == backend['errors'] == 0
|
||||
report = {'schema_version': 1, 'source_sha': source,
|
||||
'scope': 'Trusted engineering checks only', 'backend': backend,
|
||||
'toolchains': {'python': '3.13.16', 'rust': '1.99.0',
|
||||
'node': '26.11.1', 'pnpm': '11.28.4', 'uv': '0.12.24'},
|
||||
'files': files,
|
||||
'not_run': ['real PostgreSQL acceptance', 'independent VM cases',
|
||||
'user source execution', 'adverse archives and filesystem acceptance',
|
||||
'Linux isolation acceptance', 'deployment']}
|
||||
(destination / 'engineering-report.json').write_text(json.dumps(report, indent=2)+'\n')
|
||||
PYCODE
|
||||
|
||||
- name: Upload public contracts and engineering report
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
timeout-minutes: 2
|
||||
with:
|
||||
name: engineering-checks-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/kf-engineering-artifacts
|
||||
if-no-files-found: error
|
||||
include-hidden-files: false
|
||||
overwrite: false
|
||||
retention-days: 1
|
||||
compression-level: 6
|
||||
@@ -16,6 +16,7 @@
|
||||
- 沙箱未验证隔离前拒绝执行;不得用直接运行宿主命令作为临时判题实现。
|
||||
- 只建立当期需要的接口和实现;未实现能力明确报告,不提供模拟成绩或虚假任务成功状态。
|
||||
- 使用项目锁文件;新增依赖说明职责。凭据、私有代码和隐藏测试数据不进入仓库。
|
||||
- 测试备份不得积累:测试结束并保存必要验收证据后,及时删除不再使用的备份、虚拟机磁盘/快照及临时工件;当前任务仍依赖的资源须列明用途和清理条件,被替代后立即清理,不能以保留日志为由长期保留大体积副本。
|
||||
|
||||
按变更运行 `make check-backend`、`make check-judge`、`make check-frontend` 和 `make architecture`。
|
||||
跨工程骨架或契约改动运行完整 `make check`,记录实际通过项和未验证的能力。
|
||||
|
||||
@@ -5,14 +5,24 @@
|
||||
|
||||
当前版本为 **0.0.1**:沙箱已有受控隔离、C/C++ 编译及 native 执行;平台已实现真实身份、
|
||||
不可变题目、工件与幂等提交创建。事务 outbox、持久化 Coordinator、普通 Worker 和受控
|
||||
helper 工厂已接通 C/C++ 编译一次、多测试点评测;登录、题库、提交和个人成绩页面
|
||||
helper 工厂已接通 C17/C++20、固定 CPython 3.13 与 OpenJDK 21 的普通题评测;登录、题库、提交和个人成绩页面
|
||||
使用真实接口。训练、比赛、社区等扩展页面仍为开发预览。
|
||||
|
||||
管理控制台使用同源 Cookie 和服务端维护权限,支持有界 ZIP 题包上传、候选预览、
|
||||
不可变版本发布、题目下架/恢复及维护审计。初次授权与恢复走独立运维身份的本地 CLI;
|
||||
导入 Worker 仅解析数据,不执行题包内容或学生代码。操作、限额和恢复步骤见
|
||||
[题目维护运维说明](deploy/admin-problems.md)。
|
||||
|
||||
题目工作台支持当前版本的“本题我的提交”、右侧详情与完整个人详情。
|
||||
性能展示读取受验证最终 AC 的 CPU 总和与内存峰值,只比较同版本、语言、精确工具链、
|
||||
预算与可信测量环境的最近有效记录。缺测、旧环境无法验证或其他记录不足五条时明确显示
|
||||
不可比较;不会填入示例成绩。口径、查询边界和权限见[提交模块说明](backend/src/kf_oj/modules/submissions/README.md)。
|
||||
|
||||
## 工程划分
|
||||
|
||||
| 目录 | 职责 |
|
||||
| --- | --- |
|
||||
| `backend/` | Python 模块化单体、平台 API、学习 Worker、待投递事件发布器及唯一数据库迁移入口 |
|
||||
| `backend/` | Python 模块化单体、平台 API、题包导入 Worker、待投递事件发布器及唯一数据库迁移入口 |
|
||||
| `judge/` | Rust workspace:协议、沙箱、判题编排、判题 Worker 和协调器 |
|
||||
| `frontend/` | 页面、路由、请求生命周期与生成的 API 类型 |
|
||||
| `contracts/` | 版本化 JSON Schema 与 OpenAPI 快照 |
|
||||
@@ -27,12 +37,14 @@ Python 业务域分为 `identity`、`problems`、`submissions`、`community`、`
|
||||
|
||||
Rust 分为 `kf-contracts`、`kf-sandbox`、`kf-engine`、`kf-worker`、`kf-coordinator` 五个 crate。
|
||||
协调器拥有执行状态写入权,判题 Worker 不直接写评测数据库。
|
||||
Sandbox 内的 `kf-restore-io` 仅在固定离线恢复单元中复制和核验固定数据域,
|
||||
通过共享有界协议接收文件记录;不启动程序、服务或写业务状态。
|
||||
沙箱执行入口每次核验隔离与限额,任何未验证配置继续拒绝执行。
|
||||
用户程序及隔离/超限验收只在独立、整体限额的无业务凭据测试 VM 内运行。
|
||||
|
||||
## 安装与检查
|
||||
|
||||
工具链基线:Python 3.13.16 + uv、Rust 1.99.0、Node 26.10.0 + pnpm 11.28.4。
|
||||
工具链基线:Python 3.13.16 + uv、Rust 1.99.0、Node 26.11.1 + pnpm 11.28.4。
|
||||
Python 和 Rust 版本由工程配置固定,三套工程分别提交依赖锁文件。
|
||||
开发与构建使用 Linux 原生文件系统。
|
||||
|
||||
@@ -71,8 +83,12 @@ make sandbox-probe # 只读观察 Linux 环境,不创建隔离资源
|
||||
开发中的规划文档仅在本地维护,不包含在远端仓库中。
|
||||
单机提交服务的配置、最小权限、停止恢复与独立 VM 验收要求见
|
||||
[判题说明](judge/README.md#单机持久化协调与常驻-worker)和[部署说明](deploy/README.md#单机提交服务的启动与停止)。
|
||||
当前范围为单机普通题、已有账号、固定 GNU C17/C++20 与精确字节比较;自助注册、
|
||||
自测、SPJ/交互题、Python/Rust 用户程序、分布式调度和学习能力仍待后续实现。
|
||||
当前普通题执行支持固定 GNU C17/C++20、CPython 3.13.16 和 OpenJDK 21.0.12.1+1-LTS,
|
||||
使用精确字节比较。Python 固定单文件 main.py 与标准库;Java 固定默认包 Main.java/Main,
|
||||
允许有界辅助/内部/匿名类。题目必须显式声明对应语言能力,旧题目不会自动获得新语言。
|
||||
新版源码与普通 Worker 的沙箱执行已在独立 VM 验收;完整平台/浏览器及故障验收按工程阶段
|
||||
分别记录,生产部署需另行进行。自助注册、自测、SPJ/交互题、Rust 用户程序、分布式调度
|
||||
和学习能力仍待后续实现。
|
||||
|
||||
## 许可证
|
||||
|
||||
|
||||
@@ -9,6 +9,22 @@ upgrade head`,API 不在启动时创建表。迁移账号拥有该数据库对
|
||||
协调器组领取和更新评测、插入 attempt / 用例成绩及恢复通知,不能修改账号、题库或提交身份。
|
||||
Worker 没有数据库角色,也不需要数据库连接配置。
|
||||
|
||||
后续迁移另建 `kf_oj_identity_operator` 与 `kf_oj_problem_import` 两个受限 `NOLOGIN`
|
||||
权限组。前者只通过固定账号授权函数维护管理员能力/启用状态,不能读取口令哈希、
|
||||
直接修改角色字段或绕过最后一名启用管理员保护;后者仅处理已授权导入、候选/工件与
|
||||
追加审计,不能写评测状态、attempt、终态或表现指标。判题 Worker 继续无数据库身份。
|
||||
平台写入权按列收敛;当前会话与授权版本在每次维护事务内重新核对。
|
||||
|
||||
可信表现指标由 Coordinator 在有效最终报告事务中派生,平台、导入器和普通 Worker
|
||||
不得自行写入。候选与发布版本/用例不可变,成功维护回执及审计只追加;清理保留所有
|
||||
已发布版本、合法旧提交和未结束候选的工件引用。
|
||||
|
||||
升级前停止新维护写入、备份专用数据库与私有工件,并用独立迁移账号执行升级和
|
||||
`uv run --locked alembic check`。既有登录角色继续绑定各自最小权限组,迁移账号不进入
|
||||
运行服务配置。降级不作为丢弃候选或回滚一次发布的手段;含新语言/已引用数据时的
|
||||
降级拒绝须先核对原因,不能清空历史来通过检查。账号 bootstrap/恢复步骤见
|
||||
[维护运维说明](../../deploy/admin-problems.md)。
|
||||
|
||||
服务使用各自独立的普通 `LOGIN` 角色,分别加入对应权限组;登录角色不能拥有数据库/表、
|
||||
不能加入迁移账号或其他高权限组,且必须为 `NOSUPERUSER NOCREATEDB NOCREATEROLE NOBYPASSRLS`。
|
||||
初始迁移拒绝已有的高权限同名组和继承其他角色的同名组。应用启动时不运行 migration。
|
||||
|
||||
@@ -10,6 +10,8 @@ from kf_oj.infra.db.base import Base
|
||||
from kf_oj.infra.db.session import build_engine
|
||||
from kf_oj.infra.messaging import models as messaging_models # noqa: F401
|
||||
from kf_oj.modules.identity import models as identity_models # noqa: F401
|
||||
from kf_oj.modules.operations import models as operations_models # noqa: F401
|
||||
from kf_oj.modules.problems import import_models, maintenance_models # noqa: F401
|
||||
from kf_oj.modules.problems import models as problem_models # noqa: F401
|
||||
from kf_oj.modules.submissions import models as submission_models # noqa: F401
|
||||
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Immutable per-language judging configuration and extended submission language IDs.
|
||||
|
||||
Revision ID: 0002_multi_runtime
|
||||
Revises: 0001_judging_foundation
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
revision = "0002_multi_runtime"
|
||||
down_revision = "0001_judging_foundation"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
CONFIGURATION_CHECK = """
|
||||
(judging_schema_version = 1 AND language_configurations IS NULL
|
||||
AND toolchain IS NOT NULL AND compile_limits IS NOT NULL
|
||||
AND run_limits IS NOT NULL AND job_wall_time_us IS NOT NULL)
|
||||
OR
|
||||
(judging_schema_version = 2 AND language_configurations IS NOT NULL
|
||||
AND toolchain IS NULL AND compile_limits IS NULL
|
||||
AND run_limits IS NULL AND job_wall_time_us IS NULL
|
||||
AND public.kf_valid_language_configurations(languages, language_configurations))
|
||||
"""
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Metadata-only additions/widening; do not UPDATE any immutable published rows.
|
||||
op.add_column(
|
||||
"problem_versions",
|
||||
sa.Column("judging_schema_version", sa.SmallInteger(), nullable=False, server_default="1"),
|
||||
)
|
||||
op.add_column(
|
||||
"problem_versions",
|
||||
sa.Column("language_configurations", JSONB(none_as_null=True), nullable=True),
|
||||
)
|
||||
for column, kind in (
|
||||
("toolchain", JSONB()),
|
||||
("compile_limits", JSONB()),
|
||||
("run_limits", JSONB()),
|
||||
("job_wall_time_us", sa.BigInteger()),
|
||||
):
|
||||
op.alter_column("problem_versions", column, existing_type=kind, nullable=True)
|
||||
op.alter_column(
|
||||
"submissions",
|
||||
"language",
|
||||
existing_type=sa.String(8),
|
||||
type_=sa.String(16),
|
||||
existing_nullable=False,
|
||||
)
|
||||
op.drop_constraint(op.f("ck_submissions_language"), "submissions", type_="check")
|
||||
op.create_check_constraint(
|
||||
op.f("ck_submissions_language"),
|
||||
"submissions",
|
||||
"language IN ('c17','cpp20','python313','java21')",
|
||||
)
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_valid_runtime_limits(limits jsonb)
|
||||
RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
DECLARE entry record;
|
||||
BEGIN
|
||||
IF jsonb_typeof(limits) != 'object' OR
|
||||
(SELECT count(*) FROM jsonb_object_keys(limits)) != 10 THEN RETURN false; END IF;
|
||||
FOR entry IN SELECT * FROM (VALUES
|
||||
('cpu_time_us',1000::numeric,60000000::numeric),
|
||||
('wall_time_us',1000,120000000), ('memory_bytes',1,2147483648),
|
||||
('stack_bytes',1,268435456), ('processes',1,128), ('open_files',3,1024),
|
||||
('stdout_bytes',1,16777216), ('stderr_bytes',1,16777216),
|
||||
('file_bytes',1,67108864), ('workdir_bytes',1,268435456)
|
||||
) AS bounds(name,lo,hi) LOOP
|
||||
IF NOT limits ? entry.name OR jsonb_typeof(limits->entry.name) != 'number'
|
||||
OR (limits->>entry.name) !~ '^(0|[1-9][0-9]*)$'
|
||||
OR length(limits->>entry.name) > 19 THEN RETURN false; END IF;
|
||||
IF (limits->>entry.name)::numeric NOT BETWEEN entry.lo AND entry.hi
|
||||
THEN RETURN false; END IF;
|
||||
END LOOP;
|
||||
RETURN (limits->>'stack_bytes')::numeric <= (limits->>'memory_bytes')::numeric
|
||||
AND (limits->>'file_bytes')::numeric <= (limits->>'workdir_bytes')::numeric
|
||||
AND (limits->>'stdout_bytes')::numeric <= (limits->>'file_bytes')::numeric
|
||||
AND (limits->>'stderr_bytes')::numeric <= (limits->>'file_bytes')::numeric
|
||||
AND (limits->>'stdout_bytes')::numeric + (limits->>'stderr_bytes')::numeric
|
||||
<= (limits->>'workdir_bytes')::numeric;
|
||||
END $$
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_valid_language_configurations(languages jsonb, configs jsonb)
|
||||
RETURNS boolean LANGUAGE plpgsql IMMUTABLE STRICT
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
DECLARE item record; cfg jsonb; runtime jsonb; lang text; seen text[] := ARRAY[]::text[];
|
||||
phase jsonb; preparing boolean;
|
||||
BEGIN
|
||||
IF jsonb_typeof(languages) != 'array' OR jsonb_typeof(configs) != 'array'
|
||||
THEN RETURN false; END IF;
|
||||
IF jsonb_array_length(configs) NOT BETWEEN 1 AND 4 OR
|
||||
jsonb_array_length(languages) != jsonb_array_length(configs)
|
||||
THEN RETURN false; END IF;
|
||||
FOR item IN SELECT value, ordinality FROM jsonb_array_elements(configs)
|
||||
WITH ORDINALITY LOOP
|
||||
cfg := item.value;
|
||||
IF jsonb_typeof(cfg) != 'object' OR
|
||||
(SELECT count(*) FROM jsonb_object_keys(cfg)) != 5 OR NOT cfg ?&
|
||||
ARRAY['language','toolchain','compile_limits','run_limits','job_wall_time_us']
|
||||
THEN RETURN false; END IF;
|
||||
IF jsonb_typeof(cfg->'language') != 'string' THEN RETURN false; END IF;
|
||||
lang := cfg->>'language';
|
||||
IF lang NOT IN ('c17','cpp20','python313','java21') OR lang = ANY(seen)
|
||||
OR languages->(item.ordinality::int-1) != cfg->'language'
|
||||
THEN RETURN false; END IF;
|
||||
seen := array_append(seen,lang); runtime := cfg->'toolchain';
|
||||
IF jsonb_typeof(runtime) != 'object' OR
|
||||
(SELECT count(*) FROM jsonb_object_keys(runtime)) != 5 OR NOT runtime ?&
|
||||
ARRAY['toolchain_id','image_kind','image_revision','image_digest','runtime_version']
|
||||
THEN RETURN false; END IF;
|
||||
IF jsonb_typeof(runtime->'toolchain_id') != 'string' OR
|
||||
jsonb_typeof(runtime->'image_kind') != 'string' OR
|
||||
jsonb_typeof(runtime->'runtime_version') != 'string' OR
|
||||
jsonb_typeof(runtime->'image_digest') != 'string' OR
|
||||
(runtime->>'image_digest') !~ '^[0-9a-f]{64}$' OR
|
||||
(runtime->>'runtime_version') !~ '^[A-Za-z0-9][A-Za-z0-9_.+-]{0,63}$' OR
|
||||
jsonb_typeof(runtime->'image_revision') != 'number' OR
|
||||
(runtime->>'image_revision') !~ '^[1-9][0-9]{0,9}$'
|
||||
THEN RETURN false; END IF;
|
||||
IF (runtime->>'image_revision')::numeric > 4294967295 THEN RETURN false; END IF;
|
||||
IF lang IN ('c17','cpp20') AND (runtime->>'toolchain_id' != 'gnu-cxx-v1' OR
|
||||
runtime->>'image_kind' != 'linux_gnu_cxx_x86_64_v1') THEN RETURN false; END IF;
|
||||
IF lang = 'python313' AND (runtime->>'toolchain_id' != 'cpython313-v1' OR
|
||||
runtime->>'image_kind' != 'linux_cpython313_x86_64_v1' OR
|
||||
runtime->>'runtime_version' != '3.13.16') THEN RETURN false; END IF;
|
||||
IF lang = 'java21' AND (runtime->>'toolchain_id' != 'openjdk21-v1' OR
|
||||
runtime->>'image_kind' != 'linux_openjdk21_x86_64_v1' OR
|
||||
(runtime->>'runtime_version') !~
|
||||
'^21[.][0-9]+[.][0-9]+([.][0-9]+)?[+][1-9][0-9]*(-LTS)?$')
|
||||
THEN RETURN false; END IF;
|
||||
IF NOT public.kf_valid_runtime_limits(cfg->'compile_limits') OR
|
||||
NOT public.kf_valid_runtime_limits(cfg->'run_limits') THEN RETURN false; END IF;
|
||||
IF jsonb_typeof(cfg->'job_wall_time_us') != 'number' OR
|
||||
(cfg->>'job_wall_time_us') !~ '^[1-9][0-9]{0,9}$' THEN RETURN false; END IF;
|
||||
IF (cfg->>'job_wall_time_us')::numeric NOT BETWEEN 1000 AND 3600000000 OR
|
||||
(cfg->'compile_limits'->>'wall_time_us')::numeric +
|
||||
(cfg->'run_limits'->>'wall_time_us')::numeric >
|
||||
(cfg->>'job_wall_time_us')::numeric
|
||||
THEN RETURN false; END IF;
|
||||
FOR preparing, phase IN SELECT true,cfg->'compile_limits' UNION ALL
|
||||
SELECT false,cfg->'run_limits' LOOP
|
||||
IF lang = 'python313' AND ((phase->>'cpu_time_us')::numeric < 10000 OR
|
||||
(phase->>'wall_time_us')::numeric < 50000 OR
|
||||
(phase->>'memory_bytes')::numeric < 33554432 OR
|
||||
(phase->>'open_files')::numeric < 16) THEN RETURN false; END IF;
|
||||
IF lang = 'java21' AND ((phase->>'cpu_time_us')::numeric <
|
||||
CASE WHEN preparing THEN 1000000 ELSE 200000 END OR
|
||||
(phase->>'wall_time_us')::numeric <
|
||||
CASE WHEN preparing THEN 3000000 ELSE 1000000 END OR
|
||||
(phase->>'memory_bytes')::numeric <
|
||||
CASE WHEN preparing THEN 402653184 ELSE 268435456 END OR
|
||||
(phase->>'processes')::numeric < 16 OR
|
||||
(phase->>'open_files')::numeric < 32 OR
|
||||
(phase->>'stack_bytes')::numeric < 262144 OR
|
||||
(phase->>'memory_bytes')::numeric - 167772160 -
|
||||
(phase->>'processes')::numeric * 262144 < 33554432)
|
||||
THEN RETURN false; END IF;
|
||||
END LOOP;
|
||||
END LOOP;
|
||||
RETURN true;
|
||||
END $$
|
||||
""")
|
||||
op.execute("REVOKE ALL ON FUNCTION public.kf_valid_runtime_limits(jsonb) FROM PUBLIC")
|
||||
op.execute(
|
||||
"REVOKE ALL ON FUNCTION public.kf_valid_language_configurations(jsonb,jsonb) FROM PUBLIC"
|
||||
)
|
||||
op.execute("GRANT EXECUTE ON FUNCTION public.kf_valid_runtime_limits(jsonb) TO kf_oj_platform")
|
||||
op.execute(
|
||||
"GRANT EXECUTE ON FUNCTION public.kf_valid_language_configurations(jsonb,jsonb) "
|
||||
"TO kf_oj_platform"
|
||||
)
|
||||
op.create_check_constraint(
|
||||
op.f("ck_problem_versions_judging_schema"),
|
||||
"problem_versions",
|
||||
"judging_schema_version IN (1,2)",
|
||||
)
|
||||
op.create_check_constraint(
|
||||
op.f("ck_problem_versions_language_configurations"), "problem_versions", CONFIGURATION_CHECK
|
||||
)
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_submission_language() RETURNS trigger LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
DECLARE supported jsonb;
|
||||
BEGIN
|
||||
SELECT languages INTO supported FROM public.problem_versions
|
||||
WHERE id=NEW.problem_version_id;
|
||||
IF FOUND AND NOT supported @> jsonb_build_array(NEW.language) THEN
|
||||
RAISE EXCEPTION 'submission language not supported by version'
|
||||
USING ERRCODE = '23514';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$
|
||||
""")
|
||||
op.execute("REVOKE ALL ON FUNCTION public.kf_submission_language() FROM PUBLIC")
|
||||
op.execute("""
|
||||
CREATE TRIGGER kf_submission_language BEFORE INSERT OR UPDATE OF
|
||||
language,problem_version_id ON public.submissions
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_submission_language()
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Losing a runtime configuration is not a rollback strategy. Preserve all user data.
|
||||
op.execute("""
|
||||
DO $$ BEGIN
|
||||
IF EXISTS (SELECT 1 FROM problem_versions WHERE judging_schema_version = 2) OR
|
||||
EXISTS (SELECT 1 FROM submissions WHERE language NOT IN ('c17','cpp20')) THEN
|
||||
RAISE EXCEPTION 'cannot downgrade while multi-runtime data exists'
|
||||
USING ERRCODE = '23514';
|
||||
END IF;
|
||||
END $$
|
||||
""")
|
||||
op.drop_constraint(
|
||||
op.f("ck_problem_versions_language_configurations"), "problem_versions", type_="check"
|
||||
)
|
||||
op.drop_constraint(
|
||||
op.f("ck_problem_versions_judging_schema"), "problem_versions", type_="check"
|
||||
)
|
||||
op.execute("DROP TRIGGER kf_submission_language ON public.submissions")
|
||||
op.execute("DROP FUNCTION public.kf_submission_language()")
|
||||
op.execute("DROP FUNCTION public.kf_valid_language_configurations(jsonb,jsonb)")
|
||||
op.execute("DROP FUNCTION public.kf_valid_runtime_limits(jsonb)")
|
||||
op.drop_column("problem_versions", "language_configurations")
|
||||
op.drop_column("problem_versions", "judging_schema_version")
|
||||
for column, kind in (
|
||||
("toolchain", JSONB()),
|
||||
("compile_limits", JSONB()),
|
||||
("run_limits", JSONB()),
|
||||
("job_wall_time_us", sa.BigInteger()),
|
||||
):
|
||||
op.alter_column("problem_versions", column, existing_type=kind, nullable=False)
|
||||
op.drop_constraint(op.f("ck_submissions_language"), "submissions", type_="check")
|
||||
op.create_check_constraint(
|
||||
op.f("ck_submissions_language"), "submissions", "language IN ('c17','cpp20')"
|
||||
)
|
||||
op.alter_column(
|
||||
"submissions",
|
||||
"language",
|
||||
existing_type=sa.String(16),
|
||||
type_=sa.String(8),
|
||||
existing_nullable=False,
|
||||
)
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Trusted final performance projection; no re-encoding or promotion of old reports."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0003_submission_performance"
|
||||
down_revision = "0002_multi_runtime"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("judge_attempts", sa.Column("measurement_epoch", sa.String(64), nullable=True))
|
||||
op.create_check_constraint(
|
||||
op.f("ck_judge_attempts_measurement_epoch"),
|
||||
"judge_attempts",
|
||||
"measurement_epoch IS NULL OR measurement_epoch ~ '^[0-9a-f]{64}$'",
|
||||
)
|
||||
op.create_table(
|
||||
"evaluation_performance",
|
||||
sa.Column("evaluation_id", sa.Uuid(), primary_key=True),
|
||||
sa.Column("submission_id", sa.Uuid(), nullable=False, unique=True),
|
||||
sa.Column("attempt_id", sa.Uuid(), nullable=False, unique=True),
|
||||
sa.Column("problem_version_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("language", sa.String(16), nullable=False),
|
||||
sa.Column("comparison_group", sa.String(64), nullable=False),
|
||||
sa.Column("measurement_epoch", sa.String(64), nullable=False),
|
||||
sa.Column("report_digest", sa.String(64), nullable=False),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("cpu_time_us", sa.BigInteger(), nullable=True),
|
||||
sa.Column("memory_peak_bytes", sa.BigInteger(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["evaluation_id", "problem_version_id"],
|
||||
["evaluations.id", "evaluations.problem_version_id"],
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["submission_id", "problem_version_id"],
|
||||
["submissions.id", "submissions.problem_version_id"],
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["attempt_id", "evaluation_id"], ["judge_attempts.id", "judge_attempts.evaluation_id"]
|
||||
),
|
||||
sa.CheckConstraint("language IN ('c17','cpp20','python313','java21')", name="language"),
|
||||
sa.CheckConstraint("comparison_group ~ '^[0-9a-f]{64}$'", name="comparison_group"),
|
||||
sa.CheckConstraint("measurement_epoch ~ '^[0-9a-f]{64}$'", name="measurement_epoch"),
|
||||
sa.CheckConstraint("report_digest ~ '^[0-9a-f]{64}$'", name="report_digest"),
|
||||
sa.CheckConstraint("cpu_time_us IS NULL OR cpu_time_us >= 0", name="cpu"),
|
||||
sa.CheckConstraint("memory_peak_bytes IS NULL OR memory_peak_bytes >= 0", name="memory"),
|
||||
)
|
||||
for metric in ("cpu_time_us", "memory_peak_bytes"):
|
||||
op.create_index(
|
||||
f"ix_performance_{metric}_cohort",
|
||||
"evaluation_performance",
|
||||
["comparison_group", sa.text("finished_at DESC"), sa.text("evaluation_id DESC")],
|
||||
postgresql_where=sa.text(f"{metric} IS NOT NULL"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_submissions_owned_problem_history",
|
||||
"submissions",
|
||||
["account_id", "problem_version_id", sa.text("created_at DESC"), sa.text("id DESC")],
|
||||
)
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_trusted_performance() RETURNS trigger LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
BEGIN
|
||||
IF TG_OP != 'INSERT' THEN
|
||||
RAISE EXCEPTION 'final performance is immutable' USING ERRCODE = '23514';
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM public.evaluations e
|
||||
JOIN public.judge_attempts a ON a.id=e.current_attempt_id AND a.evaluation_id=e.id
|
||||
JOIN public.submissions s ON s.id=e.submission_id
|
||||
WHERE e.id=NEW.evaluation_id AND e.state='finished' AND e.verdict='accepted'
|
||||
AND e.failure_code IS NULL AND a.cleanup_complete IS TRUE
|
||||
AND e.current_attempt_id=NEW.attempt_id AND s.id=NEW.submission_id
|
||||
AND s.problem_version_id=NEW.problem_version_id AND s.language=NEW.language
|
||||
AND a.measurement_epoch=NEW.measurement_epoch
|
||||
AND e.final_report_digest=NEW.report_digest AND a.report_digest=NEW.report_digest
|
||||
AND e.updated_at=NEW.finished_at
|
||||
AND e.final_report->>'cleanup_complete'='true'
|
||||
AND (e.final_report->>'run_cpu_time_us')::bigint
|
||||
IS NOT DISTINCT FROM NEW.cpu_time_us
|
||||
AND (e.final_report->>'run_memory_peak_bytes')::bigint
|
||||
IS NOT DISTINCT FROM NEW.memory_peak_bytes
|
||||
) THEN
|
||||
RAISE EXCEPTION 'performance requires a bound trusted final AC'
|
||||
USING ERRCODE = '23514';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$
|
||||
""")
|
||||
op.execute("REVOKE ALL ON FUNCTION public.kf_trusted_performance() FROM PUBLIC")
|
||||
op.execute("""
|
||||
CREATE TRIGGER kf_trusted_performance BEFORE INSERT OR UPDATE OR DELETE
|
||||
ON public.evaluation_performance FOR EACH ROW
|
||||
EXECUTE FUNCTION public.kf_trusted_performance()
|
||||
""")
|
||||
op.execute("REVOKE ALL ON evaluation_performance FROM PUBLIC,kf_oj_platform,kf_oj_coordinator")
|
||||
op.execute("GRANT SELECT ON evaluation_performance TO kf_oj_platform,kf_oj_coordinator")
|
||||
op.execute("GRANT INSERT ON evaluation_performance TO kf_oj_coordinator")
|
||||
# Existing attempt INSERT permission covers the new column. No UPDATE epoch permission:
|
||||
# an unbound historical attempt must never be promoted by changing current configuration.
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("""
|
||||
DO $$ BEGIN
|
||||
IF EXISTS(SELECT 1 FROM evaluation_performance) OR
|
||||
EXISTS(SELECT 1 FROM judge_attempts WHERE measurement_epoch IS NOT NULL) THEN
|
||||
RAISE EXCEPTION 'cannot discard trusted measurement bindings'
|
||||
USING ERRCODE = '23514';
|
||||
END IF;
|
||||
END $$
|
||||
""")
|
||||
op.drop_table("evaluation_performance")
|
||||
op.execute("DROP FUNCTION public.kf_trusted_performance()")
|
||||
op.drop_index("ix_submissions_owned_problem_history", table_name="submissions")
|
||||
op.drop_constraint(op.f("ck_judge_attempts_measurement_epoch"), "judge_attempts", type_="check")
|
||||
op.drop_column("judge_attempts", "measurement_epoch")
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Indexed group watermark for bounded aggregate cache invalidation."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0004_performance_watermark"
|
||||
down_revision = "0003_submission_performance"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_index(
|
||||
"ix_performance_group_watermark",
|
||||
"evaluation_performance",
|
||||
["comparison_group", sa.text("finished_at DESC"), sa.text("evaluation_id DESC")],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_performance_group_watermark", table_name="evaluation_performance")
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Default-denied administrator authority and separate local operator/import roles."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0005_identity_authority"
|
||||
down_revision = "0004_performance_watermark"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"accounts",
|
||||
sa.Column("is_administrator", sa.Boolean(), server_default="false", nullable=False),
|
||||
)
|
||||
op.add_column(
|
||||
"accounts",
|
||||
sa.Column("authorization_version", sa.BigInteger(), server_default="1", nullable=False),
|
||||
)
|
||||
op.create_check_constraint(
|
||||
op.f("ck_accounts_authorization_version"), "accounts", "authorization_version >= 1"
|
||||
)
|
||||
op.create_index(
|
||||
"ix_accounts_active_administrators",
|
||||
"accounts",
|
||||
["id"],
|
||||
postgresql_where=sa.text("is_administrator AND enabled"),
|
||||
)
|
||||
# Logins and secrets are provisioned separately by an operator. Neither group
|
||||
# may inherit an existing privileged role or quietly reuse a privileged login.
|
||||
for role in ("kf_oj_identity_operator", "kf_oj_problem_import"):
|
||||
op.execute(f"""
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{role}') THEN
|
||||
CREATE ROLE {role} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE
|
||||
NOREPLICATION NOBYPASSRLS;
|
||||
ELSIF EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{role}' AND
|
||||
(rolcanlogin OR rolsuper OR rolcreatedb OR rolcreaterole
|
||||
OR rolreplication OR rolbypassrls))
|
||||
OR EXISTS (SELECT 1 FROM pg_auth_members m JOIN pg_roles r ON r.oid=m.member
|
||||
WHERE r.rolname='{role}') THEN
|
||||
RAISE EXCEPTION 'unsafe existing authority or import role';
|
||||
END IF;
|
||||
END $$
|
||||
""")
|
||||
op.execute(
|
||||
"REVOKE ALL ON ALL TABLES IN SCHEMA public "
|
||||
"FROM kf_oj_identity_operator,kf_oj_problem_import"
|
||||
)
|
||||
# Table REVOKE does not remove explicit column grants. Remove those too,
|
||||
# including grants left by an earlier downgrade or manual role provisioning.
|
||||
op.execute("""
|
||||
DO $$ DECLARE t record; BEGIN
|
||||
FOR t IN SELECT c.relname, string_agg(quote_ident(a.attname),',' ORDER BY a.attnum) cols
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
|
||||
JOIN pg_attribute a ON a.attrelid=c.oid
|
||||
WHERE n.nspname='public' AND c.relkind IN ('r','p','v','f')
|
||||
AND a.attnum>0 AND NOT a.attisdropped
|
||||
GROUP BY c.oid,c.relname LOOP
|
||||
EXECUTE format('REVOKE SELECT (%s),INSERT (%s),UPDATE (%s),REFERENCES (%s) '
|
||||
'ON TABLE public.%I FROM kf_oj_identity_operator,kf_oj_problem_import',
|
||||
t.cols,t.cols,t.cols,t.cols,t.relname);
|
||||
END LOOP;
|
||||
END $$
|
||||
""")
|
||||
op.execute("GRANT USAGE ON SCHEMA public TO kf_oj_identity_operator,kf_oj_problem_import")
|
||||
op.execute("REVOKE INSERT,UPDATE,DELETE ON accounts FROM kf_oj_platform")
|
||||
op.execute(
|
||||
"REVOKE INSERT(id,username,password_hash,enabled,created_at,"
|
||||
"is_administrator,authorization_version),"
|
||||
"UPDATE(id,username,password_hash,enabled,created_at,"
|
||||
"is_administrator,authorization_version) "
|
||||
"ON accounts FROM kf_oj_platform"
|
||||
)
|
||||
op.execute(
|
||||
"GRANT INSERT(id,username,password_hash,enabled,created_at) ON accounts TO kf_oj_platform"
|
||||
)
|
||||
op.execute("GRANT UPDATE(enabled) ON accounts TO kf_oj_platform")
|
||||
op.execute("GRANT SELECT ON accounts,sessions TO kf_oj_identity_operator")
|
||||
op.execute("GRANT UPDATE(is_administrator,enabled) ON accounts TO kf_oj_identity_operator")
|
||||
op.execute("GRANT UPDATE(revoked_at) ON sessions TO kf_oj_identity_operator")
|
||||
op.execute("REVOKE ALL ON accounts,sessions FROM kf_oj_coordinator,kf_oj_problem_import")
|
||||
# No import queue exists yet. Its role deliberately has no table privileges
|
||||
# until the import migration grants only the queue/staging tables it needs.
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_account_authority_version() RETURNS trigger LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
BEGIN
|
||||
IF NEW.authorization_version IS DISTINCT FROM OLD.authorization_version THEN
|
||||
RAISE EXCEPTION 'authorization version is server derived' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF OLD.is_administrator AND NEW.enabled IS DISTINCT FROM OLD.enabled
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','USAGE') THEN
|
||||
RAISE EXCEPTION 'administrator enablement requires the local operator'
|
||||
USING ERRCODE='42501';
|
||||
END IF;
|
||||
IF NEW.is_administrator IS DISTINCT FROM OLD.is_administrator
|
||||
OR NEW.enabled IS DISTINCT FROM OLD.enabled THEN
|
||||
NEW.authorization_version := OLD.authorization_version + 1;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_revoke_authority_sessions() RETURNS trigger LANGUAGE plpgsql
|
||||
SET search_path = pg_catalog, public AS $$
|
||||
BEGIN
|
||||
IF NEW.authorization_version != OLD.authorization_version THEN
|
||||
UPDATE public.sessions SET revoked_at=coalesce(revoked_at,clock_timestamp())
|
||||
WHERE account_id=NEW.id AND revoked_at IS NULL;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$
|
||||
""")
|
||||
for function in ("kf_account_authority_version", "kf_revoke_authority_sessions"):
|
||||
op.execute(f"REVOKE ALL ON FUNCTION public.{function}() FROM PUBLIC")
|
||||
op.execute("""
|
||||
CREATE TRIGGER kf_account_authority_version BEFORE UPDATE ON public.accounts
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_account_authority_version()
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE TRIGGER kf_revoke_authority_sessions AFTER UPDATE ON public.accounts
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_revoke_authority_sessions()
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("""
|
||||
DO $$ BEGIN
|
||||
IF EXISTS(SELECT 1 FROM accounts
|
||||
WHERE is_administrator OR authorization_version != 1) THEN
|
||||
RAISE EXCEPTION 'cannot discard authority or revocation history'
|
||||
USING ERRCODE='23514';
|
||||
END IF;
|
||||
END $$
|
||||
""")
|
||||
for name in ("kf_revoke_authority_sessions", "kf_account_authority_version"):
|
||||
op.execute(f"DROP TRIGGER {name} ON public.accounts")
|
||||
op.execute(f"DROP FUNCTION public.{name}()")
|
||||
op.execute("REVOKE ALL ON accounts,sessions FROM kf_oj_identity_operator")
|
||||
op.execute("REVOKE UPDATE(is_administrator,enabled) ON accounts FROM kf_oj_identity_operator")
|
||||
op.execute("REVOKE UPDATE(revoked_at) ON sessions FROM kf_oj_identity_operator")
|
||||
op.drop_index("ix_accounts_active_administrators", table_name="accounts")
|
||||
op.drop_constraint(op.f("ck_accounts_authorization_version"), "accounts", type_="check")
|
||||
op.drop_column("accounts", "authorization_version")
|
||||
op.drop_column("accounts", "is_administrator")
|
||||
op.execute("GRANT INSERT,UPDATE,DELETE ON accounts TO kf_oj_platform")
|
||||
# Shared NOLOGIN groups survive downgrade, as in the foundation migration;
|
||||
# removing a cluster role could break another database using the same groups.
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Fixed local authority operations and serialized last-administrator protection."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0006_authority_operations"
|
||||
down_revision = "0005_identity_authority"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
OWNER = "kf_oj_identity_authority_owner"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(f"""
|
||||
DO $$ BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{OWNER}') THEN
|
||||
CREATE ROLE {OWNER} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE
|
||||
NOREPLICATION NOBYPASSRLS;
|
||||
ELSIF EXISTS (SELECT 1 FROM pg_roles WHERE rolname='{OWNER}' AND
|
||||
(rolcanlogin OR rolsuper OR rolcreatedb OR rolcreaterole
|
||||
OR rolreplication OR rolbypassrls))
|
||||
OR EXISTS (SELECT 1 FROM pg_auth_members m JOIN pg_roles r ON r.oid=m.member
|
||||
WHERE r.rolname='{OWNER}') THEN
|
||||
RAISE EXCEPTION 'unsafe existing authority function owner';
|
||||
END IF;
|
||||
END $$
|
||||
""")
|
||||
op.create_table(
|
||||
"identity_authority_guard",
|
||||
sa.Column("id", sa.SmallInteger(), primary_key=True),
|
||||
sa.Column("generation", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.CheckConstraint("id = 1", name="singleton"),
|
||||
sa.CheckConstraint("generation >= 0", name="generation"),
|
||||
)
|
||||
op.execute("INSERT INTO public.identity_authority_guard(id) VALUES(1)")
|
||||
for role in (OWNER, "kf_oj_identity_operator"):
|
||||
op.execute(f"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM {role}")
|
||||
op.execute(f"""
|
||||
REVOKE SELECT(id,username,password_hash,enabled,created_at,
|
||||
is_administrator,authorization_version),
|
||||
UPDATE(id,username,password_hash,enabled,created_at,
|
||||
is_administrator,authorization_version) ON public.accounts FROM {role}
|
||||
""")
|
||||
op.execute(f"""
|
||||
REVOKE SELECT(token_hash,account_id,csrf_hash,expires_at,revoked_at,created_at),
|
||||
UPDATE(token_hash,account_id,csrf_hash,expires_at,revoked_at,created_at)
|
||||
ON public.sessions FROM {role}
|
||||
""")
|
||||
op.execute(f"GRANT USAGE ON SCHEMA public TO {OWNER}")
|
||||
op.execute(f"""
|
||||
GRANT SELECT(id,username,enabled,is_administrator,authorization_version),
|
||||
UPDATE(enabled,is_administrator) ON public.accounts TO {OWNER}
|
||||
""")
|
||||
op.execute(f"GRANT SELECT(account_id,revoked_at),UPDATE(revoked_at) ON sessions TO {OWNER}")
|
||||
op.execute(f"GRANT SELECT,UPDATE(generation) ON identity_authority_guard TO {OWNER}")
|
||||
# Every account write obtains the guard BEFORE it can lock an account row.
|
||||
# Updating (rather than merely locking) the singleton also makes an old
|
||||
# REPEATABLE READ/SERIALIZABLE snapshot fail safely with a serialization error.
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_serialize_account_authority() RETURNS trigger
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||
BEGIN
|
||||
UPDATE public.identity_authority_guard SET generation=generation+1 WHERE id=1;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'authority guard unavailable' USING ERRCODE='23514';
|
||||
END IF;
|
||||
RETURN NULL;
|
||||
END $$
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_protect_last_administrator() RETURNS trigger
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||
BEGIN
|
||||
IF OLD.is_administrator AND OLD.enabled AND
|
||||
(TG_OP='DELETE' OR NOT (NEW.is_administrator AND NEW.enabled)) AND
|
||||
(SELECT count(*) FROM public.accounts WHERE is_administrator AND enabled) <= 1
|
||||
THEN
|
||||
RAISE EXCEPTION 'last enabled administrator is protected' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF TG_OP='DELETE' THEN RETURN OLD; END IF;
|
||||
RETURN NEW;
|
||||
END $$
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION public.kf_account_authority_version() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public AS $$
|
||||
BEGIN
|
||||
IF NEW.authorization_version IS DISTINCT FROM OLD.authorization_version THEN
|
||||
RAISE EXCEPTION 'authorization version is server derived' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF OLD.is_administrator AND NEW.enabled IS DISTINCT FROM OLD.enabled
|
||||
AND current_user != 'kf_oj_identity_authority_owner'
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','USAGE') THEN
|
||||
RAISE EXCEPTION 'administrator enablement requires the local operator'
|
||||
USING ERRCODE='42501';
|
||||
END IF;
|
||||
IF NEW.is_administrator IS DISTINCT FROM OLD.is_administrator
|
||||
OR NEW.enabled IS DISTINCT FROM OLD.enabled THEN
|
||||
NEW.authorization_version := OLD.authorization_version + 1;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_manage_account_authority(p_username text,p_operation text)
|
||||
RETURNS TABLE(account_id uuid,username text,enabled boolean,is_administrator boolean,
|
||||
authorization_version bigint,changed boolean)
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $$
|
||||
DECLARE target record; desired_admin boolean; desired_enabled boolean; active bigint;
|
||||
BEGIN
|
||||
IF p_username IS NULL OR p_username !~ '^[a-z0-9_]{3,24}$' OR
|
||||
p_operation IS NULL OR
|
||||
p_operation NOT IN ('bootstrap','grant','revoke','enable','disable') THEN
|
||||
RAISE EXCEPTION 'invalid authority operation' USING ERRCODE='22023';
|
||||
END IF;
|
||||
UPDATE public.identity_authority_guard g SET generation=g.generation+1 WHERE g.id=1;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'authority guard unavailable' USING ERRCODE='23514';
|
||||
END IF;
|
||||
SELECT a.id,a.username,a.enabled,a.is_administrator,a.authorization_version
|
||||
INTO target FROM public.accounts a WHERE a.username=p_username FOR UPDATE;
|
||||
IF NOT FOUND THEN
|
||||
RAISE EXCEPTION 'account not found' USING ERRCODE='P0002';
|
||||
END IF;
|
||||
desired_admin:=target.is_administrator; desired_enabled:=target.enabled;
|
||||
SELECT count(*) INTO active FROM public.accounts a
|
||||
WHERE a.is_administrator AND a.enabled;
|
||||
IF p_operation='bootstrap' THEN
|
||||
IF active>0 AND NOT (target.is_administrator AND target.enabled) THEN
|
||||
RAISE EXCEPTION 'bootstrap requires no enabled administrator'
|
||||
USING ERRCODE='23514';
|
||||
END IF;
|
||||
desired_admin:=true; desired_enabled:=true;
|
||||
ELSIF p_operation='grant' THEN
|
||||
IF active=0 THEN
|
||||
RAISE EXCEPTION 'first administrator requires explicit bootstrap'
|
||||
USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF NOT target.enabled THEN
|
||||
RAISE EXCEPTION 'enable the account before granting authority'
|
||||
USING ERRCODE='23514';
|
||||
END IF;
|
||||
desired_admin:=true;
|
||||
ELSIF p_operation='revoke' THEN desired_admin:=false;
|
||||
ELSIF p_operation='enable' THEN desired_enabled:=true;
|
||||
ELSE desired_enabled:=false;
|
||||
END IF;
|
||||
changed:=desired_admin IS DISTINCT FROM target.is_administrator OR
|
||||
desired_enabled IS DISTINCT FROM target.enabled;
|
||||
IF changed THEN
|
||||
UPDATE public.accounts a SET enabled=desired_enabled,is_administrator=desired_admin
|
||||
WHERE a.id=target.id;
|
||||
END IF;
|
||||
RETURN QUERY SELECT a.id,a.username::text,a.enabled,a.is_administrator,
|
||||
a.authorization_version,changed FROM public.accounts a WHERE a.id=target.id;
|
||||
END $$
|
||||
""")
|
||||
# A narrow NOLOGIN owner holds only these column privileges. The operator
|
||||
# receives EXECUTE, never table writes or access to password/session digests.
|
||||
op.execute(f"GRANT CREATE ON SCHEMA public TO {OWNER}")
|
||||
for signature in (
|
||||
"kf_serialize_account_authority()",
|
||||
"kf_protect_last_administrator()",
|
||||
"kf_manage_account_authority(text,text)",
|
||||
):
|
||||
op.execute(f"ALTER FUNCTION public.{signature} OWNER TO {OWNER}")
|
||||
op.execute(f"REVOKE ALL ON FUNCTION public.{signature} FROM PUBLIC")
|
||||
op.execute(f"REVOKE CREATE ON SCHEMA public FROM {OWNER}")
|
||||
op.execute("""
|
||||
GRANT EXECUTE ON FUNCTION public.kf_manage_account_authority(text,text)
|
||||
TO kf_oj_identity_operator
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE TRIGGER kf_serialize_account_authority BEFORE INSERT OR UPDATE OR DELETE
|
||||
ON public.accounts FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION public.kf_serialize_account_authority()
|
||||
""")
|
||||
op.execute("""
|
||||
CREATE TRIGGER kf_protect_last_administrator BEFORE UPDATE OR DELETE
|
||||
ON public.accounts FOR EACH ROW EXECUTE FUNCTION public.kf_protect_last_administrator()
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("""
|
||||
DO $$ BEGIN
|
||||
IF EXISTS(SELECT 1 FROM public.accounts WHERE is_administrator) THEN
|
||||
RAISE EXCEPTION 'cannot remove active authority safeguards' USING ERRCODE='23514';
|
||||
END IF;
|
||||
END $$
|
||||
""")
|
||||
for name in ("kf_protect_last_administrator", "kf_serialize_account_authority"):
|
||||
op.execute(f"DROP TRIGGER {name} ON public.accounts")
|
||||
op.execute(f"DROP FUNCTION public.{name}()")
|
||||
op.execute("DROP FUNCTION public.kf_manage_account_authority(text,text)")
|
||||
op.drop_table("identity_authority_guard")
|
||||
op.execute(f"REVOKE ALL ON accounts,sessions FROM {OWNER}")
|
||||
op.execute(f"""
|
||||
REVOKE SELECT(id,username,enabled,is_administrator,authorization_version),
|
||||
UPDATE(enabled,is_administrator) ON accounts FROM {OWNER}
|
||||
""")
|
||||
op.execute(f"REVOKE SELECT(account_id,revoked_at),UPDATE(revoked_at) ON sessions FROM {OWNER}")
|
||||
op.execute("GRANT SELECT ON accounts,sessions TO kf_oj_identity_operator")
|
||||
op.execute("GRANT UPDATE(is_administrator,enabled) ON accounts TO kf_oj_identity_operator")
|
||||
op.execute("GRANT UPDATE(revoked_at) ON sessions TO kf_oj_identity_operator")
|
||||
# Preserve the v0005 trigger policy without referring to the function owner.
|
||||
op.execute("""
|
||||
CREATE OR REPLACE FUNCTION public.kf_account_authority_version() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path = pg_catalog, public AS $$
|
||||
BEGIN
|
||||
IF NEW.authorization_version IS DISTINCT FROM OLD.authorization_version THEN
|
||||
RAISE EXCEPTION 'authorization version is server derived' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF OLD.is_administrator AND NEW.enabled IS DISTINCT FROM OLD.enabled
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','USAGE') THEN
|
||||
RAISE EXCEPTION 'administrator enablement requires the local operator'
|
||||
USING ERRCODE='42501';
|
||||
END IF;
|
||||
IF NEW.is_administrator IS DISTINCT FROM OLD.is_administrator
|
||||
OR NEW.enabled IS DISTINCT FROM OLD.enabled THEN
|
||||
NEW.authorization_version := OLD.authorization_version + 1;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$
|
||||
""")
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Durable imports/candidates, append-only events and accepted-authority worker port."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from kf_oj.modules.operations.models import MaintenanceEvent
|
||||
from kf_oj.modules.problems.import_models import (
|
||||
CandidateCase,
|
||||
ImportPool,
|
||||
ProblemCandidate,
|
||||
ProblemImportOperation,
|
||||
)
|
||||
|
||||
revision = "0007_problem_imports"
|
||||
down_revision = "0006_authority_operations"
|
||||
branch_labels = depends_on = None
|
||||
|
||||
|
||||
def sql(script):
|
||||
# asyncpg prepares one statement at a time; dollar-quoted bodies stay intact.
|
||||
buffer, quote, dollar, i = [], False, False, 0
|
||||
while i < len(script):
|
||||
if not quote and script[i : i + 2] == "$$":
|
||||
dollar = not dollar
|
||||
buffer.append("$$")
|
||||
i += 2
|
||||
continue
|
||||
character = script[i]
|
||||
if not dollar and character == "'":
|
||||
if quote and script[i : i + 2] == "''":
|
||||
buffer.append("''")
|
||||
i += 2
|
||||
continue
|
||||
quote = not quote
|
||||
if character == ";" and not dollar and not quote:
|
||||
if "".join(buffer).strip():
|
||||
op.execute("".join(buffer))
|
||||
buffer = []
|
||||
else:
|
||||
buffer.append(character)
|
||||
i += 1
|
||||
if "".join(buffer).strip():
|
||||
op.execute("".join(buffer))
|
||||
|
||||
|
||||
def upgrade():
|
||||
connection = op.get_bind()
|
||||
# Cyclic candidate/operation link is installed only after both tables exist.
|
||||
for model in (
|
||||
ImportPool,
|
||||
ProblemImportOperation,
|
||||
ProblemCandidate,
|
||||
CandidateCase,
|
||||
):
|
||||
model.__table__.create(connection)
|
||||
# Freeze the historical shape. A later operations model must not cause a
|
||||
# fresh database to skip ahead and conflict with a subsequent ALTER migration.
|
||||
op.create_table(
|
||||
"problem_maintenance_events",
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()
|
||||
),
|
||||
sa.Column("actor_id", sa.Uuid(), sa.ForeignKey("accounts.id"), nullable=False),
|
||||
sa.Column("authorization_version", sa.BigInteger(), nullable=False),
|
||||
sa.Column("operation_id", sa.Uuid(), nullable=False),
|
||||
sa.Column("action", sa.String(24), nullable=False),
|
||||
sa.Column("target_id", sa.Uuid()),
|
||||
sa.CheckConstraint("authorization_version > 0", name="authority"),
|
||||
sa.CheckConstraint(
|
||||
"action IN ('upload_created','upload_received','import_ready','import_failed',"
|
||||
"'import_discarded','import_expired','published','disabled','restored')",
|
||||
name="action",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_problem_maintenance_events_page", "problem_maintenance_events", ["created_at", "id"]
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_problem_import_candidate",
|
||||
"problem_import_operations",
|
||||
"problem_candidates",
|
||||
["candidate_id"],
|
||||
["id"],
|
||||
)
|
||||
op.execute("INSERT INTO problem_import_pool(id,reserved_bytes) VALUES(1,0)")
|
||||
sql("""
|
||||
CREATE FUNCTION public.kf_lock_reserved_maintenance(actor uuid, epoch bigint)
|
||||
RETURNS void LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$
|
||||
DECLARE permitted boolean;
|
||||
BEGIN
|
||||
SELECT enabled AND is_administrator AND authorization_version=epoch
|
||||
INTO permitted FROM public.accounts WHERE id=actor FOR SHARE;
|
||||
IF permitted IS DISTINCT FROM true THEN
|
||||
RAISE EXCEPTION 'problem maintenance required' USING ERRCODE='42501';
|
||||
END IF;
|
||||
END $$;
|
||||
GRANT CREATE ON SCHEMA public TO kf_oj_identity_authority_owner;
|
||||
ALTER FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
|
||||
OWNER TO kf_oj_identity_authority_owner;
|
||||
REVOKE CREATE ON SCHEMA public FROM kf_oj_identity_authority_owner;
|
||||
REVOKE ALL ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint) FROM PUBLIC;
|
||||
GRANT EXECUTE ON FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)
|
||||
TO kf_oj_problem_import;
|
||||
GRANT SELECT,INSERT,UPDATE ON problem_import_operations TO kf_oj_platform;
|
||||
GRANT SELECT,UPDATE ON problem_import_pool TO kf_oj_platform,kf_oj_problem_import;
|
||||
GRANT SELECT ON problem_candidates,problem_candidate_cases TO kf_oj_platform;
|
||||
GRANT SELECT ON problem_import_operations TO kf_oj_problem_import;
|
||||
GRANT UPDATE(state,updated_at,lease_until,attempts,fence,
|
||||
reserved_bytes,failure_code,candidate_id)
|
||||
ON problem_import_operations TO kf_oj_problem_import;
|
||||
GRANT SELECT,INSERT,DELETE ON problem_candidates,problem_candidate_cases
|
||||
TO kf_oj_problem_import;
|
||||
GRANT SELECT,INSERT ON artifacts TO kf_oj_problem_import;
|
||||
GRANT INSERT ON problem_maintenance_events TO kf_oj_platform,kf_oj_problem_import;
|
||||
GRANT SELECT ON problem_maintenance_events TO kf_oj_platform;
|
||||
""")
|
||||
sql("""
|
||||
CREATE FUNCTION public.kf_immutable_import_candidate() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
|
||||
DECLARE candidate uuid; current_state text;
|
||||
BEGIN
|
||||
IF TG_TABLE_NAME='problem_candidates' THEN candidate=OLD.id;
|
||||
ELSE candidate=OLD.candidate_id; END IF;
|
||||
SELECT o.state INTO current_state FROM public.problem_candidates c
|
||||
JOIN public.problem_import_operations o ON o.id=c.operation_id WHERE c.id=candidate;
|
||||
IF TG_OP='UPDATE' OR current_state NOT IN ('discarded','expired','failed') THEN
|
||||
RAISE EXCEPTION 'immutable candidate' USING ERRCODE='23514';
|
||||
END IF;
|
||||
RETURN OLD;
|
||||
END $$;
|
||||
CREATE TRIGGER immutable_candidate BEFORE UPDATE OR DELETE ON problem_candidates
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
|
||||
CREATE TRIGGER immutable_candidate_case BEFORE UPDATE OR DELETE ON problem_candidate_cases
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_immutable_import_candidate();
|
||||
CREATE FUNCTION public.kf_insert_import_candidate() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$
|
||||
DECLARE operation uuid; current_state text;
|
||||
BEGIN
|
||||
IF TG_TABLE_NAME='problem_candidates' THEN operation=NEW.operation_id;
|
||||
ELSE SELECT operation_id INTO operation FROM public.problem_candidates
|
||||
WHERE id=NEW.candidate_id; END IF;
|
||||
SELECT state INTO current_state FROM public.problem_import_operations
|
||||
WHERE id=operation FOR SHARE;
|
||||
IF current_state IS DISTINCT FROM 'validating' THEN
|
||||
RAISE EXCEPTION 'candidate is not being validated' USING ERRCODE='23514';
|
||||
END IF; RETURN NEW;
|
||||
END $$;
|
||||
CREATE TRIGGER insert_candidate BEFORE INSERT ON problem_candidates
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
|
||||
CREATE TRIGGER insert_candidate_case BEFORE INSERT ON problem_candidate_cases
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_insert_import_candidate();
|
||||
CREATE FUNCTION public.kf_fixed_import_receipt() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ BEGIN
|
||||
IF NEW.id IS DISTINCT FROM OLD.id OR NEW.owner_id IS DISTINCT FROM OLD.owner_id
|
||||
OR NEW.authorization_version IS DISTINCT FROM OLD.authorization_version
|
||||
OR NEW.archive_digest IS DISTINCT FROM OLD.archive_digest
|
||||
OR NEW.archive_bytes IS DISTINCT FROM OLD.archive_bytes
|
||||
OR NEW.created_at IS DISTINCT FROM OLD.created_at
|
||||
OR NEW.expires_at IS DISTINCT FROM OLD.expires_at THEN
|
||||
RAISE EXCEPTION 'immutable import receipt' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF OLD.state IN ('ready','published') AND NEW.state IN ('ready','published')
|
||||
AND NEW.candidate_id IS DISTINCT FROM OLD.candidate_id THEN
|
||||
RAISE EXCEPTION 'immutable completed candidate receipt' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF OLD.state='published' AND (NEW.state IS DISTINCT FROM OLD.state
|
||||
OR NEW.published_version_id IS DISTINCT FROM OLD.published_version_id) THEN
|
||||
RAISE EXCEPTION 'immutable published import receipt' USING ERRCODE='23514';
|
||||
END IF;
|
||||
IF NEW.state='ready' THEN
|
||||
IF (OLD.state IS DISTINCT FROM 'ready' AND
|
||||
(NOT pg_has_role(current_user,'kf_oj_problem_import','USAGE')
|
||||
OR OLD.state!='validating')) OR NEW.candidate_id IS NULL
|
||||
OR NOT EXISTS(SELECT 1 FROM public.problem_candidates c
|
||||
WHERE c.id=NEW.candidate_id AND c.operation_id=NEW.id
|
||||
AND c.case_count=(SELECT count(*) FROM public.problem_candidate_cases b
|
||||
WHERE b.candidate_id=c.id)
|
||||
AND c.case_count-1=(SELECT max(position) FROM public.problem_candidate_cases b
|
||||
WHERE b.candidate_id=c.id)) THEN
|
||||
RAISE EXCEPTION 'invalid candidate completion' USING ERRCODE='23514';
|
||||
END IF;
|
||||
END IF; RETURN NEW;
|
||||
END $$;
|
||||
CREATE TRIGGER fixed_import_receipt BEFORE UPDATE ON problem_import_operations
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_fixed_import_receipt();
|
||||
""")
|
||||
|
||||
|
||||
def downgrade():
|
||||
connection = op.get_bind()
|
||||
if connection.exec_driver_sql(
|
||||
"SELECT EXISTS(SELECT 1 FROM problem_import_operations)"
|
||||
).scalar():
|
||||
raise RuntimeError("cannot discard durable import history")
|
||||
op.execute("DROP FUNCTION public.kf_lock_reserved_maintenance(uuid,bigint)")
|
||||
op.drop_constraint(
|
||||
"fk_problem_import_candidate", "problem_import_operations", type_="foreignkey"
|
||||
)
|
||||
for model in (
|
||||
MaintenanceEvent,
|
||||
CandidateCase,
|
||||
ProblemCandidate,
|
||||
ProblemImportOperation,
|
||||
ImportPool,
|
||||
):
|
||||
model.__table__.drop(connection)
|
||||
sql(
|
||||
"DROP FUNCTION public.kf_immutable_import_candidate(); "
|
||||
"DROP FUNCTION public.kf_fixed_import_receipt(); "
|
||||
"DROP FUNCTION public.kf_insert_import_candidate()"
|
||||
)
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Immutable maintenance receipts, explicit operator audit and read-only history."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from kf_oj.modules.problems.maintenance_models import ProblemMaintenanceOperation
|
||||
|
||||
revision = "0008_problem_maintenance"
|
||||
down_revision = "0007_problem_imports"
|
||||
branch_labels = depends_on = None
|
||||
OWNER = "kf_oj_identity_authority_owner"
|
||||
|
||||
|
||||
def upgrade():
|
||||
ProblemMaintenanceOperation.__table__.create(op.get_bind())
|
||||
op.execute("GRANT SELECT,INSERT ON problem_maintenance_operations TO kf_oj_platform")
|
||||
op.add_column(
|
||||
"problem_maintenance_events",
|
||||
sa.Column("actor_kind", sa.String(8), nullable=False, server_default=sa.text("'account'")),
|
||||
)
|
||||
op.add_column("problem_maintenance_events", sa.Column("operator_name", sa.String(63)))
|
||||
op.alter_column("problem_maintenance_events", "actor_id", nullable=True)
|
||||
op.drop_constraint(op.f("ck_problem_maintenance_events_action"), "problem_maintenance_events")
|
||||
op.create_check_constraint(
|
||||
"action",
|
||||
"problem_maintenance_events",
|
||||
"action IN ('upload_created','upload_received','import_ready','import_failed',"
|
||||
"'import_discarded','import_expired','published','disabled','restored',"
|
||||
"'authority_bootstrap','authority_grant','authority_revoke','authority_enable','authority_disable')",
|
||||
)
|
||||
op.create_check_constraint(
|
||||
"actor",
|
||||
"problem_maintenance_events",
|
||||
"(actor_kind='account' AND actor_id IS NOT NULL AND operator_name IS NULL) OR "
|
||||
"(actor_kind='operator' AND actor_id IS NULL AND operator_name IS NOT NULL)",
|
||||
)
|
||||
op.execute("""CREATE FUNCTION public.kf_append_only_maintenance() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,pg_temp AS $$ BEGIN
|
||||
RAISE EXCEPTION 'append-only maintenance history' USING ERRCODE='23514'; END $$""")
|
||||
for table in ("problem_maintenance_operations", "problem_maintenance_events"):
|
||||
op.execute(f"""CREATE TRIGGER append_only_maintenance BEFORE UPDATE OR DELETE ON {table}
|
||||
FOR EACH ROW EXECUTE FUNCTION public.kf_append_only_maintenance()""")
|
||||
# Preserve the already-verified authority implementation; its only entry
|
||||
# point becomes a fixed audited wrapper owned by the same narrow NOLOGIN role.
|
||||
op.execute(
|
||||
"ALTER FUNCTION public.kf_manage_account_authority(text,text) "
|
||||
"RENAME TO kf_manage_account_authority_unlogged"
|
||||
)
|
||||
op.execute(
|
||||
"REVOKE ALL ON FUNCTION public.kf_manage_account_authority_unlogged(text,text) "
|
||||
"FROM PUBLIC,kf_oj_identity_operator"
|
||||
)
|
||||
op.execute(f"GRANT INSERT ON problem_maintenance_events TO {OWNER}")
|
||||
op.execute("""
|
||||
CREATE FUNCTION public.kf_manage_account_authority(p_username text,p_operation text)
|
||||
RETURNS TABLE(account_id uuid,username text,enabled boolean,is_administrator boolean,
|
||||
authorization_version bigint,changed boolean)
|
||||
LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,pg_temp AS $$
|
||||
DECLARE result record; BEGIN
|
||||
SELECT * INTO STRICT result FROM
|
||||
public.kf_manage_account_authority_unlogged(p_username,p_operation);
|
||||
IF result.changed THEN
|
||||
INSERT INTO public.problem_maintenance_events
|
||||
(id,created_at,actor_id,actor_kind,operator_name,authorization_version,
|
||||
operation_id,action,target_id)
|
||||
VALUES (gen_random_uuid(),clock_timestamp(),NULL,'operator',session_user,
|
||||
result.authorization_version,gen_random_uuid(),'authority_' || p_operation,
|
||||
result.account_id);
|
||||
END IF;
|
||||
RETURN QUERY SELECT result.account_id,result.username,result.enabled,
|
||||
result.is_administrator,result.authorization_version,result.changed;
|
||||
END $$""")
|
||||
op.execute(f"GRANT CREATE ON SCHEMA public TO {OWNER}")
|
||||
op.execute(f"ALTER FUNCTION public.kf_manage_account_authority(text,text) OWNER TO {OWNER}")
|
||||
op.execute(f"REVOKE CREATE ON SCHEMA public FROM {OWNER}")
|
||||
op.execute("REVOKE ALL ON FUNCTION public.kf_manage_account_authority(text,text) FROM PUBLIC")
|
||||
op.execute(
|
||||
"GRANT EXECUTE ON FUNCTION public.kf_manage_account_authority(text,text) "
|
||||
"TO kf_oj_identity_operator"
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
if (
|
||||
op.get_bind()
|
||||
.exec_driver_sql(
|
||||
"SELECT EXISTS(SELECT 1 FROM problem_maintenance_operations) OR "
|
||||
"EXISTS(SELECT 1 FROM problem_maintenance_events WHERE actor_kind='operator')"
|
||||
)
|
||||
.scalar()
|
||||
):
|
||||
raise RuntimeError("cannot discard committed maintenance history")
|
||||
op.execute("DROP FUNCTION public.kf_manage_account_authority(text,text)")
|
||||
op.execute(
|
||||
"ALTER FUNCTION public.kf_manage_account_authority_unlogged(text,text) "
|
||||
"RENAME TO kf_manage_account_authority"
|
||||
)
|
||||
op.execute(
|
||||
"GRANT EXECUTE ON FUNCTION public.kf_manage_account_authority(text,text) "
|
||||
"TO kf_oj_identity_operator"
|
||||
)
|
||||
op.execute(f"REVOKE INSERT ON problem_maintenance_events FROM {OWNER}")
|
||||
for table in ("problem_maintenance_operations", "problem_maintenance_events"):
|
||||
op.execute(f"DROP TRIGGER append_only_maintenance ON {table}")
|
||||
op.execute("DROP FUNCTION public.kf_append_only_maintenance()")
|
||||
ProblemMaintenanceOperation.__table__.drop(op.get_bind())
|
||||
op.drop_constraint(op.f("ck_problem_maintenance_events_actor"), "problem_maintenance_events")
|
||||
op.drop_constraint(op.f("ck_problem_maintenance_events_action"), "problem_maintenance_events")
|
||||
op.create_check_constraint(
|
||||
"action",
|
||||
"problem_maintenance_events",
|
||||
"action IN ('upload_created','upload_received','import_ready','import_failed',"
|
||||
"'import_discarded','import_expired','published','disabled','restored')",
|
||||
)
|
||||
op.alter_column("problem_maintenance_events", "actor_id", nullable=False)
|
||||
op.drop_column("problem_maintenance_events", "operator_name")
|
||||
op.drop_column("problem_maintenance_events", "actor_kind")
|
||||
@@ -22,6 +22,7 @@ dependencies = [
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
kf-problem-import-worker = "kf_oj.workers.problem_import:main"
|
||||
kf-accounts = "kf_oj.modules.identity.cli:main"
|
||||
kf-problems = "kf_oj.modules.problems.cli:main"
|
||||
kf-artifacts = "kf_oj.infra.artifacts.cli:main"
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Uniform live session gates; domains still lock/recheck authority in write transactions."""
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, HTTPException
|
||||
|
||||
from kf_oj.api.dependencies import current_principal, unsafe_principal
|
||||
from kf_oj.modules.identity.public import Principal
|
||||
|
||||
|
||||
async def maintenance_reader(
|
||||
principal: Annotated[Principal, Depends(current_principal)],
|
||||
) -> Principal:
|
||||
if not principal.can_maintain_problems:
|
||||
raise HTTPException(403, "problem maintenance required")
|
||||
return principal
|
||||
|
||||
|
||||
async def maintenance_writer(
|
||||
principal: Annotated[Principal, Depends(unsafe_principal)],
|
||||
) -> Principal:
|
||||
# unsafe_principal already checks an explicit Origin and the live session's
|
||||
# bound CSRF token. This capability check cannot replace the domain's port.
|
||||
if not principal.can_maintain_problems:
|
||||
raise HTTPException(403, "problem maintenance required")
|
||||
return principal
|
||||
@@ -6,7 +6,11 @@ from fastapi.exceptions import RequestValidationError
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from kf_oj import __version__
|
||||
from kf_oj.api.availability import AvailabilityMiddleware
|
||||
from kf_oj.api.body_limits import BodyLimitMiddleware
|
||||
from kf_oj.api.dependency_errors import install_handlers
|
||||
from kf_oj.api.v1.admin_imports import router as import_router
|
||||
from kf_oj.api.v1.admin_problems import router as maintenance_router
|
||||
from kf_oj.api.v1.identity import router as identity_router
|
||||
from kf_oj.api.v1.problems import router as problems_router
|
||||
from kf_oj.api.v1.router import router
|
||||
@@ -15,11 +19,17 @@ from kf_oj.api.v1.worker_artifacts import router as worker_router
|
||||
from kf_oj.core.logging import configure_logging
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
from kf_oj.infra.coordinator import CoordinatorControl
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
from kf_oj.modules.identity.service import IdentityService
|
||||
from kf_oj.modules.operations.queries import MaintenanceAuditQueries
|
||||
from kf_oj.modules.operations.service import MaintenanceAudit
|
||||
from kf_oj.modules.problems.imports import ProblemImports
|
||||
from kf_oj.modules.problems.maintenance import ProblemMaintenance
|
||||
from kf_oj.modules.problems.service import ProblemService
|
||||
from kf_oj.modules.submissions.performance import SubmissionPerformanceQueries
|
||||
from kf_oj.modules.submissions.queries import SubmissionQueries
|
||||
from kf_oj.modules.submissions.service import SubmissionService
|
||||
from kf_oj.modules.submissions.worker_access import WorkerAccess
|
||||
@@ -30,11 +40,12 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
configure_logging(settings.log_level)
|
||||
configure_logging(settings.log_level, production=settings.environment == "production")
|
||||
logger = logging.getLogger("kf_oj.api")
|
||||
logger.info("Platform API started")
|
||||
engine = None
|
||||
store = None
|
||||
staging = None
|
||||
try:
|
||||
if settings.database_url is not None:
|
||||
engine = build_engine(settings)
|
||||
@@ -42,6 +53,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
app.state.identity = IdentityService(
|
||||
app.state.sessions, lifetime_seconds=settings.session_lifetime_seconds
|
||||
)
|
||||
app.state.submission_performance = SubmissionPerformanceQueries(app.state.sessions)
|
||||
artifacts = None
|
||||
if settings.artifact_root is not None:
|
||||
store = ArtifactStore(
|
||||
@@ -50,6 +62,28 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
max_files=settings.artifact_max_files,
|
||||
)
|
||||
artifacts = ArtifactRepository(store)
|
||||
if (
|
||||
settings.import_staging_root is not None
|
||||
and settings.judge_admission is not None
|
||||
):
|
||||
staging = ImportStaging(settings.import_staging_root)
|
||||
app.state.problem_imports = ProblemImports(
|
||||
app.state.sessions,
|
||||
staging,
|
||||
app.state.identity,
|
||||
MaintenanceAudit(),
|
||||
settings.judge_admission,
|
||||
)
|
||||
app.state.problem_maintenance = ProblemMaintenance(
|
||||
app.state.sessions,
|
||||
app.state.identity,
|
||||
MaintenanceAudit(),
|
||||
artifacts,
|
||||
settings.judge_admission,
|
||||
)
|
||||
app.state.maintenance_audit = MaintenanceAuditQueries(
|
||||
app.state.sessions, app.state.identity
|
||||
)
|
||||
app.state.problems = ProblemService(
|
||||
app.state.sessions, artifacts, settings.judge_admission
|
||||
)
|
||||
@@ -77,10 +111,16 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
app.state.worker_access = None
|
||||
app.state.submissions = None
|
||||
app.state.submission_queries = None
|
||||
app.state.submission_performance = None
|
||||
app.state.problem_imports = None
|
||||
app.state.problem_maintenance = None
|
||||
app.state.maintenance_audit = None
|
||||
if engine is not None:
|
||||
await engine.dispose()
|
||||
if store is not None:
|
||||
store.close()
|
||||
if staging is not None:
|
||||
staging.close()
|
||||
logger.info("Platform API stopped")
|
||||
|
||||
app = FastAPI(
|
||||
@@ -94,11 +134,14 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
)
|
||||
app.state.settings = settings
|
||||
app.add_middleware(BodyLimitMiddleware)
|
||||
app.add_middleware(AvailabilityMiddleware, settings=settings)
|
||||
|
||||
@app.middleware("http")
|
||||
async def private_responses(request, call_next):
|
||||
response = await call_next(request)
|
||||
if request.url.path.startswith(("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal")):
|
||||
if request.url.path.startswith(
|
||||
("/api/v1/auth", "/api/v1/submissions", "/api/v1/internal", "/api/v1/admin")
|
||||
):
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
|
||||
@@ -106,9 +149,13 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
async def invalid_request(_, __):
|
||||
return JSONResponse({"detail": "invalid request"}, status_code=422)
|
||||
|
||||
install_handlers(app)
|
||||
|
||||
app.include_router(router, prefix="/api/v1")
|
||||
app.include_router(identity_router, prefix="/api/v1")
|
||||
app.include_router(problems_router, prefix="/api/v1")
|
||||
app.include_router(worker_router, prefix="/api/v1")
|
||||
app.include_router(submission_router, prefix="/api/v1")
|
||||
app.include_router(import_router, prefix="/api/v1")
|
||||
app.include_router(maintenance_router, prefix="/api/v1")
|
||||
return app
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Admission barrier while already accepted work retains its read/cancel paths."""
|
||||
|
||||
import re
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
from kf_oj.infra.availability import current
|
||||
|
||||
|
||||
def needs_open(method: str, path: str) -> bool:
|
||||
if method in {"GET", "HEAD", "OPTIONS"}:
|
||||
return False
|
||||
if method == "POST" and path == "/api/v1/auth/logout":
|
||||
return False
|
||||
if method == "POST" and re.fullmatch(r"/api/v1/submissions/[0-9a-f-]{36}/cancel", path):
|
||||
return False
|
||||
if method == "PUT" and re.fullmatch(
|
||||
r"/api/v1/admin/problem-imports/[0-9a-f-]{36}/archive", path
|
||||
):
|
||||
# The domain route still requires a previously authorized immutable
|
||||
# receipt. This cannot create a new upload or bypass its lease/epoch.
|
||||
return False
|
||||
return path.startswith("/api/v1/")
|
||||
|
||||
|
||||
class AvailabilityMiddleware:
|
||||
def __init__(self, app, *, settings):
|
||||
self.app, self.settings = app, settings
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if (
|
||||
scope["type"] == "http"
|
||||
and self.settings.environment == "production"
|
||||
and needs_open(scope["method"], scope.get("path", ""))
|
||||
):
|
||||
try:
|
||||
state = current(
|
||||
self.settings.delivery_release_id, self.settings.delivery_configuration_id
|
||||
)
|
||||
allowed = state.state == "open"
|
||||
except (OSError, ValueError, RecursionError):
|
||||
allowed = False
|
||||
if not allowed:
|
||||
await JSONResponse(
|
||||
{"detail": "platform mutations unavailable"},
|
||||
status_code=503,
|
||||
headers={"Cache-Control": "no-store", "Retry-After": "10"},
|
||||
)(scope, receive, send)
|
||||
return
|
||||
await self.app(scope, receive, send)
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Bound request bytes and upload time before schema parsing can allocate large inputs."""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
@@ -13,6 +14,14 @@ class BodyLimitMiddleware:
|
||||
if scope["type"] != "http" or scope["method"] in {"GET", "HEAD", "OPTIONS"}:
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
if scope["method"] == "PUT" and re.fullmatch(
|
||||
r"/api/v1/admin/problem-imports/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/archive",
|
||||
scope["path"],
|
||||
):
|
||||
# Only this exact endpoint can stream. Its live administrator gate runs
|
||||
# before receive(), then its service enforces actual bytes/chunks/deadline.
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
limit = 8192 if scope["path"] == "/api/v1/auth/login" else 1024 * 1024
|
||||
if scope["path"] == "/api/v1/submissions":
|
||||
# Max256KiB UTF-8 source, worst-case JSON escaping plus bounded metadata.
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Bounded, non-secret API responses for transient dependency/capacity failures."""
|
||||
|
||||
import errno
|
||||
|
||||
from asyncpg import CannotConnectNowError, PostgresConnectionError
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
|
||||
async def temporarily_unavailable(_, __):
|
||||
# Write outcomes may be uncertain. Existing operation/submission UUIDs, not
|
||||
# automatic server retries, own recovery from a lost response.
|
||||
return JSONResponse(
|
||||
{"detail": "service temporarily unavailable"},
|
||||
status_code=503,
|
||||
headers={"Cache-Control": "no-store", "Retry-After": "5"},
|
||||
)
|
||||
|
||||
|
||||
async def filesystem_failure(_, error: OSError):
|
||||
if error.errno in {errno.ENOSPC, errno.EDQUOT, errno.EMFILE, errno.ENFILE}:
|
||||
return await temporarily_unavailable(_, error)
|
||||
# Unrelated filesystem/programming failures retain server-error semantics.
|
||||
# Never serialize path/source/query/credential text from an exception.
|
||||
return JSONResponse(
|
||||
{"detail": "internal server error"}, status_code=500, headers={"Cache-Control": "no-store"}
|
||||
)
|
||||
|
||||
|
||||
def install_handlers(app: FastAPI) -> None:
|
||||
for error in (
|
||||
ConnectionError,
|
||||
TimeoutError,
|
||||
OperationalError,
|
||||
PoolTimeoutError,
|
||||
PostgresConnectionError,
|
||||
CannotConnectNowError,
|
||||
):
|
||||
app.add_exception_handler(error, temporarily_unavailable)
|
||||
app.add_exception_handler(OSError, filesystem_failure)
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Administrator import contract; configured service availability is explicit."""
|
||||
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from kf_oj.api.admin_dependencies import maintenance_reader, maintenance_writer
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
CreateProblemImport,
|
||||
ProblemImportCapabilities,
|
||||
ProblemImportDetail,
|
||||
ProblemImportPage,
|
||||
ProblemImportSummary,
|
||||
)
|
||||
from kf_oj.infra.artifacts.store import ArtifactError
|
||||
from kf_oj.modules.identity.public import MaintenanceDenied, Principal
|
||||
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
|
||||
|
||||
router = APIRouter(prefix="/admin/problem-imports", tags=["problem-maintenance"])
|
||||
|
||||
|
||||
@router.get("/capabilities", response_model=ProblemImportCapabilities)
|
||||
async def capabilities(
|
||||
request: Request, _: Annotated[Principal, Depends(maintenance_reader)]
|
||||
) -> ProblemImportCapabilities:
|
||||
service = getattr(request.app.state, "problem_imports", None)
|
||||
if service is None:
|
||||
return ProblemImportCapabilities(uploads_available=False, validation_available=False)
|
||||
return await service.capabilities()
|
||||
|
||||
|
||||
def service(request):
|
||||
result = getattr(request.app.state, "problem_imports", None)
|
||||
if result is None:
|
||||
raise HTTPException(503, "problem imports unavailable")
|
||||
return result
|
||||
|
||||
|
||||
async def operation(awaitable):
|
||||
try:
|
||||
return await awaitable
|
||||
except MaintenanceDenied:
|
||||
raise HTTPException(403, "problem maintenance required") from None
|
||||
except ImportConflict:
|
||||
raise HTTPException(409, "import request conflict") from None
|
||||
except ImportMissing:
|
||||
raise HTTPException(404, "import unavailable") from None
|
||||
except TimeoutError:
|
||||
raise HTTPException(408, "upload deadline exceeded") from None
|
||||
except (ImportUnavailable, ArtifactError, OSError, DBAPIError):
|
||||
raise HTTPException(503, "problem imports unavailable") from None
|
||||
except ClientDisconnect:
|
||||
raise HTTPException(400, "upload interrupted") from None
|
||||
|
||||
|
||||
@router.post("", status_code=201, response_model=ProblemImportSummary)
|
||||
async def create(
|
||||
body: CreateProblemImport,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||
):
|
||||
if request.headers.get("content-type", "").split(";", 1)[0] != "application/json":
|
||||
raise HTTPException(415, "JSON request required")
|
||||
return await operation(service(request).create(principal, body))
|
||||
|
||||
|
||||
@router.get("", response_model=ProblemImportPage)
|
||||
async def page(
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||
after: UUID | None = None,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
):
|
||||
return await operation(service(request).list(principal, after, limit))
|
||||
|
||||
|
||||
@router.get("/{operation_id}", response_model=ProblemImportDetail)
|
||||
async def detail(
|
||||
operation_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||
):
|
||||
return await operation(service(request).detail(principal, operation_id))
|
||||
|
||||
|
||||
@router.put(
|
||||
"/{operation_id}/archive",
|
||||
response_model=ProblemImportSummary,
|
||||
openapi_extra={
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {"application/zip": {"schema": {"type": "string", "format": "binary"}}},
|
||||
}
|
||||
},
|
||||
)
|
||||
async def upload(
|
||||
operation_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||
):
|
||||
if request.url.path != f"/api/v1/admin/problem-imports/{operation_id}/archive":
|
||||
raise HTTPException(422, "canonical operation UUID required")
|
||||
if request.headers.get("content-type", "").split(";", 1)[0] != "application/zip":
|
||||
raise HTTPException(415, "ZIP request required")
|
||||
if request.headers.get("content-encoding") not in (None, "identity"):
|
||||
raise HTTPException(415, "encoded upload unavailable")
|
||||
length = request.headers.get("content-length")
|
||||
if length is not None:
|
||||
try:
|
||||
if not length.isascii() or not length.isdecimal() or len(length) > 12:
|
||||
raise ValueError
|
||||
length = int(length)
|
||||
except ValueError:
|
||||
raise HTTPException(413, "invalid archive length") from None
|
||||
|
||||
async def chunks():
|
||||
while True:
|
||||
message = await request.receive()
|
||||
if message["type"] == "http.disconnect":
|
||||
raise ClientDisconnect()
|
||||
if message["type"] != "http.request":
|
||||
raise ImportConflict("invalid upload frame")
|
||||
yield message.get("body", b"")
|
||||
if not message.get("more_body", False):
|
||||
return
|
||||
|
||||
return await operation(service(request).upload(principal, operation_id, chunks(), length))
|
||||
|
||||
|
||||
@router.post("/{operation_id}/discard", response_model=ProblemImportSummary)
|
||||
async def discard(
|
||||
operation_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||
):
|
||||
return await operation(service(request).discard(principal, operation_id))
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Live administrator projections and actor-bound immutable mutations."""
|
||||
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
|
||||
from kf_oj.api.admin_dependencies import maintenance_reader, maintenance_writer
|
||||
from kf_oj.contracts.problem_maintenance import (
|
||||
AuditAction,
|
||||
ChangeProblemAvailability,
|
||||
MaintainedProblem,
|
||||
MaintainedProblemPage,
|
||||
MaintainedVersionPage,
|
||||
MaintenanceAuditPage,
|
||||
MaintenanceReceipt,
|
||||
PublishCandidate,
|
||||
)
|
||||
from kf_oj.infra.artifacts.store import ArtifactError
|
||||
from kf_oj.modules.identity.public import MaintenanceDenied, Principal
|
||||
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
|
||||
from kf_oj.modules.problems.public import ProblemUnavailable
|
||||
|
||||
router = APIRouter(prefix="/admin", tags=["problem-maintenance"])
|
||||
|
||||
|
||||
def service(request, name="problem_maintenance"):
|
||||
result = getattr(request.app.state, name, None)
|
||||
if result is None:
|
||||
raise HTTPException(503, "problem maintenance unavailable")
|
||||
return result
|
||||
|
||||
|
||||
async def operation(awaitable):
|
||||
try:
|
||||
return await awaitable
|
||||
except MaintenanceDenied:
|
||||
raise HTTPException(403, "problem maintenance required") from None
|
||||
except ImportConflict:
|
||||
raise HTTPException(409, "maintenance precondition conflict") from None
|
||||
except ImportMissing:
|
||||
raise HTTPException(404, "maintenance object unavailable") from None
|
||||
except (ImportUnavailable, ArtifactError, ProblemUnavailable, OSError, DBAPIError):
|
||||
raise HTTPException(503, "problem maintenance unavailable") from None
|
||||
except ValueError:
|
||||
raise HTTPException(422, "invalid maintenance request") from None
|
||||
|
||||
|
||||
@router.get("/problems", response_model=MaintainedProblemPage)
|
||||
async def page(
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||
after: Annotated[str | None, Query(pattern=r"^[a-z0-9][a-z0-9_.-]{0,63}$")] = None,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
enabled: bool | None = None,
|
||||
):
|
||||
return await operation(
|
||||
service(request).page(principal, after=after, limit=limit, enabled=enabled)
|
||||
)
|
||||
|
||||
|
||||
@router.get("/problem-slugs/{slug}", response_model=MaintainedProblem)
|
||||
async def by_slug(
|
||||
slug: str, request: Request, principal: Annotated[Principal, Depends(maintenance_reader)]
|
||||
):
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from kf_oj.contracts.judging import Label
|
||||
|
||||
try:
|
||||
TypeAdapter(Label).validate_python(slug)
|
||||
except ValueError:
|
||||
raise HTTPException(422, "invalid problem slug") from None
|
||||
return await operation(service(request).by_slug(principal, slug))
|
||||
|
||||
|
||||
@router.get("/problems/{problem_id}", response_model=MaintainedProblem)
|
||||
async def detail(
|
||||
problem_id: UUID, request: Request, principal: Annotated[Principal, Depends(maintenance_reader)]
|
||||
):
|
||||
return await operation(service(request).detail(principal, problem_id))
|
||||
|
||||
|
||||
@router.get("/problems/{problem_id}/versions", response_model=MaintainedVersionPage)
|
||||
async def versions(
|
||||
problem_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||
before: Annotated[int | None, Query(ge=1)] = None,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
):
|
||||
return await operation(
|
||||
service(request).versions(principal, problem_id, before=before, limit=limit)
|
||||
)
|
||||
|
||||
|
||||
@router.get("/problem-maintenance/{operation_id}", response_model=MaintenanceReceipt)
|
||||
async def receipt(
|
||||
operation_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||
):
|
||||
return await operation(service(request).receipt(principal, operation_id))
|
||||
|
||||
|
||||
@router.post("/problem-publications", response_model=MaintenanceReceipt, status_code=201)
|
||||
async def publish(
|
||||
body: PublishCandidate,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||
):
|
||||
return await operation(service(request).publish(principal, body))
|
||||
|
||||
|
||||
@router.post("/problems/{problem_id}/disable", response_model=MaintenanceReceipt)
|
||||
async def disable(
|
||||
problem_id: UUID,
|
||||
body: ChangeProblemAvailability,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||
):
|
||||
return await operation(
|
||||
service(request).availability(principal, problem_id, body, enabled=False)
|
||||
)
|
||||
|
||||
|
||||
@router.post("/problems/{problem_id}/restore", response_model=MaintenanceReceipt)
|
||||
async def restore(
|
||||
problem_id: UUID,
|
||||
body: ChangeProblemAvailability,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_writer)],
|
||||
):
|
||||
return await operation(service(request).availability(principal, problem_id, body, enabled=True))
|
||||
|
||||
|
||||
@router.get("/maintenance-audit", response_model=MaintenanceAuditPage)
|
||||
async def audit(
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(maintenance_reader)],
|
||||
before: Annotated[str | None, Query(max_length=100)] = None,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
action: AuditAction | None = None,
|
||||
):
|
||||
return await operation(
|
||||
service(request, "maintenance_audit").page(
|
||||
principal, before=before, limit=limit, action=action
|
||||
)
|
||||
)
|
||||
@@ -9,13 +9,24 @@ from kf_oj.api.dependencies import (
|
||||
trusted_origin,
|
||||
unsafe_principal,
|
||||
)
|
||||
from kf_oj.contracts.identity import CurrentUser, LoginRequest, LoginResponse
|
||||
from kf_oj.contracts.identity import AccountCapabilities, CurrentUser, LoginRequest, LoginResponse
|
||||
from kf_oj.modules.identity.public import AuthenticationFailed, LoginThrottled, Principal
|
||||
from kf_oj.modules.identity.service import IdentityService
|
||||
|
||||
router = APIRouter(prefix="/auth", tags=["identity"])
|
||||
|
||||
|
||||
def current_user(principal: Principal) -> CurrentUser:
|
||||
return CurrentUser(
|
||||
id=principal.account_id,
|
||||
username=principal.username,
|
||||
capabilities=AccountCapabilities(
|
||||
problem_maintenance=principal.can_maintain_problems,
|
||||
authorization_version=str(principal.authorization_version),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/login", response_model=LoginResponse, dependencies=[Depends(trusted_origin)])
|
||||
async def login(
|
||||
body: LoginRequest,
|
||||
@@ -58,7 +69,7 @@ async def login(
|
||||
)
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return LoginResponse(
|
||||
user=CurrentUser(id=issued.principal.account_id, username=issued.principal.username),
|
||||
user=current_user(issued.principal),
|
||||
expires_at=issued.expires_at,
|
||||
csrf_token=issued.csrf_token,
|
||||
)
|
||||
@@ -67,7 +78,7 @@ async def login(
|
||||
@router.get("/me", response_model=CurrentUser)
|
||||
async def me(response: Response, principal: Annotated[Principal, Depends(current_principal)]):
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return CurrentUser(id=principal.account_id, username=principal.username)
|
||||
return current_user(principal)
|
||||
|
||||
|
||||
@router.post("/logout", status_code=204, dependencies=[Depends(unsafe_principal)])
|
||||
|
||||
@@ -4,6 +4,8 @@ from uuid import UUID
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
|
||||
|
||||
from kf_oj.api.dependencies import current_principal, unsafe_principal
|
||||
from kf_oj.contracts.judging_v2 import Language
|
||||
from kf_oj.contracts.submission_performance import SubmissionPerformance
|
||||
from kf_oj.contracts.submissions import (
|
||||
CancellationAccepted,
|
||||
CreateSubmission,
|
||||
@@ -65,9 +67,11 @@ async def history(
|
||||
principal: Annotated[Principal, Depends(current_principal)],
|
||||
after: Annotated[str | None, Query(max_length=128)] = None,
|
||||
limit: Annotated[int, Query(ge=1, le=100)] = 20,
|
||||
problem_version_id: UUID | None = None,
|
||||
language: Language | None = None,
|
||||
):
|
||||
try:
|
||||
return await queries(request).page(principal, after, limit)
|
||||
return await queries(request).page(principal, after, limit, problem_version_id, language)
|
||||
except ValueError:
|
||||
raise HTTPException(422, "invalid history request") from None
|
||||
|
||||
@@ -86,6 +90,23 @@ async def detail(
|
||||
raise HTTPException(503, "stored result unavailable") from None
|
||||
|
||||
|
||||
@router.get("/{submission_id}/performance", response_model=SubmissionPerformance)
|
||||
async def performance(
|
||||
submission_id: UUID,
|
||||
request: Request,
|
||||
principal: Annotated[Principal, Depends(current_principal)],
|
||||
):
|
||||
service = getattr(request.app.state, "submission_performance", None)
|
||||
if service is None:
|
||||
raise HTTPException(503, "performance service unavailable")
|
||||
try:
|
||||
return await service.detail(principal, submission_id)
|
||||
except SubmissionUnavailable:
|
||||
raise HTTPException(404, "submission unavailable") from None
|
||||
except ValueError:
|
||||
raise HTTPException(503, "performance unavailable") from None
|
||||
|
||||
|
||||
@router.get("/{submission_id}/source", response_model=SubmissionSource)
|
||||
async def source(
|
||||
submission_id: UUID,
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Versioned problem-import DTOs; hidden bytes, paths and toolchain selection stay private."""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import Field, StrictBool, model_validator
|
||||
|
||||
from kf_oj.contracts.judging import Digest, Identifier, Label, RunLimits, Version, WireModel
|
||||
from kf_oj.contracts.judging_v2 import Language
|
||||
from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample
|
||||
|
||||
MAX_ARCHIVE_BYTES = 64 * 1024**2
|
||||
MAX_UNPACKED_BYTES = 64 * 1024**2
|
||||
MAX_ARCHIVE_FILES = 534
|
||||
MAX_ARCHIVE_NAME_BYTES = 128
|
||||
MAX_UPLOAD_CHUNKS = 4096
|
||||
MAX_UPLOAD_CHUNK_BYTES = 1024**2
|
||||
UPLOAD_DEADLINE_SECONDS = 30
|
||||
VALIDATION_DEADLINE_SECONDS = 60
|
||||
MAX_IMPORT_ATTEMPTS = 2
|
||||
IMPORT_LIFETIME_SECONDS = 86400
|
||||
MAX_OWNER_CANDIDATES = 10
|
||||
MAX_TOTAL_CANDIDATES = 32
|
||||
STAGING_MAX_BYTES = 256 * 1024**2
|
||||
STAGING_RESERVATION_BYTES = MAX_ARCHIVE_BYTES + MAX_UNPACKED_BYTES
|
||||
|
||||
|
||||
class ImportState(StrEnum):
|
||||
UPLOADING = "uploading"
|
||||
UPLOADED = "uploaded"
|
||||
VALIDATING = "validating"
|
||||
READY = "ready"
|
||||
FAILED = "failed"
|
||||
PUBLISHED = "published"
|
||||
DISCARDED = "discarded"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
class ImportFailure(StrEnum):
|
||||
INVALID_ARCHIVE = "invalid_archive"
|
||||
INVALID_MANIFEST = "invalid_manifest"
|
||||
INVALID_DATA = "invalid_data"
|
||||
DIGEST_MISMATCH = "digest_mismatch"
|
||||
POLICY_REJECTED = "policy_rejected"
|
||||
UPLOAD_INTERRUPTED = "upload_interrupted"
|
||||
UPLOAD_DEADLINE = "upload_deadline"
|
||||
VALIDATION_DEADLINE = "validation_deadline"
|
||||
VALIDATION_UNAVAILABLE = "validation_unavailable"
|
||||
STORAGE_UNAVAILABLE = "storage_unavailable"
|
||||
AUTHORITY_REVOKED = "authority_revoked"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
class ProblemImportLimits(WireModel):
|
||||
schema_version: Version = 1
|
||||
archive_format: Literal["single_level_zip"] = "single_level_zip"
|
||||
manifest_name: Literal["problem.json"] = "problem.json"
|
||||
manifest_schema_versions: tuple[Literal[1, 2], ...] = (1, 2)
|
||||
languages: tuple[Language, ...] = tuple(Language)
|
||||
archive_bytes: int = MAX_ARCHIVE_BYTES
|
||||
unpacked_bytes: int = MAX_UNPACKED_BYTES
|
||||
files: int = MAX_ARCHIVE_FILES
|
||||
filename_ascii_bytes: int = MAX_ARCHIVE_NAME_BYTES
|
||||
upload_chunks: int = MAX_UPLOAD_CHUNKS
|
||||
chunk_bytes: int = MAX_UPLOAD_CHUNK_BYTES
|
||||
upload_deadline_seconds: int = UPLOAD_DEADLINE_SECONDS
|
||||
validation_deadline_seconds: int = VALIDATION_DEADLINE_SECONDS
|
||||
global_concurrency: Literal[1] = 1
|
||||
owner_concurrency: Literal[1] = 1
|
||||
owner_candidates: int = MAX_OWNER_CANDIDATES
|
||||
total_candidates: int = MAX_TOTAL_CANDIDATES
|
||||
staging_bytes: int = STAGING_MAX_BYTES
|
||||
candidate_lifetime_seconds: int = IMPORT_LIFETIME_SECONDS
|
||||
|
||||
|
||||
class ProblemImportCapabilities(WireModel):
|
||||
schema_version: Version = 1
|
||||
limits: ProblemImportLimits = ProblemImportLimits()
|
||||
uploads_available: StrictBool
|
||||
validation_available: StrictBool
|
||||
|
||||
|
||||
class CreateProblemImport(WireModel):
|
||||
schema_version: Version = 1
|
||||
operation_id: Identifier
|
||||
archive_digest: Digest
|
||||
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
|
||||
|
||||
|
||||
class LegacyImportBudget(WireModel):
|
||||
compile_limits: RunLimits
|
||||
run_limits: RunLimits
|
||||
job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=3600000000)]
|
||||
|
||||
|
||||
class CandidatePreview(WireModel):
|
||||
candidate_id: Identifier
|
||||
slug: Label
|
||||
title: Annotated[str, Field(strict=True, min_length=1, max_length=160)]
|
||||
statement: Annotated[str, Field(strict=True, min_length=1, max_length=262144)]
|
||||
samples: Annotated[tuple[PublicSample, ...], Field(max_length=10)]
|
||||
judging_schema_version: Annotated[int, Field(strict=True, ge=1, le=2)]
|
||||
languages: Annotated[tuple[Language, ...], Field(min_length=1, max_length=4)]
|
||||
legacy_budget: LegacyImportBudget | None
|
||||
language_configurations: Annotated[
|
||||
tuple[PublicLanguageConfiguration, ...], Field(max_length=4)
|
||||
] = ()
|
||||
case_count: Annotated[int, Field(strict=True, ge=1, le=256)]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def bounded_text(self):
|
||||
if len(self.statement.encode("utf-8")) > 262144 or "\x00" in self.statement + self.title:
|
||||
raise ValueError("invalid public text")
|
||||
languages = [c.language for c in self.language_configurations]
|
||||
if len(set(languages)) != len(languages):
|
||||
raise ValueError("duplicate language configuration")
|
||||
if len(set(self.languages)) != len(self.languages):
|
||||
raise ValueError("duplicate language")
|
||||
if self.judging_schema_version == 1:
|
||||
if (
|
||||
self.language_configurations
|
||||
or self.legacy_budget is None
|
||||
or any(
|
||||
language not in (Language.C17, Language.CPP20) for language in self.languages
|
||||
)
|
||||
):
|
||||
raise ValueError("invalid legacy candidate")
|
||||
elif tuple(languages) != self.languages or self.legacy_budget is not None:
|
||||
raise ValueError("invalid multi-runtime candidate")
|
||||
return self
|
||||
|
||||
|
||||
class ProblemImportSummary(WireModel):
|
||||
schema_version: Version = 1
|
||||
operation_id: Identifier
|
||||
state: ImportState
|
||||
expected_archive_digest: Digest
|
||||
archive_bytes: Annotated[int, Field(strict=True, ge=1, le=MAX_ARCHIVE_BYTES)]
|
||||
created_at: datetime
|
||||
expires_at: datetime
|
||||
updated_at: datetime
|
||||
failure_code: ImportFailure | None
|
||||
candidate_id: Identifier | None
|
||||
published_version_id: Identifier | None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def consistent_state(self):
|
||||
if self.expires_at <= self.created_at or self.updated_at < self.created_at:
|
||||
raise ValueError("invalid import time range")
|
||||
if (self.state in (ImportState.READY, ImportState.PUBLISHED)) != (
|
||||
self.candidate_id is not None
|
||||
):
|
||||
raise ValueError("candidate state mismatch")
|
||||
if (self.state == ImportState.PUBLISHED) != (self.published_version_id is not None):
|
||||
raise ValueError("published state mismatch")
|
||||
if self.state in (ImportState.FAILED, ImportState.EXPIRED):
|
||||
if self.failure_code is None:
|
||||
raise ValueError("failure code required")
|
||||
elif self.failure_code is not None:
|
||||
raise ValueError("unexpected failure code")
|
||||
return self
|
||||
|
||||
|
||||
class ProblemImportDetail(ProblemImportSummary):
|
||||
candidate: CandidatePreview | None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def candidate_matches(self):
|
||||
if (self.candidate_id is None) != (self.candidate is None):
|
||||
raise ValueError("candidate preview missing")
|
||||
if self.candidate and self.candidate.candidate_id != self.candidate_id:
|
||||
raise ValueError("candidate preview mismatch")
|
||||
return self
|
||||
|
||||
|
||||
class ProblemImportPage(WireModel):
|
||||
schema_version: Version = 1
|
||||
items: Annotated[tuple[ProblemImportSummary, ...], Field(max_length=100)]
|
||||
next_after: Identifier | None
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Root-to-API availability binding; independent from immutable judging protocols."""
|
||||
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
|
||||
|
||||
from kf_oj.contracts.judging import integer_version
|
||||
|
||||
AVAILABILITY_PATH = "/etc/kf-admission/status.json"
|
||||
|
||||
|
||||
class AvailabilityRecord(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||
release_id: Annotated[str, Field(pattern=r"^sd-[0-9a-f]{12}$")]
|
||||
configuration_id: Annotated[str, Field(pattern=r"^cfg-[0-9a-f]{24}$")]
|
||||
state: Literal["open", "draining", "quiescent"]
|
||||
@@ -5,8 +5,23 @@ import json
|
||||
from pathlib import Path
|
||||
|
||||
from kf_oj.api.app import create_app
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
CreateProblemImport,
|
||||
ProblemImportCapabilities,
|
||||
ProblemImportDetail,
|
||||
)
|
||||
from kf_oj.contracts.availability import AvailabilityRecord
|
||||
from kf_oj.contracts.judge import JudgeTaskNotification
|
||||
from kf_oj.contracts.judging import WIRE_MODELS
|
||||
from kf_oj.contracts.judging_v2 import WIRE_MODELS as WIRE_MODELS_V2
|
||||
from kf_oj.contracts.problem_maintenance import (
|
||||
ChangeProblemAvailability,
|
||||
MaintenanceAuditPage,
|
||||
MaintenanceReceipt,
|
||||
PublishCandidate,
|
||||
)
|
||||
from kf_oj.contracts.restore_io import RestoreIOReply, RestoreIORequest
|
||||
from kf_oj.contracts.submission_performance import SubmissionPerformance
|
||||
from kf_oj.core.settings import Settings
|
||||
|
||||
|
||||
@@ -33,6 +48,39 @@ def main() -> None:
|
||||
"budgets, result consistency, UUID canonical form and UTF-8 byte limits."
|
||||
)
|
||||
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
|
||||
performance = SubmissionPerformance.model_json_schema()
|
||||
performance["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||
performance["$id"] = "urn:kf-oj:submission-performance:v1"
|
||||
performance["$comment"] = (
|
||||
"Anonymous bounded aggregates, decimal raw units and strict same-snapshot "
|
||||
"counts; also run model validation and server-side ownership/environment checks."
|
||||
)
|
||||
outputs[root / "contracts/schemas/submission-performance.v1.schema.json"] = performance
|
||||
for name, model in {
|
||||
"delivery-availability": AvailabilityRecord,
|
||||
"restore-io-request": RestoreIORequest,
|
||||
"restore-io-reply": RestoreIOReply,
|
||||
"problem-publish": PublishCandidate,
|
||||
"problem-availability": ChangeProblemAvailability,
|
||||
"problem-maintenance": MaintenanceReceipt,
|
||||
"problem-maintenance-audit": MaintenanceAuditPage,
|
||||
"problem-import-create": CreateProblemImport,
|
||||
"problem-import": ProblemImportDetail,
|
||||
"problem-import-capabilities": ProblemImportCapabilities,
|
||||
}.items():
|
||||
document = model.model_json_schema()
|
||||
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||
document["$id"] = f"urn:kf-oj:{name}:v1"
|
||||
outputs[root / f"contracts/schemas/{name}.v1.schema.json"] = document
|
||||
for name, model in WIRE_MODELS_V2.items():
|
||||
document = model.model_json_schema()
|
||||
document["$schema"] = "https://json-schema.org/draft/2020-12/schema"
|
||||
document["$id"] = f"urn:kf-oj:{name}:v2"
|
||||
document["$comment"] = (
|
||||
"Multi-runtime protocol. Also validate runtime/source/attempt bindings, startup "
|
||||
"and node budgets, canonical bounded class sets and result consistency."
|
||||
)
|
||||
outputs[root / f"contracts/schemas/{name}.v2.schema.json"] = document
|
||||
stale = []
|
||||
for path, document in outputs.items():
|
||||
content = json.dumps(document, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from datetime import datetime
|
||||
from typing import Annotated
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import Field, SecretStr
|
||||
|
||||
@@ -11,9 +11,16 @@ class LoginRequest(WireModel):
|
||||
password: Annotated[SecretStr, Field(min_length=12, max_length=128)]
|
||||
|
||||
|
||||
class AccountCapabilities(WireModel):
|
||||
schema_version: Literal[1] = 1
|
||||
problem_maintenance: bool = False
|
||||
authorization_version: Annotated[str, Field(pattern=r"^[1-9][0-9]{0,18}$")] = "1"
|
||||
|
||||
|
||||
class CurrentUser(WireModel):
|
||||
id: Identifier
|
||||
username: str
|
||||
capabilities: AccountCapabilities = Field(default_factory=AccountCapabilities)
|
||||
|
||||
|
||||
class LoginResponse(WireModel):
|
||||
|
||||
@@ -375,7 +375,7 @@ WIRE_MODELS = {
|
||||
}
|
||||
|
||||
|
||||
def decode_wire[T: WireModel](model: type[T], payload: bytes) -> T:
|
||||
def decode_wire[T: BaseModel](model: type[T], payload: bytes) -> T:
|
||||
"""UTF-8, bounded size and no ambiguous duplicate members before model validation."""
|
||||
if len(payload) > MAX_WIRE_BYTES:
|
||||
raise ValueError("wire payload exceeds byte budget")
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
"""Explicit multi-runtime protocol; existing v1 snapshots remain byte-for-byte stable."""
|
||||
|
||||
import re
|
||||
from enum import StrEnum
|
||||
from typing import Annotated, Literal, Self
|
||||
|
||||
from pydantic import BeforeValidator, Field, model_validator
|
||||
|
||||
from kf_oj.contracts import judging as v1
|
||||
|
||||
Version = Annotated[Literal[2], BeforeValidator(v1.integer_version)]
|
||||
MAX_CLASS_FILES = 256
|
||||
JVM_NATIVE_RESERVE_BYTES = 160 * v1.MIB
|
||||
JVM_THREAD_STACK_BYTES = 256 * 1024
|
||||
|
||||
|
||||
class Language(StrEnum):
|
||||
C17 = "c17"
|
||||
CPP20 = "cpp20"
|
||||
PYTHON313 = "python313"
|
||||
JAVA21 = "java21"
|
||||
|
||||
|
||||
class PreparationKind(StrEnum):
|
||||
COMPILE = "compile"
|
||||
SYNTAX_CHECK = "syntax_check"
|
||||
|
||||
|
||||
class ProgramKind(StrEnum):
|
||||
NATIVE = "native_executable"
|
||||
PYTHON = "python_source"
|
||||
JAVA = "java_classes"
|
||||
|
||||
|
||||
class RuntimeIdentity(v1.WireModel):
|
||||
"""An exact identity matched against trusted configuration, never a host path."""
|
||||
|
||||
toolchain_id: v1.Label
|
||||
image_kind: Literal[
|
||||
"linux_gnu_cxx_x86_64_v1", "linux_cpython313_x86_64_v1", "linux_openjdk21_x86_64_v1"
|
||||
]
|
||||
image_revision: Annotated[int, Field(strict=True, ge=1, le=(1 << 32) - 1)]
|
||||
image_digest: v1.Digest
|
||||
runtime_version: Annotated[
|
||||
str, Field(strict=True, pattern=r"^[A-Za-z0-9][A-Za-z0-9_.+-]{0,63}$")
|
||||
]
|
||||
|
||||
|
||||
def runtime_contract(language: Language) -> tuple[str, str, str, ProgramKind, PreparationKind]:
|
||||
"""Fixed source/profile names. Runtime callers cannot introduce argv or paths."""
|
||||
return {
|
||||
Language.C17: (
|
||||
"main.c",
|
||||
"compile_c",
|
||||
"native",
|
||||
ProgramKind.NATIVE,
|
||||
PreparationKind.COMPILE,
|
||||
),
|
||||
Language.CPP20: (
|
||||
"main.cpp",
|
||||
"compile_cpp",
|
||||
"native",
|
||||
ProgramKind.NATIVE,
|
||||
PreparationKind.COMPILE,
|
||||
),
|
||||
Language.PYTHON313: (
|
||||
"main.py",
|
||||
"python_check",
|
||||
"python_run",
|
||||
ProgramKind.PYTHON,
|
||||
PreparationKind.SYNTAX_CHECK,
|
||||
),
|
||||
Language.JAVA21: (
|
||||
"Main.java",
|
||||
"java_compile",
|
||||
"java_run",
|
||||
ProgramKind.JAVA,
|
||||
PreparationKind.COMPILE,
|
||||
),
|
||||
}[language]
|
||||
|
||||
|
||||
def validate_runtime(language: Language, identity: RuntimeIdentity) -> None:
|
||||
expected = {
|
||||
Language.C17: ("gnu-cxx-v1", "linux_gnu_cxx_x86_64_v1"),
|
||||
Language.CPP20: ("gnu-cxx-v1", "linux_gnu_cxx_x86_64_v1"),
|
||||
Language.PYTHON313: ("cpython313-v1", "linux_cpython313_x86_64_v1"),
|
||||
Language.JAVA21: ("openjdk21-v1", "linux_openjdk21_x86_64_v1"),
|
||||
}[language]
|
||||
if (identity.toolchain_id, identity.image_kind) != expected:
|
||||
raise ValueError("runtime family differs from language contract")
|
||||
if language == Language.PYTHON313 and identity.runtime_version != "3.13.16":
|
||||
raise ValueError("unsupported CPython version")
|
||||
if language == Language.JAVA21 and not re.fullmatch(
|
||||
r"21\.[0-9]+\.[0-9]+(?:\.[0-9]+)?\+[1-9][0-9]*(?:-LTS)?", identity.runtime_version
|
||||
):
|
||||
raise ValueError("JDK version must identify a fixed Java 21 build")
|
||||
|
||||
|
||||
def validate_startup_budget(language: Language, limits: v1.RunLimits, *, preparing: bool) -> None:
|
||||
"""Admission floors, not multipliers on problem limits; real execution still required."""
|
||||
if language == Language.PYTHON313:
|
||||
floor = (10_000, 50_000, 32 * v1.MIB, 1, 16)
|
||||
elif language == Language.JAVA21:
|
||||
floor = (
|
||||
(1_000_000, 3_000_000, 384 * v1.MIB, 16, 32)
|
||||
if preparing
|
||||
else (200_000, 1_000_000, 256 * v1.MIB, 16, 32)
|
||||
)
|
||||
else:
|
||||
return
|
||||
actual = (
|
||||
limits.cpu_time_us,
|
||||
limits.wall_time_us,
|
||||
limits.memory_bytes,
|
||||
limits.processes,
|
||||
limits.open_files,
|
||||
)
|
||||
if any(value < minimum for value, minimum in zip(actual, floor, strict=True)):
|
||||
raise ValueError("budget below runtime startup policy")
|
||||
if language == Language.JAVA21:
|
||||
jvm_heap_bytes(limits)
|
||||
|
||||
|
||||
def jvm_heap_bytes(limits: v1.RunLimits) -> int:
|
||||
"""Reserve capped metaspace/code/native and every allowed thread stack inside cgroup."""
|
||||
reserve = JVM_NATIVE_RESERVE_BYTES + limits.processes * JVM_THREAD_STACK_BYTES
|
||||
heap = (limits.memory_bytes - reserve) // v1.MIB * v1.MIB
|
||||
if heap < 32 * v1.MIB or limits.stack_bytes < JVM_THREAD_STACK_BYTES:
|
||||
raise ValueError("budget cannot cover JVM heap and explicit non-heap reserve")
|
||||
return heap
|
||||
|
||||
|
||||
class LanguageConfiguration(v1.WireModel):
|
||||
language: Language
|
||||
toolchain: RuntimeIdentity
|
||||
compile_limits: v1.RunLimits
|
||||
run_limits: v1.RunLimits
|
||||
job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=v1.MAX_JOB_WALL_US)]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def runtime_and_budget(self) -> Self:
|
||||
validate_runtime(self.language, self.toolchain)
|
||||
validate_startup_budget(self.language, self.compile_limits, preparing=True)
|
||||
validate_startup_budget(self.language, self.run_limits, preparing=False)
|
||||
if self.compile_limits.wall_time_us + self.run_limits.wall_time_us > self.job_wall_time_us:
|
||||
raise ValueError("job deadline cannot cover preparation and execution")
|
||||
return self
|
||||
|
||||
|
||||
class JobManifest(v1.JobManifest):
|
||||
schema_version: Version
|
||||
language: Language
|
||||
toolchain: RuntimeIdentity
|
||||
|
||||
@model_validator(mode="after")
|
||||
def runtime_and_budget(self) -> Self:
|
||||
validate_runtime(self.language, self.toolchain)
|
||||
validate_startup_budget(self.language, self.compile_limits, preparing=True)
|
||||
validate_startup_budget(self.language, self.run_limits, preparing=False)
|
||||
return self
|
||||
|
||||
|
||||
class ProgramFile(v1.WireModel):
|
||||
path: Annotated[str, Field(strict=True, min_length=1, max_length=128)]
|
||||
artifact: v1.ArtifactReference
|
||||
|
||||
|
||||
class CompiledProgram(v1.WireModel):
|
||||
"""Sealed file identities bound to a particular source, runtime and lease attempt."""
|
||||
|
||||
kind: ProgramKind
|
||||
binding: v1.LeaseBinding
|
||||
source: v1.ArtifactReference
|
||||
toolchain: RuntimeIdentity
|
||||
files: Annotated[tuple[ProgramFile, ...], Field(min_length=1, max_length=MAX_CLASS_FILES)]
|
||||
|
||||
@property
|
||||
def bytes(self) -> int:
|
||||
return sum(file.artifact.bytes for file in self.files)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def bounded_files(self) -> Self:
|
||||
names = [file.path for file in self.files]
|
||||
if names != sorted(set(names)):
|
||||
raise ValueError("program files must have unique canonical order")
|
||||
if not 1 <= self.source.bytes <= v1.MAX_SOURCE_BYTES:
|
||||
raise ValueError("program source outside policy")
|
||||
if self.bytes > v1.MAX_PACKAGE_BYTES or any(
|
||||
file.artifact.bytes == 0 for file in self.files
|
||||
):
|
||||
raise ValueError("program files empty or outside byte policy")
|
||||
if self.kind == ProgramKind.NATIVE:
|
||||
validate_runtime(Language.C17, self.toolchain)
|
||||
if names != ["main"] or self.toolchain.image_kind != "linux_gnu_cxx_x86_64_v1":
|
||||
raise ValueError("native program requires its fixed executable and runtime")
|
||||
elif self.kind == ProgramKind.PYTHON:
|
||||
validate_runtime(Language.PYTHON313, self.toolchain)
|
||||
if names != ["main.py"] or self.files[0].artifact != self.source:
|
||||
raise ValueError("Python program must be the exact checked source")
|
||||
else:
|
||||
validate_runtime(Language.JAVA21, self.toolchain)
|
||||
if "Main.class" not in names or any(
|
||||
not re.fullmatch(r"[A-Za-z_$][A-Za-z0-9_$]*\.class", name) for name in names
|
||||
):
|
||||
raise ValueError("Java requires a bounded default-package Main class set")
|
||||
return self
|
||||
|
||||
|
||||
class CompilationReport(v1.CompilationReport):
|
||||
program: CompiledProgram | None
|
||||
|
||||
|
||||
class JudgementReport(v1.JudgementReport):
|
||||
schema_version: Version
|
||||
compilation: CompilationReport
|
||||
|
||||
@model_validator(mode="after")
|
||||
def program_attempt(self) -> Self:
|
||||
if (
|
||||
self.compilation.program is not None
|
||||
and self.compilation.program.binding != self.binding
|
||||
):
|
||||
raise ValueError("program belongs to another attempt")
|
||||
return self
|
||||
|
||||
def validate_for_manifest(self, manifest: JobManifest, digest: str) -> None:
|
||||
super().validate_for_manifest(manifest, digest)
|
||||
program = self.compilation.program
|
||||
if program is not None and (
|
||||
program.source != manifest.source
|
||||
or program.toolchain != manifest.toolchain
|
||||
or program.kind != runtime_contract(manifest.language)[3]
|
||||
):
|
||||
raise ValueError("program differs from authorized source or runtime")
|
||||
|
||||
|
||||
class LeaseGrant(v1.LeaseGrant):
|
||||
schema_version: Version
|
||||
|
||||
|
||||
class StageReport(v1.StageReport):
|
||||
schema_version: Version
|
||||
preparation_kind: PreparationKind
|
||||
|
||||
def validate_for_manifest(self, manifest: JobManifest) -> None:
|
||||
if self.binding.evaluation_id != manifest.evaluation_id or (
|
||||
self.preparation_kind != runtime_contract(manifest.language)[4]
|
||||
):
|
||||
raise ValueError("stage differs from authorized language preparation")
|
||||
|
||||
|
||||
class StoppedReport(v1.StoppedReport):
|
||||
schema_version: Version
|
||||
|
||||
|
||||
class RuntimeCapability(v1.WireModel):
|
||||
language: Language
|
||||
toolchain: RuntimeIdentity
|
||||
compile_ceiling: v1.RunLimits
|
||||
run_ceiling: v1.RunLimits
|
||||
|
||||
@model_validator(mode="after")
|
||||
def runtime_and_budget(self) -> Self:
|
||||
validate_runtime(self.language, self.toolchain)
|
||||
validate_startup_budget(self.language, self.compile_ceiling, preparing=True)
|
||||
validate_startup_budget(self.language, self.run_ceiling, preparing=False)
|
||||
return self
|
||||
|
||||
|
||||
class AdmissionPolicy(v1.WireModel):
|
||||
schema_version: Version
|
||||
max_concurrent_jobs: Annotated[int, Field(strict=True, ge=1, le=64)] = 1
|
||||
prefetch: Annotated[int, Field(strict=True, ge=1, le=64)] = 1
|
||||
max_retries: Annotated[int, Field(strict=True, ge=0, le=4)] = 2
|
||||
max_job_wall_time_us: Annotated[int, Field(strict=True, ge=1000, le=v1.MAX_JOB_WALL_US)]
|
||||
max_package_bytes: Annotated[int, Field(strict=True, ge=1, le=v1.MAX_PACKAGE_BYTES)]
|
||||
capabilities: Annotated[tuple[RuntimeCapability, ...], Field(min_length=1, max_length=4)]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def consistent(self) -> Self:
|
||||
if self.prefetch > self.max_concurrent_jobs:
|
||||
raise ValueError("prefetch exceeds available job slots")
|
||||
if len({cap.language for cap in self.capabilities}) != len(self.capabilities):
|
||||
raise ValueError("duplicate language capability")
|
||||
return self
|
||||
|
||||
def admit(self, manifest: JobManifest, attempt_number: int) -> None:
|
||||
# Revalidate constructed/copied values as well as transport-decoded models.
|
||||
manifest = JobManifest.model_validate(manifest.model_dump(mode="json"))
|
||||
AdmissionPolicy.model_validate(self.model_dump(mode="json"))
|
||||
if type(attempt_number) is not int or not 1 <= attempt_number <= self.max_retries + 1:
|
||||
raise ValueError("attempt retry budget exhausted")
|
||||
cap = next((cap for cap in self.capabilities if cap.language == manifest.language), None)
|
||||
if cap is None or (
|
||||
cap.toolchain != manifest.toolchain
|
||||
or not manifest.compile_limits.fits(cap.compile_ceiling)
|
||||
or not manifest.run_limits.fits(cap.run_ceiling)
|
||||
or manifest.job_wall_time_us > self.max_job_wall_time_us
|
||||
or manifest.package_bytes() > self.max_package_bytes
|
||||
):
|
||||
raise ValueError("job exceeds trusted runtime capability or budget")
|
||||
|
||||
|
||||
class AdmissionRegistry(v1.WireModel):
|
||||
"""Explicit operator opt-in for both protocol generations; no silent conversion."""
|
||||
|
||||
schema_version: Version
|
||||
legacy: v1.AdmissionPolicy | None
|
||||
multi_runtime: AdmissionPolicy | None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def at_least_one_protocol(self) -> Self:
|
||||
if self.legacy is None and self.multi_runtime is None:
|
||||
raise ValueError("node must advertise at least one explicit protocol capability")
|
||||
if self.legacy is not None and self.multi_runtime is not None:
|
||||
for field in ("max_concurrent_jobs", "prefetch", "max_retries"):
|
||||
if getattr(self.legacy, field) != getattr(self.multi_runtime, field):
|
||||
raise ValueError("protocol policies disagree on node capacity or retries")
|
||||
return self
|
||||
|
||||
def admit(self, manifest: v1.JobManifest | JobManifest, attempt_number: int) -> None:
|
||||
AdmissionRegistry.model_validate(self.model_dump(mode="json"))
|
||||
policy = self.legacy if manifest.schema_version == 1 else self.multi_runtime
|
||||
admit_manifest(policy, manifest, attempt_number)
|
||||
|
||||
|
||||
NodeAdmission = v1.AdmissionPolicy | AdmissionPolicy | AdmissionRegistry
|
||||
|
||||
|
||||
def admit_manifest(
|
||||
policy: NodeAdmission | None, manifest: v1.JobManifest | JobManifest, attempt_number: int
|
||||
) -> None:
|
||||
if type(manifest) not in (v1.JobManifest, JobManifest):
|
||||
raise ValueError("unsupported manifest model")
|
||||
# Copied/constructed values must pass the original strict protocol validators too.
|
||||
manifest = type(manifest).model_validate(manifest.model_dump(mode="json"))
|
||||
if isinstance(policy, AdmissionRegistry):
|
||||
policy.admit(manifest, attempt_number)
|
||||
elif manifest.schema_version == 1 and type(policy) is v1.AdmissionPolicy:
|
||||
v1.AdmissionPolicy.model_validate(policy.model_dump(mode="json")).admit(
|
||||
manifest, attempt_number
|
||||
)
|
||||
elif manifest.schema_version == 2 and type(policy) is AdmissionPolicy:
|
||||
policy.admit(manifest, attempt_number)
|
||||
else:
|
||||
raise ValueError("node does not advertise this manifest protocol")
|
||||
|
||||
|
||||
WIRE_MODELS = {
|
||||
"judge-manifest": JobManifest,
|
||||
"judge-lease": LeaseGrant,
|
||||
"judge-stage": StageReport,
|
||||
"judge-result": JudgementReport,
|
||||
"judge-admission": AdmissionPolicy,
|
||||
"judge-stopped": StoppedReport,
|
||||
"judge-registry": AdmissionRegistry,
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Version dispatch at storage/transport boundaries; never rewrite legacy snapshots."""
|
||||
|
||||
from pydantic import ConfigDict, RootModel
|
||||
|
||||
from kf_oj.contracts import judging as v1
|
||||
from kf_oj.contracts import judging_v2 as v2
|
||||
|
||||
Manifest = v1.JobManifest | v2.JobManifest
|
||||
Report = v1.JudgementReport | v2.JudgementReport
|
||||
|
||||
|
||||
class ManifestEnvelope(RootModel[Manifest]):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
class ReportEnvelope(RootModel[Report]):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
def decode_manifest(payload: bytes) -> Manifest:
|
||||
return v1.decode_wire(ManifestEnvelope, payload).root
|
||||
|
||||
|
||||
def decode_report(payload: bytes) -> Report:
|
||||
return v1.decode_wire(ReportEnvelope, payload).root
|
||||
|
||||
|
||||
def manifest_bytes(manifest: Manifest) -> bytes:
|
||||
return v1.manifest_bytes(manifest)
|
||||
|
||||
|
||||
def validate_report(report: Report, manifest: Manifest, digest: str) -> None:
|
||||
if report.schema_version != manifest.schema_version:
|
||||
raise ValueError("report protocol differs from authorized manifest")
|
||||
report.validate_for_manifest(manifest, digest)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Bounded administrator projections and immutable operation receipts, separate from judging."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import Field, StrictBool
|
||||
|
||||
from kf_oj.contracts.admin_problems import CandidatePreview
|
||||
from kf_oj.contracts.judging import Identifier, Label, Version, WireModel
|
||||
from kf_oj.contracts.problems import ProblemSummary
|
||||
|
||||
MaintenanceAction = Literal["publish", "disable", "restore"]
|
||||
AuditAction = Literal[
|
||||
"upload_created",
|
||||
"upload_received",
|
||||
"import_ready",
|
||||
"import_failed",
|
||||
"import_discarded",
|
||||
"import_expired",
|
||||
"published",
|
||||
"disabled",
|
||||
"restored",
|
||||
"authority_bootstrap",
|
||||
"authority_grant",
|
||||
"authority_revoke",
|
||||
"authority_enable",
|
||||
"authority_disable",
|
||||
]
|
||||
|
||||
|
||||
class MaintainedProblem(WireModel):
|
||||
id: Identifier
|
||||
slug: Label
|
||||
enabled: StrictBool
|
||||
active_version_id: Identifier | None
|
||||
revision: Annotated[int, Field(strict=True, ge=0)]
|
||||
title: Annotated[str | None, Field(max_length=160)]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class MaintainedProblemPage(WireModel):
|
||||
schema_version: Version = 1
|
||||
items: Annotated[tuple[MaintainedProblem, ...], Field(max_length=100)]
|
||||
next_after: Label | None
|
||||
|
||||
|
||||
class MaintainedVersion(ProblemSummary):
|
||||
published_at: datetime
|
||||
case_count: Annotated[int, Field(strict=True, ge=1, le=256)]
|
||||
|
||||
|
||||
class MaintainedVersionPage(WireModel):
|
||||
schema_version: Version = 1
|
||||
items: Annotated[tuple[MaintainedVersion, ...], Field(max_length=100)]
|
||||
next_before: Annotated[int | None, Field(strict=True, ge=1)]
|
||||
|
||||
|
||||
class PublishCandidate(WireModel):
|
||||
schema_version: Version = 1
|
||||
operation_id: Identifier
|
||||
import_operation_id: Identifier
|
||||
candidate_id: Identifier
|
||||
expected_active_version_id: Identifier | None
|
||||
|
||||
|
||||
class ChangeProblemAvailability(WireModel):
|
||||
schema_version: Version = 1
|
||||
operation_id: Identifier
|
||||
expected_active_version_id: Identifier
|
||||
expected_enabled: StrictBool
|
||||
|
||||
|
||||
class MaintenanceReceipt(WireModel):
|
||||
schema_version: Version = 1
|
||||
operation_id: Identifier
|
||||
action: MaintenanceAction
|
||||
completed_at: datetime
|
||||
problem: MaintainedProblem
|
||||
published_version_id: Identifier | None = None
|
||||
|
||||
|
||||
class MaintenanceAuditEvent(WireModel):
|
||||
id: Identifier
|
||||
created_at: datetime
|
||||
actor_kind: Literal["account", "operator"]
|
||||
actor_id: Identifier | None
|
||||
operator_name: Annotated[str | None, Field(max_length=63)]
|
||||
authorization_version: Annotated[str, Field(pattern=r"^[1-9][0-9]{0,18}$")]
|
||||
operation_id: Identifier
|
||||
action: AuditAction
|
||||
target_id: Identifier | None
|
||||
result: Literal["committed"] = "committed"
|
||||
|
||||
|
||||
class MaintenanceAuditPage(WireModel):
|
||||
schema_version: Version = 1
|
||||
items: Annotated[tuple[MaintenanceAuditEvent, ...], Field(max_length=100)]
|
||||
next_before: Annotated[str | None, Field(max_length=100)]
|
||||
|
||||
|
||||
# This alias deliberately uses the existing sanitized candidate contract. No new
|
||||
# endpoint exposes the private manifest, artifact digest or hidden case names.
|
||||
PublicCandidate = CandidatePreview
|
||||
@@ -2,7 +2,8 @@ from typing import Annotated, Literal
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from kf_oj.contracts.judging import Identifier, Label, Language, RunLimits, WireModel
|
||||
from kf_oj.contracts.judging import Identifier, Label, RunLimits, WireModel
|
||||
from kf_oj.contracts.judging_v2 import Language, PreparationKind, runtime_contract
|
||||
|
||||
|
||||
class PublicSample(WireModel):
|
||||
@@ -20,10 +21,36 @@ class ProblemSummary(WireModel):
|
||||
languages: tuple[Language, ...]
|
||||
|
||||
|
||||
class PublicLanguageConfiguration(WireModel):
|
||||
language: Language
|
||||
runtime_version: str
|
||||
source_name: str
|
||||
entrypoint: str
|
||||
preparation_kind: PreparationKind
|
||||
compile_limits: RunLimits
|
||||
run_limits: RunLimits
|
||||
job_wall_time_us: int
|
||||
|
||||
@classmethod
|
||||
def from_configuration(cls, configuration):
|
||||
source, _, _, _, preparation = runtime_contract(configuration.language)
|
||||
return cls(
|
||||
language=configuration.language,
|
||||
runtime_version=configuration.toolchain.runtime_version,
|
||||
source_name=source,
|
||||
entrypoint="Main" if configuration.language == Language.JAVA21 else source,
|
||||
preparation_kind=preparation,
|
||||
compile_limits=configuration.compile_limits,
|
||||
run_limits=configuration.run_limits,
|
||||
job_wall_time_us=configuration.job_wall_time_us,
|
||||
)
|
||||
|
||||
|
||||
class ProblemDetail(ProblemSummary):
|
||||
statement: str
|
||||
samples: tuple[PublicSample, ...]
|
||||
run_limits: RunLimits
|
||||
run_limits: RunLimits | None
|
||||
language_configurations: tuple[PublicLanguageConfiguration, ...] = ()
|
||||
|
||||
|
||||
class ProblemPage(WireModel):
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Private offline restore byte-copy protocol; conveys no execution authority."""
|
||||
|
||||
import re
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
BeforeValidator,
|
||||
ConfigDict,
|
||||
Field,
|
||||
RootModel,
|
||||
StrictInt,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from kf_oj.contracts.judging import integer_version
|
||||
|
||||
MAX_FILES = 98304
|
||||
MAX_LOGICAL_BYTES = 12 * 1024**3
|
||||
MAX_FILE_BYTES = 256 * 1024**2
|
||||
MAX_BATCH_BYTES = 64 * 1024**2
|
||||
DOMAINS = frozenset(
|
||||
"postgres artifacts imports app-config judge-config factory operations releases "
|
||||
"images dependencies proxy recovery".split()
|
||||
)
|
||||
Count = Annotated[StrictInt, Field(ge=0, le=MAX_FILES)]
|
||||
ByteCount = Annotated[StrictInt, Field(ge=0, le=MAX_LOGICAL_BYTES)]
|
||||
|
||||
|
||||
class Record(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
|
||||
class RestoreFile(Record):
|
||||
path: Annotated[str, Field(min_length=1, max_length=512)]
|
||||
uid: Annotated[StrictInt, Field(ge=0, le=1205)]
|
||||
gid: Annotated[StrictInt, Field(ge=0, le=1205)]
|
||||
mode: StrictInt
|
||||
bytes: Annotated[StrictInt, Field(ge=0, le=MAX_FILE_BYTES)]
|
||||
sha256: Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_member(self) -> "RestoreFile":
|
||||
parts = self.path.split("/")
|
||||
if (
|
||||
len(parts) < 2
|
||||
or parts[0] not in DOMAINS
|
||||
or any(
|
||||
p in {"", ".", "..", ".git", ".ssh", ".localplan", "docs"}
|
||||
or re.fullmatch(r"[A-Za-z0-9_.+@-]{1,255}", p) is None
|
||||
for p in parts
|
||||
)
|
||||
):
|
||||
raise ValueError("restore file must be below a fixed backup domain")
|
||||
if self.uid not in {0, 1200, 1201, 1202, 1203, 1204, 1205} or self.gid not in (
|
||||
{0, 1203, 1204, 1205} if self.uid == 0 else {self.uid}
|
||||
):
|
||||
raise ValueError("unsupported restore identity")
|
||||
if self.mode not in {0o400, 0o444, 0o600, 0o640, 0o644, 0o555, 0o755}:
|
||||
raise ValueError("unsupported restore file mode")
|
||||
if self.sha256 in {"a" * 64, "0" * 64}:
|
||||
raise ValueError("placeholder digest refused")
|
||||
return self
|
||||
|
||||
|
||||
class Open(Record):
|
||||
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||
kind: Literal["restore.io.open"]
|
||||
backup_id: Annotated[str, Field(pattern=r"^bk-[0-9a-f]{32}$")]
|
||||
recovery_id: Annotated[str, Field(pattern=r"^rr-[0-9a-f]{32}$")]
|
||||
expected_files: Annotated[StrictInt, Field(ge=1, le=MAX_FILES)]
|
||||
expected_logical_bytes: ByteCount
|
||||
deadline_monotonic_ns: Annotated[StrictInt, Field(ge=1, le=2**63 - 1)]
|
||||
|
||||
|
||||
class Batch(Record):
|
||||
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||
kind: Literal["restore.io.batch"]
|
||||
batch_id: Annotated[StrictInt, Field(ge=1, le=MAX_FILES)]
|
||||
files: Annotated[list[RestoreFile], Field(min_length=1, max_length=256)]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_batch(self) -> "Batch":
|
||||
if len(self.files) > 1 and sum(f.bytes for f in self.files) > MAX_BATCH_BYTES:
|
||||
raise ValueError("restore batch byte bound exceeded")
|
||||
if len({f.path for f in self.files}) != len(self.files):
|
||||
raise ValueError("duplicate restore batch member")
|
||||
return self
|
||||
|
||||
|
||||
class Finish(Record):
|
||||
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||
kind: Literal["restore.io.finish"]
|
||||
|
||||
|
||||
class RestoreIORequest(RootModel[Annotated[Open | Batch | Finish, Field(discriminator="kind")]]):
|
||||
pass
|
||||
|
||||
|
||||
class RestoreIOReply(Record):
|
||||
schema_version: Annotated[Literal[1], BeforeValidator(integer_version)]
|
||||
kind: Literal["restore.io.reply"]
|
||||
stage: Literal["opened", "verified", "finished", "refused"]
|
||||
batch_id: Count
|
||||
verified_files: Count
|
||||
verified_logical_bytes: ByteCount
|
||||
execution_verified: Annotated[
|
||||
Literal[False],
|
||||
BeforeValidator(lambda value: value if type(value) is bool else _bad_boolean()),
|
||||
]
|
||||
error: Literal["none", "protocol", "guard", "deadline", "filesystem", "bytes", "identity"]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_reply(self) -> "RestoreIOReply":
|
||||
if (self.stage == "refused") != (self.error != "none"):
|
||||
raise ValueError("restore refusal state differs")
|
||||
if self.stage == "opened" and (
|
||||
self.batch_id or self.verified_files or self.verified_logical_bytes
|
||||
):
|
||||
raise ValueError("restore open cannot claim verified data")
|
||||
if self.stage in {"verified", "finished"} and not (
|
||||
1 <= self.batch_id <= self.verified_files
|
||||
):
|
||||
raise ValueError("restore completion has no verified batch")
|
||||
return self
|
||||
|
||||
|
||||
def _bad_boolean() -> bool:
|
||||
raise ValueError("execution verification must be a boolean false")
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Bounded anonymous aggregates; raw measurements never pass through JS floats."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Literal, Self
|
||||
|
||||
from pydantic import AfterValidator, Field, StrictBool, model_validator
|
||||
|
||||
from kf_oj.contracts.judging import I64_MAX, Digest, Identifier, Version, WireModel
|
||||
from kf_oj.contracts.judging_v2 import Language
|
||||
|
||||
MAX_PERFORMANCE_SAMPLES = 10_000
|
||||
MIN_PERFORMANCE_SAMPLES = 5
|
||||
MAX_PERFORMANCE_BUCKETS = 64
|
||||
|
||||
|
||||
def bounded_decimal(value: str) -> str:
|
||||
if int(value) > I64_MAX:
|
||||
raise ValueError("measurement exceeds signed storage range")
|
||||
return value
|
||||
|
||||
|
||||
DecimalMeasurement = Annotated[
|
||||
str,
|
||||
Field(strict=True, min_length=1, max_length=19, pattern=r"^(0|[1-9][0-9]*)$"),
|
||||
AfterValidator(bounded_decimal),
|
||||
]
|
||||
SampleCount = Annotated[int, Field(strict=True, ge=0, le=MAX_PERFORMANCE_SAMPLES)]
|
||||
UnavailableReason = Literal[
|
||||
"not_accepted", "measurement_missing", "environment_unverified", "insufficient_samples"
|
||||
]
|
||||
|
||||
|
||||
class PerformanceBucket(WireModel):
|
||||
start: DecimalMeasurement
|
||||
end: DecimalMeasurement
|
||||
upper_inclusive: StrictBool
|
||||
count: SampleCount
|
||||
|
||||
@model_validator(mode="after")
|
||||
def interval(self) -> Self:
|
||||
if int(self.end) <= int(self.start):
|
||||
raise ValueError("empty or reversed interval")
|
||||
return self
|
||||
|
||||
|
||||
class PerformanceMetric(WireModel):
|
||||
unit: Literal["cpu_microseconds", "bytes"]
|
||||
current: DecimalMeasurement | None
|
||||
available: StrictBool
|
||||
reason: UnavailableReason | None
|
||||
sample_count: SampleCount
|
||||
window_truncated: StrictBool
|
||||
oldest_sample_at: datetime | None
|
||||
# These counts are computed against the original integers, never rounded bins.
|
||||
strictly_slower_count: SampleCount | None
|
||||
tied_count: SampleCount | None
|
||||
buckets: Annotated[tuple[PerformanceBucket, ...], Field(max_length=MAX_PERFORMANCE_BUCKETS)]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def aggregate(self) -> Self:
|
||||
if self.oldest_sample_at is not None and self.oldest_sample_at.utcoffset() is None:
|
||||
raise ValueError("sample timestamp must have a timezone")
|
||||
if self.available:
|
||||
if (
|
||||
self.reason is not None
|
||||
or self.current is None
|
||||
or self.sample_count < MIN_PERFORMANCE_SAMPLES
|
||||
or self.strictly_slower_count is None
|
||||
or self.tied_count is None
|
||||
or self.oldest_sample_at is None
|
||||
or not self.buckets
|
||||
):
|
||||
raise ValueError("incomplete available aggregate")
|
||||
if self.strictly_slower_count + self.tied_count > self.sample_count:
|
||||
raise ValueError("comparison counts exceed the cohort")
|
||||
if sum(b.count for b in self.buckets) != self.sample_count:
|
||||
raise ValueError("histogram loses samples")
|
||||
if int(self.buckets[0].start) != 0:
|
||||
raise ValueError("histogram must include zero")
|
||||
for index, bucket in enumerate(self.buckets):
|
||||
if bucket.upper_inclusive != (index == len(self.buckets) - 1):
|
||||
raise ValueError("only the last interval includes its upper endpoint")
|
||||
if index and self.buckets[index - 1].end != bucket.start:
|
||||
raise ValueError("histogram intervals must be contiguous")
|
||||
elif (
|
||||
self.reason is None
|
||||
or self.buckets
|
||||
or self.strictly_slower_count is not None
|
||||
or self.tied_count is not None
|
||||
):
|
||||
raise ValueError("unavailable aggregate cannot expose comparison data")
|
||||
if self.reason == "measurement_missing" and self.current is not None:
|
||||
raise ValueError("missing measurement cannot claim a value")
|
||||
if self.reason == "insufficient_samples" and (
|
||||
self.sample_count >= MIN_PERFORMANCE_SAMPLES or self.current is None
|
||||
):
|
||||
raise ValueError("inconsistent insufficient cohort")
|
||||
return self
|
||||
|
||||
|
||||
class SubmissionPerformance(WireModel):
|
||||
schema_version: Version = 1
|
||||
kind: Literal["judge.performance"] = "judge.performance"
|
||||
submission_id: Identifier
|
||||
problem_version_id: Identifier
|
||||
language: Language
|
||||
as_of: datetime
|
||||
# Opaque trusted comparison identity, not a worker name, path or credential.
|
||||
comparison_group: Digest | None
|
||||
window_limit: Literal[10_000] = MAX_PERFORMANCE_SAMPLES
|
||||
minimum_samples: Literal[5] = MIN_PERFORMANCE_SAMPLES
|
||||
cpu: PerformanceMetric
|
||||
memory: PerformanceMetric
|
||||
|
||||
@model_validator(mode="after")
|
||||
def snapshot(self) -> Self:
|
||||
if self.as_of.utcoffset() is None:
|
||||
raise ValueError("snapshot timestamp must have a timezone")
|
||||
if self.cpu.unit != "cpu_microseconds" or self.memory.unit != "bytes":
|
||||
raise ValueError("metric unit mismatch")
|
||||
for metric in (self.cpu, self.memory):
|
||||
if metric.available and self.comparison_group is None:
|
||||
raise ValueError("comparison requires trusted environment binding")
|
||||
if metric.oldest_sample_at is not None and metric.oldest_sample_at > self.as_of:
|
||||
raise ValueError("sample is newer than the snapshot")
|
||||
return self
|
||||
@@ -8,11 +8,11 @@ from kf_oj.contracts.judging import (
|
||||
CompilationStatus,
|
||||
FailureCode,
|
||||
Identifier,
|
||||
Language,
|
||||
Measurement,
|
||||
Verdict,
|
||||
WireModel,
|
||||
)
|
||||
from kf_oj.contracts.judging_v2 import Language
|
||||
|
||||
EvaluationState = Literal[
|
||||
"queued", "claimed", "compiling", "running", "cancelling", "finished", "cancelled", "failed"
|
||||
|
||||
@@ -16,7 +16,53 @@ class JsonFormatter(logging.Formatter):
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def configure_logging(level: str) -> None:
|
||||
class ProductionFormatter(logging.Formatter):
|
||||
"""Closed event vocabulary; arbitrary messages/arguments/tracebacks are private data."""
|
||||
|
||||
EVENTS = {
|
||||
("kf_oj.api", "Platform API started"): "api_started",
|
||||
("kf_oj.api", "Platform API stopped"): "api_stopped",
|
||||
("kf_oj.outbox", "outbox broker unavailable"): "outbox_broker_unavailable",
|
||||
("kf_oj.outbox", "outbox database unavailable"): "outbox_database_unavailable",
|
||||
}
|
||||
LOGGERS = {"kf_oj.api", "kf_oj.outbox", "uvicorn", "uvicorn.error", "uvicorn.access"}
|
||||
LEVELS = {
|
||||
logging.DEBUG: "DEBUG",
|
||||
logging.INFO: "INFO",
|
||||
logging.WARNING: "WARNING",
|
||||
logging.ERROR: "ERROR",
|
||||
logging.CRITICAL: "CRITICAL",
|
||||
}
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
# Do not call getMessage/formatException: library errors can contain
|
||||
# connection URIs, query parameters, source or diagnostic body bytes.
|
||||
event = "infrastructure_event"
|
||||
if isinstance(record.msg, str) and not record.args:
|
||||
event = self.EVENTS.get((record.name, record.msg), event)
|
||||
return json.dumps(
|
||||
{
|
||||
"timestamp": datetime.fromtimestamp(record.created, UTC).isoformat(),
|
||||
"level": self.LEVELS.get(record.levelno, "UNKNOWN"),
|
||||
"logger": record.name if record.name in self.LOGGERS else "infrastructure",
|
||||
"message": event,
|
||||
"exception_present": record.exc_info is not None,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
|
||||
def configure_logging(level: str, *, production: bool = False) -> None:
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(JsonFormatter())
|
||||
handler.setFormatter(ProductionFormatter() if production else JsonFormatter())
|
||||
logging.basicConfig(level=level, handlers=[handler], force=True)
|
||||
if production:
|
||||
# Uvicorn installs its own handlers before lifespan, which otherwise
|
||||
# bypass the root formatter when reporting an unhandled application error.
|
||||
for name in ("uvicorn", "uvicorn.error"):
|
||||
logger = logging.getLogger(name)
|
||||
logger.handlers.clear()
|
||||
logger.propagate = True
|
||||
access = logging.getLogger("uvicorn.access")
|
||||
access.handlers.clear()
|
||||
access.propagate = False
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import Literal
|
||||
from pydantic import Field, SecretStr, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from kf_oj.contracts.judging import AdmissionPolicy
|
||||
from kf_oj.contracts.judging_v2 import NodeAdmission
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
@@ -13,6 +13,7 @@ class Settings(BaseSettings):
|
||||
environment: Literal["development", "test", "production"] = "development"
|
||||
log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
|
||||
database_url: SecretStr | None = None
|
||||
identity_operator_database_url: SecretStr | None = None
|
||||
amqp_url: SecretStr | None = None
|
||||
browser_origins: tuple[str, ...] = (
|
||||
"http://localhost:5173",
|
||||
@@ -22,11 +23,15 @@ class Settings(BaseSettings):
|
||||
)
|
||||
session_lifetime_seconds: int = Field(default=43200, ge=300, le=86400)
|
||||
artifact_root: Path | None = None
|
||||
import_staging_root: Path | None = None
|
||||
import_worker_cgroup: Path | None = None
|
||||
artifact_max_bytes: int = Field(default=536870912, ge=1, le=16 * 1024**3)
|
||||
artifact_max_files: int = Field(default=4096, ge=1, le=65536)
|
||||
judge_admission: AdmissionPolicy | None = None
|
||||
judge_admission: NodeAdmission | None = None
|
||||
coordinator_url: str | None = None
|
||||
coordinator_token: SecretStr | None = None
|
||||
delivery_release_id: str | None = Field(default=None, pattern=r"^sd-[0-9a-f]{12}$")
|
||||
delivery_configuration_id: str | None = Field(default=None, pattern=r"^cfg-[0-9a-f]{24}$")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def secure_origins(self):
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Bounded single-level ZIP reader; no extractall, metadata restoration or content execution."""
|
||||
|
||||
import binascii
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import struct
|
||||
import time
|
||||
import zlib
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
MAX_ARCHIVE_BYTES,
|
||||
MAX_ARCHIVE_FILES,
|
||||
MAX_UNPACKED_BYTES,
|
||||
)
|
||||
|
||||
NAME = re.compile(rb"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\Z")
|
||||
CHUNK = 65536
|
||||
LOCAL = struct.Struct("<4s5H3I2H")
|
||||
CENTRAL = struct.Struct("<4s6H3I5H2I")
|
||||
END = struct.Struct("<4s4H2IH")
|
||||
|
||||
|
||||
class ArchiveError(ValueError):
|
||||
"""A fixed sanitized failure; never echo an archive name, data or local path."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArchiveMember:
|
||||
name: str
|
||||
method: int
|
||||
crc32: int
|
||||
compressed_bytes: int
|
||||
bytes: int
|
||||
offset: int
|
||||
flags: int
|
||||
|
||||
|
||||
class ArchiveReader:
|
||||
def __init__(self, descriptor: int, expected_bytes: int, *, deadline: float):
|
||||
self.descriptor, self.deadline = descriptor, deadline
|
||||
self.before = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(self.before.st_mode)
|
||||
or self.before.st_uid != os.getuid()
|
||||
or self.before.st_nlink != 1
|
||||
or stat.S_IMODE(self.before.st_mode) != 0o600
|
||||
or not 22 <= expected_bytes <= MAX_ARCHIVE_BYTES
|
||||
or self.before.st_size != expected_bytes
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
self.size = expected_bytes
|
||||
self.unpacked_bytes, self.steps = 0, 0
|
||||
self.read_members: set[str] = set()
|
||||
self.members = self._members()
|
||||
|
||||
def _tick(self) -> None:
|
||||
self.steps += 1
|
||||
if self.steps > 8192 or time.monotonic() >= self.deadline:
|
||||
raise ArchiveError("archive_deadline")
|
||||
|
||||
def _read(self, offset: int, size: int) -> bytes:
|
||||
self._tick()
|
||||
if not 0 <= size <= 1024**2 or offset < 0 or offset + size > self.size:
|
||||
raise ArchiveError("invalid_archive")
|
||||
data = os.pread(self.descriptor, size, offset)
|
||||
if len(data) != size:
|
||||
raise ArchiveError("invalid_archive")
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _extra(data: bytes) -> None:
|
||||
offset, fields = 0, 0
|
||||
# These bounded timestamps/UID attributes are ignored, never restored.
|
||||
# ZIP64, Unicode aliases and all other extension semantics are unsupported.
|
||||
while offset < len(data):
|
||||
if offset + 4 > len(data) or fields >= 32:
|
||||
raise ArchiveError("invalid_archive")
|
||||
kind, size = struct.unpack_from("<HH", data, offset)
|
||||
offset += 4
|
||||
if kind not in (0x5455, 0x7875, 0x5855, 0x7855, 0x000A) or offset + size > len(data):
|
||||
raise ArchiveError("invalid_archive")
|
||||
offset += size
|
||||
fields += 1
|
||||
|
||||
def _members(self) -> tuple[ArchiveMember, ...]:
|
||||
tail_offset = max(0, self.size - 1046)
|
||||
tail = self._read(tail_offset, self.size - tail_offset)
|
||||
position = tail.rfind(b"PK\x05\x06")
|
||||
if position < 0 or position + END.size > len(tail):
|
||||
raise ArchiveError("invalid_archive")
|
||||
sig, disk, cd_disk, disk_count, count, cd_bytes, cd_offset, comment = END.unpack_from(
|
||||
tail, position
|
||||
)
|
||||
end_offset = tail_offset + position
|
||||
if (
|
||||
sig != b"PK\x05\x06"
|
||||
or disk != 0
|
||||
or cd_disk != 0
|
||||
or disk_count != count
|
||||
or not 1 <= count <= MAX_ARCHIVE_FILES
|
||||
or comment > 1024
|
||||
or end_offset + END.size + comment != self.size
|
||||
or not 46 <= cd_bytes <= 1024**2
|
||||
or cd_offset + cd_bytes != end_offset
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
directory = self._read(cd_offset, cd_bytes)
|
||||
offset, records, names = 0, [], set()
|
||||
for _ in range(count):
|
||||
if offset + CENTRAL.size > len(directory):
|
||||
raise ArchiveError("invalid_archive")
|
||||
values = CENTRAL.unpack_from(directory, offset)
|
||||
(
|
||||
sig,
|
||||
_,
|
||||
needed,
|
||||
flags,
|
||||
method,
|
||||
_,
|
||||
_,
|
||||
crc,
|
||||
compressed,
|
||||
size,
|
||||
name_size,
|
||||
extra_size,
|
||||
comment_size,
|
||||
disk,
|
||||
_,
|
||||
attributes,
|
||||
local_offset,
|
||||
) = values
|
||||
allowed_flags = 0x0808 | (0x0006 if method == 8 else 0)
|
||||
if (
|
||||
sig != b"PK\x01\x02"
|
||||
or needed not in (10, 20)
|
||||
or method not in (0, 8)
|
||||
or (method == 8 and needed != 20)
|
||||
or flags & ~allowed_flags
|
||||
or disk != 0
|
||||
or not 1 <= name_size <= 128
|
||||
or extra_size > 1024
|
||||
or comment_size > 256
|
||||
or size > MAX_UNPACKED_BYTES
|
||||
or compressed > MAX_ARCHIVE_BYTES
|
||||
or local_offset + LOCAL.size > cd_offset
|
||||
or attributes & 0x10
|
||||
or stat.S_IFMT(attributes >> 16) not in (0, stat.S_IFREG)
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
start = offset + CENTRAL.size
|
||||
finish = start + name_size + extra_size + comment_size
|
||||
if finish > len(directory):
|
||||
raise ArchiveError("invalid_archive")
|
||||
name_bytes = directory[start : start + name_size]
|
||||
if not NAME.fullmatch(name_bytes):
|
||||
raise ArchiveError("invalid_archive")
|
||||
name = name_bytes.decode("ascii")
|
||||
if name.lower() in names:
|
||||
raise ArchiveError("invalid_archive")
|
||||
names.add(name.lower())
|
||||
self._extra(directory[start + name_size : start + name_size + extra_size])
|
||||
records.append(ArchiveMember(name, method, crc, compressed, size, local_offset, flags))
|
||||
offset = finish
|
||||
if offset != len(directory) or sum(m.bytes for m in records) > MAX_UNPACKED_BYTES:
|
||||
raise ArchiveError("invalid_archive")
|
||||
cursor, members = 0, []
|
||||
for member in sorted(records, key=lambda m: m.offset):
|
||||
if member.offset != cursor:
|
||||
raise ArchiveError("invalid_archive")
|
||||
sig, needed, flags, method, _, _, crc, compressed, size, name_size, extra_size = (
|
||||
LOCAL.unpack(self._read(cursor, LOCAL.size))
|
||||
)
|
||||
if (
|
||||
sig != b"PK\x03\x04"
|
||||
or needed not in (10, 20)
|
||||
or flags != member.flags
|
||||
or method != member.method
|
||||
or (method == 8 and needed != 20)
|
||||
or name_size != len(member.name)
|
||||
or extra_size > 1024
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
metadata = self._read(cursor + LOCAL.size, name_size + extra_size)
|
||||
if metadata[:name_size] != member.name.encode("ascii"):
|
||||
raise ArchiveError("invalid_archive")
|
||||
self._extra(metadata[name_size:])
|
||||
data_offset = cursor + LOCAL.size + name_size + extra_size
|
||||
cursor = data_offset + member.compressed_bytes
|
||||
if cursor > cd_offset or (method == 0 and member.compressed_bytes != member.bytes):
|
||||
raise ArchiveError("invalid_archive")
|
||||
if flags & 8:
|
||||
if (crc, compressed, size) not in (
|
||||
(0, 0, 0),
|
||||
(member.crc32, member.compressed_bytes, member.bytes),
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
marker = self._read(cursor, 4)
|
||||
descriptor_size = 16 if marker == b"PK\x07\x08" else 12
|
||||
descriptor = self._read(cursor, descriptor_size)
|
||||
described = struct.unpack("<III", descriptor[-12:])
|
||||
if described != (member.crc32, member.compressed_bytes, member.bytes):
|
||||
raise ArchiveError("invalid_archive")
|
||||
cursor += descriptor_size
|
||||
elif (crc, compressed, size) != (member.crc32, member.compressed_bytes, member.bytes):
|
||||
raise ArchiveError("invalid_archive")
|
||||
members.append(
|
||||
ArchiveMember(
|
||||
member.name,
|
||||
method,
|
||||
member.crc32,
|
||||
member.compressed_bytes,
|
||||
member.bytes,
|
||||
data_offset,
|
||||
flags,
|
||||
)
|
||||
)
|
||||
if cursor != cd_offset:
|
||||
raise ArchiveError("invalid_archive")
|
||||
return tuple(members)
|
||||
|
||||
def verify_digest(self, expected: str) -> None:
|
||||
digest = hashlib.sha256()
|
||||
for offset in range(0, self.size, CHUNK):
|
||||
digest.update(self._read(offset, min(CHUNK, self.size - offset)))
|
||||
if digest.hexdigest() != expected:
|
||||
raise ArchiveError("archive_digest_mismatch")
|
||||
|
||||
def verify_unchanged(self) -> None:
|
||||
after = os.fstat(self.descriptor)
|
||||
fields = (
|
||||
"st_dev",
|
||||
"st_ino",
|
||||
"st_nlink",
|
||||
"st_mode",
|
||||
"st_uid",
|
||||
"st_size",
|
||||
"st_mtime_ns",
|
||||
"st_ctime_ns",
|
||||
)
|
||||
if any(getattr(self.before, f) != getattr(after, f) for f in fields):
|
||||
raise ArchiveError("archive_changed")
|
||||
|
||||
@staticmethod
|
||||
def _nested(prefix: bytes) -> bool:
|
||||
return prefix.startswith(
|
||||
(
|
||||
b"PK\x03\x04",
|
||||
b"PK\x05\x06",
|
||||
b"PK\x07\x08",
|
||||
b"\x1f\x8b",
|
||||
b"BZh",
|
||||
b"\xfd7zXZ\x00",
|
||||
b"Rar!\x1a\x07",
|
||||
b"7z\xbc\xaf\x27\x1c",
|
||||
)
|
||||
) or (len(prefix) >= 262 and prefix[257:262] == b"ustar")
|
||||
|
||||
def chunks(self, name: str, maximum: int) -> Iterator[bytes]:
|
||||
member = next((m for m in self.members if m.name == name), None)
|
||||
if member is None or name in self.read_members or member.bytes > maximum:
|
||||
raise ArchiveError("invalid_archive")
|
||||
self.read_members.add(name)
|
||||
used, crc, prefix = 0, 0, bytearray()
|
||||
inflater = zlib.decompressobj(-15) if member.method == 8 else None
|
||||
try:
|
||||
for offset in range(0, member.compressed_bytes, CHUNK):
|
||||
pending = self._read(
|
||||
member.offset + offset, min(CHUNK, member.compressed_bytes - offset)
|
||||
)
|
||||
while pending:
|
||||
self._tick()
|
||||
if inflater:
|
||||
data = inflater.decompress(pending, min(CHUNK, member.bytes - used + 1))
|
||||
pending = inflater.unconsumed_tail
|
||||
if inflater.unused_data:
|
||||
raise ArchiveError("invalid_archive")
|
||||
else:
|
||||
data, pending = pending, b""
|
||||
used += len(data)
|
||||
self.unpacked_bytes += len(data)
|
||||
if (
|
||||
used > member.bytes
|
||||
or used > maximum
|
||||
or self.unpacked_bytes > MAX_UNPACKED_BYTES
|
||||
):
|
||||
raise ArchiveError("invalid_archive")
|
||||
prefix.extend(data[: max(0, 512 - len(prefix))])
|
||||
if self._nested(bytes(prefix)):
|
||||
raise ArchiveError("invalid_archive")
|
||||
crc = binascii.crc32(data, crc)
|
||||
if data:
|
||||
yield data
|
||||
if (inflater and not inflater.eof) or used != member.bytes or crc != member.crc32:
|
||||
raise ArchiveError("invalid_archive")
|
||||
except zlib.error:
|
||||
raise ArchiveError("invalid_archive") from None
|
||||
|
||||
def read(self, name: str, maximum: int) -> bytes:
|
||||
return b"".join(self.chunks(name, maximum))
|
||||
|
||||
def extract_remaining(self, directory: int) -> dict[str, tuple[str, int]]:
|
||||
info = os.fstat(directory)
|
||||
if (
|
||||
not stat.S_ISDIR(info.st_mode)
|
||||
or info.st_uid != os.getuid()
|
||||
or stat.S_IMODE(info.st_mode) != 0o700
|
||||
):
|
||||
raise ArchiveError("invalid_extraction_directory")
|
||||
references = {}
|
||||
for member in self.members:
|
||||
if member.name in self.read_members:
|
||||
continue
|
||||
fd = os.open(
|
||||
member.name,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
dir_fd=directory,
|
||||
)
|
||||
try:
|
||||
digest, size = hashlib.sha256(), 0
|
||||
for chunk in self.chunks(member.name, member.bytes):
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
remaining = memoryview(chunk)
|
||||
while remaining:
|
||||
written = os.write(fd, remaining)
|
||||
if written <= 0:
|
||||
raise ArchiveError("extraction_unavailable")
|
||||
remaining = remaining[written:]
|
||||
os.fsync(fd)
|
||||
references[member.name] = (digest.hexdigest(), size)
|
||||
finally:
|
||||
os.close(fd)
|
||||
os.fsync(directory)
|
||||
self.verify_unchanged()
|
||||
return references
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Read-only private backup inventory, separate from garbage collection and grading."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from kf_oj.contracts.judging import ArtifactReference
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.artifacts.models import Artifact
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
|
||||
MAX_REFERENCES = 65536
|
||||
|
||||
|
||||
def verify_existing(root: Path, reference: ArtifactReference) -> None:
|
||||
"""No mkdir, lock creation or writable descriptor, even for a missing store."""
|
||||
root = root.absolute()
|
||||
if root.resolve(strict=True) != root:
|
||||
raise ValueError("backup storage path contains links")
|
||||
directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
try:
|
||||
info = os.fstat(directory)
|
||||
if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700:
|
||||
raise ValueError("backup storage root identity differs")
|
||||
descriptor = os.open(
|
||||
reference.digest,
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||
dir_fd=directory,
|
||||
)
|
||||
with os.fdopen(descriptor, "rb") as source:
|
||||
before = os.fstat(source.fileno())
|
||||
if (
|
||||
not stat.S_ISREG(before.st_mode)
|
||||
or before.st_nlink != 1
|
||||
or before.st_uid != os.getuid()
|
||||
or stat.S_IMODE(before.st_mode) != 0o600
|
||||
or before.st_size != reference.bytes
|
||||
):
|
||||
raise ValueError("backup artifact identity or length differs")
|
||||
digest = hashlib.sha256()
|
||||
remaining = reference.bytes
|
||||
while remaining:
|
||||
block = source.read(min(65536, remaining))
|
||||
if not block:
|
||||
raise ValueError("backup artifact truncated")
|
||||
digest.update(block)
|
||||
remaining -= len(block)
|
||||
after = os.fstat(source.fileno())
|
||||
if (
|
||||
source.read(1)
|
||||
or digest.hexdigest() != reference.digest
|
||||
or any(
|
||||
getattr(before, key) != getattr(after, key)
|
||||
for key in ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns")
|
||||
)
|
||||
):
|
||||
raise ValueError("backup artifact changed or digest differs")
|
||||
finally:
|
||||
os.close(directory)
|
||||
|
||||
|
||||
async def inventory() -> dict:
|
||||
"""Every ready registry object covers all enforced ready-state foreign keys.
|
||||
|
||||
The operator separately proves a stopped-writing cut. This CLI neither
|
||||
claims that cut nor reads domain ORM models or changes a lifecycle state.
|
||||
"""
|
||||
settings = Settings()
|
||||
if settings.artifact_root is None:
|
||||
raise ValueError("artifact root required")
|
||||
engine = build_engine(settings)
|
||||
try:
|
||||
sessions = build_session_factory(engine)
|
||||
async with sessions.begin() as session:
|
||||
await session.execute(text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY"))
|
||||
rows = await session.execute(
|
||||
select(Artifact.digest, Artifact.bytes)
|
||||
.where(Artifact.state == "ready")
|
||||
.order_by(Artifact.digest)
|
||||
.limit(MAX_REFERENCES + 1)
|
||||
)
|
||||
objects = list(rows)
|
||||
if len(objects) > MAX_REFERENCES:
|
||||
raise ValueError("backup registry count exceeds bound")
|
||||
references = []
|
||||
for digest, size in objects:
|
||||
reference = ArtifactReference(digest=digest, bytes=size)
|
||||
# A 64-KiB buffer checks actual bytes without source/case output.
|
||||
await asyncio.to_thread(verify_existing, settings.artifact_root, reference)
|
||||
references.append(reference.model_dump(mode="json"))
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"kind": "kf.private.artifact-backup-inventory",
|
||||
"read_only": True,
|
||||
"quiescence_verified": False,
|
||||
"boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(),
|
||||
"storage_identity": {
|
||||
"device": settings.artifact_root.stat().st_dev,
|
||||
"inode": settings.artifact_root.stat().st_ino,
|
||||
"uid": settings.artifact_root.stat().st_uid,
|
||||
},
|
||||
"objects": references,
|
||||
}
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Private read-only artifact backup inventory")
|
||||
parser.parse_args()
|
||||
try:
|
||||
result = asyncio.run(asyncio.wait_for(inventory(), timeout=300))
|
||||
print(json.dumps(result, separators=(",", ":"), sort_keys=True))
|
||||
except (ValueError, OSError, SQLAlchemyError, TimeoutError):
|
||||
parser.exit(1, "Backup artifact inventory refused; no metadata or grade was modified.\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -50,6 +50,28 @@ class ArtifactRepository:
|
||||
raise ArtifactError("artifact metadata unavailable")
|
||||
return await asyncio.to_thread(self.store.read, reference)
|
||||
|
||||
async def put_file(
|
||||
self, session: AsyncSession, descriptor: int, reference: ArtifactReference
|
||||
) -> ArtifactReference:
|
||||
await lock_digest(session, reference.digest)
|
||||
existing = await session.get(Artifact, reference.digest)
|
||||
if existing is not None and (
|
||||
existing.state != "ready" or existing.bytes != reference.bytes
|
||||
):
|
||||
raise ArtifactError("artifact metadata unavailable")
|
||||
task = asyncio.create_task(asyncio.to_thread(self.store.put_file, descriptor, reference))
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
except BaseException:
|
||||
await task
|
||||
raise
|
||||
await session.execute(
|
||||
insert(Artifact)
|
||||
.values(digest=reference.digest, bytes=reference.bytes)
|
||||
.on_conflict_do_nothing(index_elements=[Artifact.digest])
|
||||
)
|
||||
return reference
|
||||
|
||||
async def check_metadata(self, session: AsyncSession, reference: ArtifactReference) -> None:
|
||||
metadata = await session.get(Artifact, reference.digest)
|
||||
if metadata is None or metadata.state != "ready" or metadata.bytes != reference.bytes:
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Owned, private, bounded import staging. A kernel lock survives parent/child handoff."""
|
||||
|
||||
import fcntl
|
||||
import os
|
||||
import stat
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
from kf_oj.contracts.admin_problems import MAX_ARCHIVE_FILES, STAGING_MAX_BYTES
|
||||
from kf_oj.infra.artifacts.store import ArtifactError
|
||||
|
||||
|
||||
class ImportStaging:
|
||||
def __init__(self, root: Path):
|
||||
root = root.absolute()
|
||||
root.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||
if root.resolve() != root:
|
||||
raise ArtifactError("invalid staging root")
|
||||
self.root = root
|
||||
self.fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
self.directory(self.fd)
|
||||
|
||||
@staticmethod
|
||||
def directory(fd):
|
||||
s = os.fstat(fd)
|
||||
if (
|
||||
not stat.S_ISDIR(s.st_mode)
|
||||
or s.st_uid != os.getuid()
|
||||
or stat.S_IMODE(s.st_mode) != 0o700
|
||||
):
|
||||
raise ArtifactError("invalid staging directory")
|
||||
|
||||
def close(self):
|
||||
os.close(self.fd)
|
||||
|
||||
@staticmethod
|
||||
def names(descriptor, maximum):
|
||||
result = []
|
||||
with os.scandir(descriptor) as entries:
|
||||
for entry in entries:
|
||||
if len(result) >= maximum:
|
||||
raise ArtifactError("staging file capacity exhausted")
|
||||
result.append(entry.name)
|
||||
return result
|
||||
|
||||
def operation_ids(self):
|
||||
identifiers = []
|
||||
for name in self.names(self.fd, 33):
|
||||
if name == ".lock":
|
||||
continue
|
||||
try:
|
||||
identifier = UUID(name)
|
||||
if str(identifier) != name:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise ArtifactError("unexpected staging entry") from None
|
||||
identifiers.append(identifier)
|
||||
return identifiers
|
||||
|
||||
@contextmanager
|
||||
def exclusive(self):
|
||||
lock = os.open(
|
||||
".lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=self.fd
|
||||
)
|
||||
try:
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(os.fstat(lock))
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise ArtifactError("import capacity busy") from None
|
||||
yield lock
|
||||
finally:
|
||||
os.close(lock)
|
||||
|
||||
def usage(self):
|
||||
total, count = 0, 0
|
||||
for name in self.names(self.fd, 33):
|
||||
if name == ".lock":
|
||||
continue
|
||||
try:
|
||||
if str(UUID(name)) != name:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise ArtifactError("unexpected staging entry") from None
|
||||
descriptor = self.operation(UUID(name))
|
||||
try:
|
||||
for entry in self.names(descriptor, 3):
|
||||
count += 1
|
||||
if count > 2 * (MAX_ARCHIVE_FILES + 2):
|
||||
raise ArtifactError("staging file capacity exhausted")
|
||||
if entry == "members":
|
||||
child = self.members(descriptor)
|
||||
try:
|
||||
for member in self.names(child, MAX_ARCHIVE_FILES):
|
||||
count += 1
|
||||
info = os.stat(member, dir_fd=child, follow_symlinks=False)
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(info)
|
||||
total += info.st_size
|
||||
finally:
|
||||
os.close(child)
|
||||
elif entry in ("archive", "validated.json"):
|
||||
info = os.stat(entry, dir_fd=descriptor, follow_symlinks=False)
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(info)
|
||||
total += info.st_size
|
||||
else:
|
||||
raise ArtifactError("unexpected staging entry")
|
||||
if count > 2 * (MAX_ARCHIVE_FILES + 2) or total > STAGING_MAX_BYTES:
|
||||
raise ArtifactError("staging capacity exhausted")
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
return total
|
||||
|
||||
def operation(self, operation_id: UUID, *, create=False):
|
||||
name = str(operation_id)
|
||||
if create:
|
||||
try:
|
||||
os.mkdir(name, 0o700, dir_fd=self.fd)
|
||||
except FileExistsError:
|
||||
pass
|
||||
descriptor = os.open(
|
||||
name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=self.fd
|
||||
)
|
||||
try:
|
||||
self.directory(descriptor)
|
||||
except Exception:
|
||||
os.close(descriptor)
|
||||
raise
|
||||
return descriptor
|
||||
|
||||
@staticmethod
|
||||
def members(operation, *, create=False):
|
||||
if create:
|
||||
os.mkdir("members", 0o700, dir_fd=operation)
|
||||
descriptor = os.open(
|
||||
"members", os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=operation
|
||||
)
|
||||
try:
|
||||
ImportStaging.directory(descriptor)
|
||||
except Exception:
|
||||
os.close(descriptor)
|
||||
raise
|
||||
return descriptor
|
||||
|
||||
def remove(self, operation_id):
|
||||
try:
|
||||
descriptor = self.operation(operation_id)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
try:
|
||||
names = self.names(descriptor, 3)
|
||||
if len(names) > 3:
|
||||
raise ArtifactError("unexpected staging entry")
|
||||
for name in names:
|
||||
if name == "members":
|
||||
child = self.members(descriptor)
|
||||
try:
|
||||
entries = self.names(child, MAX_ARCHIVE_FILES)
|
||||
if len(entries) > MAX_ARCHIVE_FILES:
|
||||
raise ArtifactError("unexpected staging entry")
|
||||
for member in entries:
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(
|
||||
os.stat(member, dir_fd=child, follow_symlinks=False)
|
||||
)
|
||||
os.unlink(member, dir_fd=child)
|
||||
finally:
|
||||
os.close(child)
|
||||
os.rmdir("members", dir_fd=descriptor)
|
||||
elif name in ("archive", "validated.json"):
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(os.stat(name, dir_fd=descriptor, follow_symlinks=False))
|
||||
os.unlink(name, dir_fd=descriptor)
|
||||
else:
|
||||
raise ArtifactError("unexpected staging entry")
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
os.rmdir(str(operation_id), dir_fd=self.fd)
|
||||
os.fsync(self.fd)
|
||||
|
||||
def reset_validation(self, operation_id):
|
||||
directory = self.operation(operation_id)
|
||||
try:
|
||||
names = self.names(directory, 3)
|
||||
if "members" in names:
|
||||
members = self.members(directory)
|
||||
try:
|
||||
for name in self.names(members, MAX_ARCHIVE_FILES):
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
|
||||
ArtifactStore._regular(os.stat(name, dir_fd=members, follow_symlinks=False))
|
||||
os.unlink(name, dir_fd=members)
|
||||
finally:
|
||||
os.close(members)
|
||||
os.rmdir("members", dir_fd=directory)
|
||||
if "validated.json" in names:
|
||||
os.unlink("validated.json", dir_fd=directory)
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
@@ -168,6 +168,135 @@ class ArtifactStore:
|
||||
with self.exclusive():
|
||||
return self._read(reference)
|
||||
|
||||
def verify(self, reference: ArtifactReference) -> None:
|
||||
"""Verify publication inputs with a 64 KiB buffer, without exposing hidden bytes."""
|
||||
ArtifactReference.model_validate(reference)
|
||||
with self.exclusive():
|
||||
try:
|
||||
descriptor = os.open(
|
||||
reference.digest,
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||
dir_fd=self._directory,
|
||||
)
|
||||
with os.fdopen(descriptor, "rb") as source:
|
||||
before = os.fstat(source.fileno())
|
||||
self._regular(before)
|
||||
if before.st_size != reference.bytes:
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
digest = hashlib.sha256()
|
||||
remaining = reference.bytes
|
||||
while remaining:
|
||||
data = source.read(min(65536, remaining))
|
||||
if not data:
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
digest.update(data)
|
||||
remaining -= len(data)
|
||||
after = os.fstat(source.fileno())
|
||||
if (
|
||||
source.read(1)
|
||||
or digest.hexdigest() != reference.digest
|
||||
or any(
|
||||
getattr(before, k) != getattr(after, k)
|
||||
for k in (
|
||||
"st_dev",
|
||||
"st_ino",
|
||||
"st_nlink",
|
||||
"st_mode",
|
||||
"st_uid",
|
||||
"st_size",
|
||||
"st_mtime_ns",
|
||||
"st_ctime_ns",
|
||||
)
|
||||
)
|
||||
):
|
||||
raise ArtifactError("artifact input changed")
|
||||
except OSError:
|
||||
raise ArtifactError("artifact unavailable") from None
|
||||
|
||||
def put_file(self, source: int, expected: ArtifactReference) -> ArtifactReference:
|
||||
"""Adopt a validated private FD without buffering a hidden testcase in memory."""
|
||||
before = os.fstat(source)
|
||||
self._regular(before)
|
||||
if before.st_size != expected.bytes:
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
|
||||
def unchanged(descriptor, original):
|
||||
current = os.fstat(descriptor)
|
||||
fields = (
|
||||
"st_dev",
|
||||
"st_ino",
|
||||
"st_nlink",
|
||||
"st_mode",
|
||||
"st_uid",
|
||||
"st_size",
|
||||
"st_mtime_ns",
|
||||
"st_ctime_ns",
|
||||
)
|
||||
if any(getattr(current, k) != getattr(original, k) for k in fields):
|
||||
raise ArtifactError("artifact input changed")
|
||||
|
||||
def stream(descriptor, *, output=None):
|
||||
digest = hashlib.sha256()
|
||||
for offset in range(0, expected.bytes, 65536):
|
||||
data = os.pread(descriptor, min(65536, expected.bytes - offset), offset)
|
||||
if len(data) != min(65536, expected.bytes - offset):
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
digest.update(data)
|
||||
if output is not None:
|
||||
output.write(data)
|
||||
if digest.hexdigest() != expected.digest:
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
|
||||
with self.exclusive():
|
||||
entries = self._entries()
|
||||
if any(name == expected.digest for name, _ in entries):
|
||||
existing = os.open(
|
||||
expected.digest,
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||
dir_fd=self._directory,
|
||||
)
|
||||
try:
|
||||
self._regular(os.fstat(existing))
|
||||
if os.fstat(existing).st_size != expected.bytes:
|
||||
raise ArtifactError("artifact digest or length mismatch")
|
||||
stream(source)
|
||||
stream(existing)
|
||||
finally:
|
||||
os.close(existing)
|
||||
else:
|
||||
if (
|
||||
len(entries) + 1 > self.max_files
|
||||
or sum(s.st_size for _, s in entries) + expected.bytes > self.max_bytes
|
||||
):
|
||||
raise ArtifactError("storage capacity exhausted")
|
||||
temporary = ".tmp-" + uuid4().hex
|
||||
descriptor = os.open(
|
||||
temporary,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
dir_fd=self._directory,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as out:
|
||||
stream(source, output=out)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
unchanged(source, before)
|
||||
os.rename(
|
||||
temporary,
|
||||
expected.digest,
|
||||
src_dir_fd=self._directory,
|
||||
dst_dir_fd=self._directory,
|
||||
)
|
||||
os.fsync(self._directory)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temporary, dir_fd=self._directory)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
unchanged(source, before)
|
||||
return expected
|
||||
|
||||
def remove(self, digest: str) -> None:
|
||||
if not DIGEST.fullmatch(digest):
|
||||
raise ArtifactError("invalid artifact identifier")
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Ordinary read-only view of the fixed protected Root admission record."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
from kf_oj.contracts.availability import AVAILABILITY_PATH, AvailabilityRecord
|
||||
|
||||
GATE = Path(AVAILABILITY_PATH)
|
||||
|
||||
|
||||
def protected_record() -> AvailabilityRecord:
|
||||
for path in GATE.parents:
|
||||
info = path.lstat()
|
||||
if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
|
||||
raise ValueError("protected availability directory required")
|
||||
fd = os.open(GATE, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if (
|
||||
not stat.S_ISREG(info.st_mode)
|
||||
or info.st_nlink != 1
|
||||
or info.st_uid != 0
|
||||
or info.st_gid != 0
|
||||
or stat.S_IMODE(info.st_mode) != 0o644
|
||||
or info.st_size > 4096
|
||||
):
|
||||
raise ValueError("protected availability file required")
|
||||
with os.fdopen(fd, "rb", closefd=False) as source:
|
||||
contents = source.read(4097)
|
||||
if len(contents) > 4096:
|
||||
raise ValueError("availability size bound")
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def unique(pairs):
|
||||
value = {}
|
||||
for key, item in pairs:
|
||||
if key in value:
|
||||
raise ValueError("duplicate availability field")
|
||||
value[key] = item
|
||||
return value
|
||||
|
||||
return AvailabilityRecord.model_validate(json.loads(contents, object_pairs_hook=unique))
|
||||
|
||||
|
||||
def current(release: str | None, batch: str | None) -> AvailabilityRecord:
|
||||
value = protected_record()
|
||||
if value.release_id != release or value.configuration_id != batch:
|
||||
raise ValueError("availability is not this running API binding")
|
||||
return value
|
||||
@@ -13,6 +13,7 @@ def build_engine(settings: Settings) -> AsyncEngine:
|
||||
max_overflow=0,
|
||||
pool_timeout=5,
|
||||
connect_args={
|
||||
"timeout": 5,
|
||||
"command_timeout": 10,
|
||||
"server_settings": {
|
||||
"statement_timeout": "10000",
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Read-only bounded outbox counts, without broker or delivery dependencies."""
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
|
||||
class OutboxQueries:
|
||||
def __init__(self, sessions):
|
||||
self.sessions = sessions
|
||||
|
||||
async def status(self):
|
||||
async with self.sessions() as session, session.begin():
|
||||
await session.execute(text("SET TRANSACTION READ ONLY"))
|
||||
await session.execute(text("SET LOCAL statement_timeout='1500'"))
|
||||
await session.execute(text("SET LOCAL lock_timeout='500'"))
|
||||
safe = await session.scalar(
|
||||
text("""
|
||||
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
|
||||
OR rolreplication OR rolbypassrls)
|
||||
AND pg_has_role(current_user,'kf_oj_platform','USAGE')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
|
||||
AND NOT has_column_privilege(current_user,'evaluations','state','UPDATE')
|
||||
FROM pg_roles WHERE rolname=session_user
|
||||
""")
|
||||
)
|
||||
if safe is not True:
|
||||
raise ValueError("dedicated outbox projection role required")
|
||||
row = (
|
||||
(
|
||||
await session.execute(
|
||||
text("""
|
||||
SELECT
|
||||
(SELECT count(*) FROM (SELECT 1 FROM outbox
|
||||
WHERE delivered_at IS NULL LIMIT 129) p) pending,
|
||||
(SELECT count(*) FROM (SELECT 1 FROM outbox WHERE delivered_at IS NULL
|
||||
AND locked_until>clock_timestamp() LIMIT 129) l) locked
|
||||
""")
|
||||
)
|
||||
)
|
||||
.mappings()
|
||||
.one()
|
||||
)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"pending_capped": min(row["pending"], 128),
|
||||
"locked_capped": min(row["locked"], 128),
|
||||
"truncated": row["pending"] > 128 or row["locked"] > 128,
|
||||
}
|
||||
@@ -11,6 +11,8 @@ from sqlalchemy import text
|
||||
from kf_oj.contracts.judge import JudgeTaskNotification
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
from kf_oj.infra.messaging.outbox_queries import OutboxQueries
|
||||
from kf_oj.infra.messaging.publisher_diagnostics import emit_failure
|
||||
|
||||
QUEUE = "kf.judge.evaluations.v1"
|
||||
QUEUE_ARGUMENTS = {
|
||||
@@ -25,6 +27,9 @@ class OutboxRepository:
|
||||
def __init__(self, sessions):
|
||||
self.sessions = sessions
|
||||
|
||||
async def status(self):
|
||||
return await OutboxQueries(self.sessions).status()
|
||||
|
||||
async def claim(self):
|
||||
owner = uuid4()
|
||||
async with self.sessions.begin() as s:
|
||||
@@ -124,6 +129,7 @@ class ConfirmedBroker:
|
||||
|
||||
|
||||
async def run(settings: Settings, stop: asyncio.Event):
|
||||
_startup_phase = "configuration"
|
||||
if settings.amqp_url is None:
|
||||
raise ValueError("AMQP configuration required")
|
||||
engine = build_engine(settings)
|
||||
@@ -134,7 +140,9 @@ async def run(settings: Settings, stop: asyncio.Event):
|
||||
for name in ("aio_pika", "aiormq", "pamqp"):
|
||||
logging.getLogger(name).disabled = True
|
||||
try:
|
||||
_startup_phase = "database_connect"
|
||||
async with engine.connect() as c:
|
||||
_startup_phase = "database_authority"
|
||||
allowed = await c.scalar(
|
||||
text("""SELECT NOT r.rolsuper AND NOT r.rolbypassrls
|
||||
AND pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
||||
@@ -143,6 +151,7 @@ async def run(settings: Settings, stop: asyncio.Event):
|
||||
)
|
||||
if not allowed:
|
||||
raise ValueError("dedicated platform database role required")
|
||||
_startup_phase = "running"
|
||||
while not stop.is_set():
|
||||
try:
|
||||
item = await repository.claim()
|
||||
@@ -175,7 +184,11 @@ def main():
|
||||
|
||||
try:
|
||||
asyncio.run(entry())
|
||||
except Exception:
|
||||
except Exception as error:
|
||||
try:
|
||||
emit_failure(error, run.__code__)
|
||||
except BaseException:
|
||||
pass # Diagnostics must never replace the original fatal result.
|
||||
raise SystemExit("outbox unavailable") from None
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Bounded outbox failure locations; never serialize errors or connection inputs."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import PurePath
|
||||
from types import CodeType
|
||||
|
||||
PREFIX = "KF_OUTBOX_START_FAILURE "
|
||||
PHASES = {"configuration", "database_connect", "database_authority", "running"}
|
||||
ERROR_TYPES = {
|
||||
"ValueError",
|
||||
"ValidationError",
|
||||
"TimeoutError",
|
||||
"OSError",
|
||||
"PermissionError",
|
||||
"ConnectionRefusedError",
|
||||
"ConnectionResetError",
|
||||
"MemoryError",
|
||||
"RuntimeError",
|
||||
"InvalidPasswordError",
|
||||
"InvalidAuthorizationSpecificationError",
|
||||
"CannotConnectNowError",
|
||||
"InterfaceError",
|
||||
"OperationalError",
|
||||
"DBAPIError",
|
||||
"ProgrammingError",
|
||||
"DataError",
|
||||
"InternalServerError",
|
||||
"PostgresError",
|
||||
"InvalidCatalogNameError",
|
||||
"InsufficientPrivilegeError",
|
||||
"TooManyConnectionsError",
|
||||
"QueryCanceledError",
|
||||
"ConnectionDoesNotExistError",
|
||||
"InternalClientError",
|
||||
"ReadOnlySQLTransactionError",
|
||||
}
|
||||
FILES = {
|
||||
"publisher.py",
|
||||
"session.py",
|
||||
"settings.py",
|
||||
"base_events.py",
|
||||
"runners.py",
|
||||
"tasks.py",
|
||||
"engine.py",
|
||||
"base.py",
|
||||
"default.py",
|
||||
"connection.py",
|
||||
"transaction.py",
|
||||
"connect_utils.py",
|
||||
"asyncpg.py",
|
||||
"_concurrency_py3k.py",
|
||||
"impl.py",
|
||||
"selector_events.py",
|
||||
"main.py",
|
||||
}
|
||||
FUNCTIONS = {
|
||||
"main",
|
||||
"entry",
|
||||
"run",
|
||||
"build_engine",
|
||||
"connect",
|
||||
"execute",
|
||||
"scalar",
|
||||
"_execute",
|
||||
"__execute",
|
||||
"_do_execute",
|
||||
"_execute_context",
|
||||
"_exec_single_context",
|
||||
"_handle_dbapi_exception",
|
||||
"_wrap_pool_connect",
|
||||
"raw_connection",
|
||||
"run_until_complete",
|
||||
"wait_for",
|
||||
"__aenter__",
|
||||
"__aexit__",
|
||||
"_connect",
|
||||
"_connect_addr",
|
||||
"__connect_addr",
|
||||
"_get_pool_impl",
|
||||
"_create_connection",
|
||||
"_do_get",
|
||||
"_invoke_creator",
|
||||
"await_only",
|
||||
"greenlet_spawn",
|
||||
"_connection_for_bind",
|
||||
"start",
|
||||
"_async_ping",
|
||||
"_do_ping_w_event",
|
||||
"__init__",
|
||||
"_sock_connect_cb",
|
||||
"sock_connect",
|
||||
"_ensure_connected",
|
||||
"_get_statement",
|
||||
"_execute_internal",
|
||||
"close",
|
||||
"dispose",
|
||||
"_cleanup_stmts",
|
||||
}
|
||||
|
||||
|
||||
def failure_record(error: Exception, run_code: CodeType) -> dict:
|
||||
"""Read existing exception/frame metadata, without altering the failed operation."""
|
||||
pending: list[tuple[str, BaseException]] = [("error", error)]
|
||||
seen = set()
|
||||
records = []
|
||||
phase = "unknown"
|
||||
while pending and len(records) < 4:
|
||||
relation, item = pending.pop(0)
|
||||
if id(item) in seen:
|
||||
continue
|
||||
seen.add(id(item))
|
||||
kind = type(item).__name__
|
||||
state = getattr(item, "sqlstate", None)
|
||||
if type(state) is not str or re.fullmatch(r"[A-Z0-9]{5}", state) is None:
|
||||
state = None
|
||||
frames = []
|
||||
trace = item.__traceback__
|
||||
depth = 0
|
||||
while trace is not None and depth < 32:
|
||||
code = trace.tb_frame.f_code
|
||||
if code is run_code:
|
||||
actual = trace.tb_frame.f_locals.get("_startup_phase")
|
||||
if type(actual) is str and actual in PHASES:
|
||||
phase = actual
|
||||
name = PurePath(code.co_filename).name
|
||||
if name in FILES and code.co_name in FUNCTIONS and 1 <= trace.tb_lineno <= 100000:
|
||||
frames.append({"file": name, "line": trace.tb_lineno, "function": code.co_name})
|
||||
trace = trace.tb_next
|
||||
depth += 1
|
||||
records.append(
|
||||
{
|
||||
"relation": relation,
|
||||
"type": kind if kind in ERROR_TYPES else "other",
|
||||
"sqlstate": state,
|
||||
"frames": frames[-4:],
|
||||
"frames_truncated": len(frames) > 4 or trace is not None,
|
||||
}
|
||||
)
|
||||
for link in ("orig", "__cause__", "__context__"):
|
||||
child = getattr(item, link, None)
|
||||
if isinstance(child, BaseException) and id(child) not in seen:
|
||||
pending.append(
|
||||
({"orig": "orig", "__cause__": "cause", "__context__": "context"}[link], child)
|
||||
)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"component": "outbox",
|
||||
"phase": phase,
|
||||
"errors": records,
|
||||
"chain_truncated": any(id(item) not in seen for _, item in pending),
|
||||
}
|
||||
|
||||
|
||||
def emit_failure(error: Exception, run_code: CodeType) -> None:
|
||||
encoded = json.dumps(failure_record(error, run_code), separators=(",", ":"), ensure_ascii=True)
|
||||
if len(encoded) > 3072:
|
||||
raise ValueError("outbox diagnostic bound exceeded")
|
||||
print(PREFIX + encoded, file=sys.stderr, flush=True)
|
||||
@@ -4,7 +4,7 @@
|
||||
`public.py` 提供已验证主体,`service.py` 管理账号和可撤销会话,`repository.py` 封装持久化。
|
||||
用户身份不能来自前端自报字段或模型参数。
|
||||
|
||||
本地管理命令为 `uv run --locked kf-accounts create <username>` / `disable <username>`。
|
||||
本地管理命令为 `uv run --locked kf-accounts create <username>`。
|
||||
创建口令由隐藏终端提示读取和确认,不接受命令行参数、环境中的默认口令或自助注册。
|
||||
用户名为 3–24 个小写 ASCII 字母、数字或下划线;口令 12–128 字符,最多 512 UTF-8 字节。
|
||||
密码摘要由新增的 `argon2-cffi` 依赖提供 Argon2id,固定 19 MiB/2 次/1 并行度。
|
||||
@@ -14,6 +14,39 @@
|
||||
登录接口为 `/api/v1/auth/login`,退出 `/logout`,当前主体 `/me`。会话使用随机 256-bit token,
|
||||
数据库只保存 token / CSRF 摘要;默认 12 小时有效,单账号保留至多 10 个有效会话。
|
||||
账号禁用立即拒绝登录和旧会话。请求只采用 ASGI 连接 peer,不读取客户端转发地址。
|
||||
|
||||
`0005_identity_authority` 为旧账号保留普通用户默认值,并加入管理员标记与服务端派生的授权版本。
|
||||
启用状态或管理员标记实际变化时,同一事务递增授权版本并撤销旧会话;事务回滚也回滚撤销。
|
||||
平台账号仅可插入普通账号列、更新启用状态,不能写管理员标记或授权版本;管理员的启用状态
|
||||
需要独立本地运维身份。`kf_oj_identity_operator` 与 `kf_oj_problem_import` 为无登录的独立权限组,
|
||||
口令与登录身份由运维另行配置,平台不继承运维组。导入组目前没有表权限,后续导入迁移只授予
|
||||
队列与候选所需权限,不授予评测状态写入权。迁移不创建默认管理员。
|
||||
保留授权或撤销历史后拒绝回退这次迁移。
|
||||
|
||||
`0006_authority_operations` 将运维组进一步收敛为固定函数的 EXECUTE 权限;不能直接读取
|
||||
口令/会话摘要或修改账号/会话。函数由独立 NOLOGIN 角色持有,仅有授权所需列权限,固定
|
||||
`search_path` 并逐项限定操作。所有账号写操作在行锁前更新同一个事务守卫;撤销/禁用/删除
|
||||
最后一名启用管理员由数据库拒绝。重复授予/撤销不递增授权版本,实际变更仍原子撤销会话。
|
||||
旧可重复读快照的并发变更产生序列化失败,调用方须结束整个事务后重新操作;不吞掉失败。
|
||||
并发语义参见[PostgreSQL事务隔离](https://www.postgresql.org/docs/18/transaction-iso.html),
|
||||
固定函数的权限与搜索路径参见[CREATE FUNCTION](https://www.postgresql.org/docs/18/sql-createfunction.html)。
|
||||
|
||||
在单独的本地运维环境配置 `KF_OJ_IDENTITY_OPERATOR_DATABASE_URL`,使用具有
|
||||
`kf_oj_identity_operator` 权限组的普通 LOGIN,禁止超级用户、建库/建角色/复制/绕过 RLS
|
||||
以及平台/协调器/导入组成员。API、Worker 和常规服务不配置此凭据,CLI 不回退平台身份。
|
||||
先显式创建普通账号,再运行 `uv run --locked kf-accounts bootstrap-admin <username>`。
|
||||
没有启用管理员时只有明确 bootstrap 可首次授权;不创建默认账号或口令。已有管理员后使用
|
||||
`grant-admin` / `revoke-admin` / `disable` / `enable`。恢复已禁用管理员使用 `enable`;恢复失去
|
||||
全部管理员的受控数据库备份时可再次显式 bootstrap。最后一名启用管理员不能被撤销或禁用,
|
||||
应先授予另一名账号。操作的锁等待最多 2 秒、SQL 最多 5 秒,CLI 只输出账号/能力/授权版本和
|
||||
是否变化;错误不输出连接串、SQL 参数或驱动诊断。网页管理 API 与题包导入仍待后续实现。
|
||||
|
||||
当前主体与登录响应附带版本为 1 的 `capabilities`,其中维护能力和十进制授权版本来自当前
|
||||
有效会话关联的账号;浏览器只展示这些信息,不能提交它们来获得授权。模块公开端口
|
||||
`ProblemMaintenancePort.require_problem_maintenance` 在调用方写事务中重新检查账号、授权版本
|
||||
与实时会话,并按账号→会话的顺序持有 SHARE 锁直到提交/回滚,防止验证之后撤权的竞态。
|
||||
普通构造的 `Principal` 默认没有维护能力;会话摘要仅在服务端主体内传递,不进入 DTO 或 repr。
|
||||
|
||||
受控启动使用 `--no-proxy-headers`,避免服务器中间件预先替换 peer;反向代理后的用户共享
|
||||
该连接预算,引入可信代理的独立限流策略之前不能直接打开客户端转发头信任。
|
||||
每 peer 至多 10 次登录/15 分钟,预算通过真实数据库串行更新,失败尝试也消耗预算。
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Local-only composition for fixed database authority operations."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from kf_oj.modules.identity.passwords import USERNAME
|
||||
|
||||
AuthorityOperation = Literal["bootstrap", "grant", "revoke", "enable", "disable"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AuthorityResult:
|
||||
account_id: UUID
|
||||
username: str
|
||||
enabled: bool
|
||||
is_administrator: bool
|
||||
authorization_version: int
|
||||
changed: bool
|
||||
|
||||
|
||||
async def manage_authority(
|
||||
sessions: async_sessionmaker, username: str, operation: AuthorityOperation
|
||||
) -> AuthorityResult:
|
||||
if not USERNAME.fullmatch(username) or operation not in (
|
||||
"bootstrap",
|
||||
"grant",
|
||||
"revoke",
|
||||
"enable",
|
||||
"disable",
|
||||
):
|
||||
raise ValueError("invalid authority operation")
|
||||
try:
|
||||
async with sessions.begin() as session:
|
||||
await session.execute(text("SET LOCAL statement_timeout='5s'"))
|
||||
await session.execute(text("SET LOCAL lock_timeout='2s'"))
|
||||
# The CLI must use a dedicated, unprivileged local operator login.
|
||||
# Web/platform, import and coordinator credentials are never reused.
|
||||
safe = await session.scalar(
|
||||
text("""
|
||||
SELECT NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication
|
||||
OR rolbypassrls)
|
||||
AND rolcanlogin AND pg_has_role(current_user,'kf_oj_identity_operator','USAGE')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_problem_import','MEMBER')
|
||||
FROM pg_roles WHERE rolname=current_user
|
||||
""")
|
||||
)
|
||||
if not safe:
|
||||
raise ValueError("a dedicated unprivileged identity operator login is required")
|
||||
row = (
|
||||
(
|
||||
await session.execute(
|
||||
text(
|
||||
"SELECT * FROM public.kf_manage_account_authority(:username,:operation)"
|
||||
),
|
||||
{"username": username, "operation": operation},
|
||||
)
|
||||
)
|
||||
.mappings()
|
||||
.one()
|
||||
)
|
||||
return AuthorityResult(**row)
|
||||
except DBAPIError as error:
|
||||
code = getattr(error.orig, "sqlstate", None)
|
||||
messages = {
|
||||
"P0002": "account not found",
|
||||
"23514": (
|
||||
"authority change rejected; preserve an enabled administrator "
|
||||
"and use explicit bootstrap"
|
||||
),
|
||||
"40001": "concurrent authority change rejected; retry the whole operation",
|
||||
"55P03": "authority operation busy; retry later",
|
||||
"57014": "authority operation timed out; verify current state before retrying",
|
||||
"42501": "identity operator permissions unavailable",
|
||||
}
|
||||
# Never print a connection URL, SQL bind values or driver diagnostics.
|
||||
raise ValueError(messages.get(code, "authority database operation unavailable")) from None
|
||||
@@ -8,28 +8,58 @@ from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
from kf_oj.modules.identity.authority import manage_authority
|
||||
from kf_oj.modules.identity.service import IdentityService
|
||||
|
||||
|
||||
async def run(command: str, username: str, password: str | None = None) -> None:
|
||||
engine = build_engine(Settings())
|
||||
settings = Settings()
|
||||
authority_commands = {
|
||||
"bootstrap-admin": "bootstrap",
|
||||
"grant-admin": "grant",
|
||||
"revoke-admin": "revoke",
|
||||
"enable": "enable",
|
||||
"disable": "disable",
|
||||
}
|
||||
if command != "create" and command not in authority_commands:
|
||||
raise ValueError("unknown account command")
|
||||
# Ordinary account creation uses the platform identity; authority management
|
||||
# always requires a separately configured login and never falls back to it.
|
||||
if command in authority_commands:
|
||||
if settings.identity_operator_database_url is None:
|
||||
raise ValueError("KF_OJ_IDENTITY_OPERATOR_DATABASE_URL is required")
|
||||
if settings.identity_operator_database_url == settings.database_url:
|
||||
raise ValueError("operator and platform database identities must be separate")
|
||||
settings = settings.model_copy(
|
||||
update={"database_url": settings.identity_operator_database_url}
|
||||
)
|
||||
engine = build_engine(settings)
|
||||
try:
|
||||
if command in authority_commands:
|
||||
result = await manage_authority(
|
||||
build_session_factory(engine), username, authority_commands[command]
|
||||
)
|
||||
print(
|
||||
f"Account {result.username}: enabled={result.enabled}, "
|
||||
f"administrator={result.is_administrator}, "
|
||||
f"authorization_version={result.authorization_version}, changed={result.changed}."
|
||||
)
|
||||
return
|
||||
service = IdentityService(build_session_factory(engine))
|
||||
if command == "create":
|
||||
assert password is not None
|
||||
await service.create_account(username, password)
|
||||
print("Account created.")
|
||||
else:
|
||||
if not await service.disable_account(username):
|
||||
raise ValueError("account not found")
|
||||
print("Account disabled; sessions revoked.")
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Manage local OJ accounts")
|
||||
parser.add_argument("command", choices=("create", "disable"))
|
||||
parser.add_argument(
|
||||
"command",
|
||||
choices=("create", "disable", "enable", "bootstrap-admin", "grant-admin", "revoke-admin"),
|
||||
)
|
||||
parser.add_argument("username")
|
||||
args = parser.parse_args()
|
||||
password = None
|
||||
|
||||
@@ -3,24 +3,56 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import Boolean, CheckConstraint, DateTime, ForeignKey, Index, String, Text
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
SmallInteger,
|
||||
String,
|
||||
Text,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from kf_oj.infra.db.base import Base
|
||||
from kf_oj.infra.db.columns import CreatedAt
|
||||
|
||||
|
||||
class AuthorityGuard(Base):
|
||||
"""Migration-owned singleton; runtime roles have no access."""
|
||||
|
||||
__tablename__ = "identity_authority_guard"
|
||||
__table_args__ = (
|
||||
CheckConstraint("id = 1", name="singleton"),
|
||||
CheckConstraint("generation >= 0", name="generation"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
|
||||
generation: Mapped[int] = mapped_column(BigInteger, server_default="0")
|
||||
|
||||
|
||||
class Account(CreatedAt, Base):
|
||||
__tablename__ = "accounts"
|
||||
__table_args__ = (
|
||||
CheckConstraint("username ~ '^[a-z0-9_]{3,24}$'", name="canonical_username"),
|
||||
CheckConstraint("char_length(password_hash) BETWEEN 40 AND 256", name="password_hash_size"),
|
||||
CheckConstraint("authorization_version >= 1", name="authorization_version"),
|
||||
Index(
|
||||
"ix_accounts_active_administrators",
|
||||
"id",
|
||||
postgresql_where=text("is_administrator AND enabled"),
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
username: Mapped[str] = mapped_column(String(24), unique=True)
|
||||
password_hash: Mapped[str] = mapped_column(Text)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, server_default="true")
|
||||
is_administrator: Mapped[bool] = mapped_column(Boolean, server_default="false")
|
||||
authorization_version: Mapped[int] = mapped_column(BigInteger, server_default="1")
|
||||
|
||||
|
||||
class Session(CreatedAt, Base):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Only verified principals cross business module boundaries."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
@@ -8,11 +9,30 @@ from uuid import UUID
|
||||
class Principal:
|
||||
account_id: UUID
|
||||
username: str
|
||||
can_maintain_problems: bool = False
|
||||
authorization_version: int = 1
|
||||
session_token_hash: str | None = field(default=None, repr=False, compare=False)
|
||||
|
||||
|
||||
class MaintenanceDenied(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ProblemMaintenancePort(Protocol):
|
||||
async def require_problem_maintenance(self, session, principal: Principal) -> None:
|
||||
"""Recheck and lock live authority/session inside the caller's write transaction."""
|
||||
...
|
||||
|
||||
|
||||
class AuthenticationFailed(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ReservedMaintenancePort(Protocol):
|
||||
async def require_reserved_maintenance(self, session, actor_id: UUID, epoch: int) -> None:
|
||||
"""Recheck/lock an immutable accepted receipt, with no login/session secret access."""
|
||||
...
|
||||
|
||||
|
||||
class LoginThrottled(ValueError):
|
||||
pass
|
||||
|
||||
@@ -51,19 +51,31 @@ async def account_by_username(session: AsyncSession, username: str) -> Account |
|
||||
return await session.scalar(select(Account).where(Account.username == username))
|
||||
|
||||
|
||||
async def current_session(session: AsyncSession, token_hash: str):
|
||||
return (
|
||||
await session.execute(
|
||||
select(Session, Account)
|
||||
.join(Account)
|
||||
.where(
|
||||
Session.token_hash == token_hash,
|
||||
Session.revoked_at.is_(None),
|
||||
Session.expires_at > func.now(),
|
||||
Account.enabled.is_(True),
|
||||
)
|
||||
async def lock_authority(session: AsyncSession, account_id) -> Account | None:
|
||||
return await session.scalar(
|
||||
select(Account)
|
||||
.where(Account.id == account_id)
|
||||
.with_for_update(read=True)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
|
||||
|
||||
async def current_session(session: AsyncSession, token_hash: str, *, lock_session: bool = False):
|
||||
query = (
|
||||
select(Session, Account)
|
||||
.join(Account)
|
||||
.where(
|
||||
Session.token_hash == token_hash,
|
||||
Session.revoked_at.is_(None),
|
||||
Session.expires_at > func.clock_timestamp(),
|
||||
Account.enabled.is_(True),
|
||||
)
|
||||
).one_or_none()
|
||||
)
|
||||
if lock_session:
|
||||
query = query.with_for_update(read=True, of=Session).execution_options(
|
||||
populate_existing=True
|
||||
)
|
||||
return (await session.execute(query)).one_or_none()
|
||||
|
||||
|
||||
async def prune_sessions(session: AsyncSession, account_id) -> None:
|
||||
|
||||
@@ -7,7 +7,8 @@ from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import func, select, text, update
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from kf_oj.modules.identity import repository
|
||||
@@ -18,7 +19,12 @@ from kf_oj.modules.identity.passwords import (
|
||||
validate_account,
|
||||
verify_password,
|
||||
)
|
||||
from kf_oj.modules.identity.public import AuthenticationFailed, LoginThrottled, Principal
|
||||
from kf_oj.modules.identity.public import (
|
||||
AuthenticationFailed,
|
||||
LoginThrottled,
|
||||
MaintenanceDenied,
|
||||
Principal,
|
||||
)
|
||||
|
||||
TOKEN = re.compile(r"[0-9a-f]{64}\Z")
|
||||
|
||||
@@ -107,7 +113,7 @@ class IdentityService:
|
||||
expires_at=expires,
|
||||
)
|
||||
)
|
||||
principal = Principal(account.id, account.username)
|
||||
principal = self._principal(account, token_digest(token))
|
||||
return IssuedSession(principal, expires, token, csrf)
|
||||
|
||||
async def authenticate(self, token: str | None, *, csrf_token: str | None = None) -> Principal:
|
||||
@@ -123,9 +129,62 @@ class IdentityService:
|
||||
record.csrf_hash, token_digest(csrf_token)
|
||||
):
|
||||
raise AuthenticationFailed("invalid request token")
|
||||
return Principal(account.id, account.username)
|
||||
return self._principal(account, record.token_hash)
|
||||
|
||||
@staticmethod
|
||||
def _principal(account: Account, session_token_hash: str) -> Principal:
|
||||
return Principal(
|
||||
account.id,
|
||||
account.username,
|
||||
account.is_administrator,
|
||||
account.authorization_version,
|
||||
session_token_hash,
|
||||
)
|
||||
|
||||
async def require_problem_maintenance(self, session, principal: Principal) -> None:
|
||||
if not session.in_transaction():
|
||||
raise ValueError("maintenance authorization requires an active write transaction")
|
||||
if (
|
||||
not principal.can_maintain_problems
|
||||
or principal.session_token_hash is None
|
||||
or not TOKEN.fullmatch(principal.session_token_hash)
|
||||
):
|
||||
raise MaintenanceDenied("problem maintenance required")
|
||||
# Account first, then session: authority changes use that same lock order.
|
||||
# SHARE locks last until the caller commits/rolls back; a stale principal,
|
||||
# explicit logout, revocation or expired session cannot authorize a write.
|
||||
account = await repository.lock_authority(session, principal.account_id)
|
||||
if (
|
||||
account is None
|
||||
or not account.enabled
|
||||
or not account.is_administrator
|
||||
or account.authorization_version != principal.authorization_version
|
||||
):
|
||||
raise MaintenanceDenied("problem maintenance required")
|
||||
row = await repository.current_session(
|
||||
session, principal.session_token_hash, lock_session=True
|
||||
)
|
||||
if row is None or row[1].id != account.id:
|
||||
raise MaintenanceDenied("problem maintenance required")
|
||||
|
||||
async def logout(self, token: str, csrf_token: str) -> None:
|
||||
await self.authenticate(token, csrf_token=csrf_token)
|
||||
async with self.sessions.begin() as session:
|
||||
await repository.revoke(session, token_digest(token))
|
||||
|
||||
|
||||
class ReservedMaintenance:
|
||||
"""Import worker uses only the fixed, narrow SQL authority port."""
|
||||
|
||||
async def require_reserved_maintenance(self, session, actor_id, epoch):
|
||||
if not session.in_transaction() or type(epoch) is not int or epoch < 1:
|
||||
raise MaintenanceDenied("invalid maintenance receipt")
|
||||
try:
|
||||
await session.execute(
|
||||
text("SELECT public.kf_lock_reserved_maintenance(:actor,:epoch)"),
|
||||
{"actor": actor_id, "epoch": epoch},
|
||||
)
|
||||
except DBAPIError as error:
|
||||
if getattr(error.orig, "sqlstate", None) == "42501":
|
||||
raise MaintenanceDenied("problem maintenance revoked") from None
|
||||
raise
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
# operations
|
||||
|
||||
负责平台侧节点、任务积压、重试原因和审计查询的管理视图。
|
||||
对判题状态的操作必须调用协调器协议;观测模块不成为第二个状态写入入口。
|
||||
当前只有目录边界,节点登记、指标聚合和管理员接口待实现。
|
||||
维护审计通过 `public.py` 的追加端口写入调用方事务;本模块不成为评测状态的写入入口。
|
||||
上传、解析完成/失败、丢弃、发布、下架与恢复记录真实账号、授权版本、操作 UUID、对象与
|
||||
数据库时间。发布回执和审计同时提交;审计失败使整个业务事务回滚,不确认半发布。
|
||||
|
||||
本机受控身份 CLI 使用已核验的窄权限数据库函数。授权变化由固定包装函数在同一事务
|
||||
追加审计,主体类型为 operator,操作者为数据库 `session_user`,对象为实际变更账号;
|
||||
不把授予/撤销冒充成该账号自己的操作。旧账号事件保留 account 主体类型。
|
||||
|
||||
管理员 `/api/v1/admin/maintenance-audit` 按时间和 UUID 的复合游标递减读取,每页至多
|
||||
100项,可选择有限动作类型;只展示已提交事件。导入失败动作仍是失败,committed 表示
|
||||
审计事务提交。响应不含口令、会话令牌、工件引用、内部路径或任意诊断正文。
|
||||
|
||||
应用数据库角色无审计 UPDATE/DELETE 权限,数据库触发器还阻止历史改写。
|
||||
导入 Worker 只有追加权限,不能读取整个维护审计;Coordinator 无审计写入权限。
|
||||
节点管理、部署控制及跨账号业务管理仍未开放。
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Append-only bounded maintenance events, independent of evaluation state."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import BigInteger, CheckConstraint, ForeignKey, Index, String, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from kf_oj.infra.db.base import Base
|
||||
from kf_oj.infra.db.columns import CreatedAt
|
||||
|
||||
|
||||
class MaintenanceEvent(CreatedAt, Base):
|
||||
__tablename__ = "problem_maintenance_events"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"action IN ('upload_created','upload_received','import_ready',"
|
||||
"'import_failed','import_discarded','import_expired','published',"
|
||||
"'disabled','restored','authority_bootstrap','authority_grant',"
|
||||
"'authority_revoke','authority_enable','authority_disable')",
|
||||
name="action",
|
||||
),
|
||||
CheckConstraint("authorization_version > 0", name="authority"),
|
||||
CheckConstraint(
|
||||
"(actor_kind='account' AND actor_id IS NOT NULL AND operator_name IS NULL) "
|
||||
"OR (actor_kind='operator' AND actor_id IS NULL AND operator_name IS NOT NULL)",
|
||||
name="actor",
|
||||
),
|
||||
Index("ix_problem_maintenance_events_page", "created_at", "id"),
|
||||
)
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
actor_id: Mapped[UUID | None] = mapped_column(ForeignKey("accounts.id"))
|
||||
actor_kind: Mapped[str] = mapped_column(
|
||||
String(8), default="account", server_default=text("'account'")
|
||||
)
|
||||
operator_name: Mapped[str | None] = mapped_column(String(63))
|
||||
authorization_version: Mapped[int] = mapped_column(BigInteger)
|
||||
operation_id: Mapped[UUID] = mapped_column()
|
||||
action: Mapped[str] = mapped_column(String(24))
|
||||
target_id: Mapped[UUID | None] = mapped_column()
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Accepted actor receipts and finite maintenance events; no private data/log text."""
|
||||
|
||||
from typing import Literal, Protocol
|
||||
from uuid import UUID
|
||||
|
||||
Action = Literal[
|
||||
"upload_created",
|
||||
"upload_received",
|
||||
"import_ready",
|
||||
"import_failed",
|
||||
"import_discarded",
|
||||
"import_expired",
|
||||
"published",
|
||||
"disabled",
|
||||
"restored",
|
||||
]
|
||||
|
||||
|
||||
class MaintenanceAuditPort(Protocol):
|
||||
async def record(
|
||||
self,
|
||||
session,
|
||||
actor_id: UUID,
|
||||
authorization_version: int,
|
||||
operation_id: UUID,
|
||||
action: Action,
|
||||
target_id: UUID | None = None,
|
||||
) -> None: ...
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Only authorized administrators read bounded append-only maintenance projections."""
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, text, tuple_
|
||||
|
||||
from kf_oj.contracts.problem_maintenance import MaintenanceAuditEvent, MaintenanceAuditPage
|
||||
from kf_oj.modules.identity.public import ProblemMaintenancePort
|
||||
from kf_oj.modules.operations.models import MaintenanceEvent
|
||||
|
||||
|
||||
class MaintenanceAuditQueries:
|
||||
def __init__(self, sessions, identity: ProblemMaintenancePort):
|
||||
self.sessions, self.identity = sessions, identity
|
||||
|
||||
async def page(self, principal, *, before=None, limit=20, action=None):
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError("invalid page limit")
|
||||
cursor = None
|
||||
if before is not None:
|
||||
stamp, identifier = before.split("|")
|
||||
stamp = datetime.fromisoformat(stamp)
|
||||
if stamp.tzinfo is None:
|
||||
raise ValueError("invalid audit cursor")
|
||||
cursor = stamp, UUID(identifier)
|
||||
async with self.sessions.begin() as session:
|
||||
await session.execute(text("SET LOCAL statement_timeout='5s'"))
|
||||
await session.execute(text("SET LOCAL lock_timeout='2s'"))
|
||||
await self.identity.require_problem_maintenance(session, principal)
|
||||
q = select(MaintenanceEvent)
|
||||
if cursor:
|
||||
q = q.where(tuple_(MaintenanceEvent.created_at, MaintenanceEvent.id) < cursor)
|
||||
if action:
|
||||
q = q.where(MaintenanceEvent.action == action)
|
||||
rows = list(
|
||||
await session.scalars(
|
||||
q.order_by(
|
||||
MaintenanceEvent.created_at.desc(), MaintenanceEvent.id.desc()
|
||||
).limit(limit + 1)
|
||||
)
|
||||
)
|
||||
visible = rows[:limit]
|
||||
return MaintenanceAuditPage(
|
||||
items=tuple(
|
||||
MaintenanceAuditEvent(
|
||||
id=r.id,
|
||||
created_at=r.created_at,
|
||||
actor_kind=r.actor_kind,
|
||||
actor_id=r.actor_id,
|
||||
operator_name=r.operator_name,
|
||||
authorization_version=str(r.authorization_version),
|
||||
operation_id=r.operation_id,
|
||||
action=r.action,
|
||||
target_id=r.target_id,
|
||||
)
|
||||
for r in visible
|
||||
),
|
||||
next_before=f"{visible[-1].created_at.isoformat()}|{visible[-1].id}"
|
||||
if len(rows) > limit
|
||||
else None,
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from kf_oj.modules.operations.models import MaintenanceEvent
|
||||
from kf_oj.modules.operations.public import Action
|
||||
|
||||
|
||||
class MaintenanceAudit:
|
||||
async def record(
|
||||
self, session, actor_id, authorization_version, operation_id, action: Action, target_id=None
|
||||
):
|
||||
if not session.in_transaction():
|
||||
raise ValueError("audit requires a write transaction")
|
||||
# Supplying the trusted DB timestamp avoids ORM INSERT ... RETURNING:
|
||||
# the ordinary importer has append-only authority and cannot SELECT this table.
|
||||
session.add(
|
||||
MaintenanceEvent(
|
||||
id=uuid4(),
|
||||
created_at=await session.scalar(select(func.clock_timestamp())),
|
||||
actor_id=actor_id,
|
||||
authorization_version=authorization_version,
|
||||
operation_id=operation_id,
|
||||
action=action,
|
||||
target_id=target_id,
|
||||
)
|
||||
)
|
||||
@@ -10,7 +10,7 @@
|
||||
要求配置 `KF_OJ_ARTIFACT_ROOT` 和可信 `KF_OJ_JUDGE_ADMISSION` JSON;同一准入配置应交付给节点,
|
||||
不能由题目放宽。结构示例来自共享 `judge-admission.valid.json`,不是当前宿主隔离验收证明。
|
||||
|
||||
清单固定 C17/C++20、工具链/镜像版本、精确字节比较、编译/运行/整个作业预算,并声明按序
|
||||
v1 清单固定 C17/C++20、工具链/镜像版本、精确字节比较、编译/运行/整个作业预算,并声明按序
|
||||
测试点 `{key,input_file,expected_file}` 和最多十对公开样例 `{input_file,output_file}`。
|
||||
题面以 `statement_file` 读取 UTF-8。每份样例至多 64 KiB,响应采用 base64 保留二进制数据。
|
||||
同一 slug 受事务锁保护,每次发布递增 revision;发布后题面、限额、样例、测试点不可修改。
|
||||
@@ -19,3 +19,62 @@
|
||||
`GET /api/v1/problems` 以 slug 游标分页(`after` / `limit`,最多 100 项),
|
||||
`GET /api/v1/problems/versions/{uuid}` 读取已发布版本。禁用逻辑题目时其版本对公开读取不可用。
|
||||
内部判题投影仅供授权应用编排使用;外部 Worker 读取由 submissions 的有效租约端口控制。
|
||||
|
||||
|
||||
v2 使用 `schema_version: 2` 与 `language_configurations`,每个配置包含 `language`、
|
||||
完整 `toolchain` 身份、`compile_limits`、`run_limits` 和 `job_wall_time_us`。
|
||||
可分别声明 `c17`、`cpp20`、`python313`、`java21`;不再同时提供 v1 顶层工具链或预算。
|
||||
所有语言按可信节点配置准入,缺失能力/错误补丁版本/镜像摘要/过宽预算在发布前拒绝。
|
||||
`judge.admission.example.json` 的镜像摘要为占位值,使用前替换为管理员制作镜像的实际摘要。
|
||||
公开题面仅显示固定语言、版本、入口与预算,不公布宿主镜像路径和隐藏工件。
|
||||
|
||||
导入同一 slug 的 v2 版本保持原 v1 版本与提交不变。已有提交按其保存的协议版本解析,
|
||||
不能重新序列化旧 canonical 数据来补入新字段。Java 使用 Main.java 与默认包 Main;
|
||||
Python 使用 main.py。每个新版本的语言集合独立,浏览器不得为旧版本补开新语言。
|
||||
|
||||
|
||||
Web题包导入是独立的待发布流程:`/api/v1/admin/problem-imports` 使用真实维护会话,
|
||||
写入还要求明确Origin与绑定CSRF。先以稳定操作UUID、ZIP摘要和精确字节数建立容量预约,
|
||||
再向该操作的 `/archive` 流式PUT;同键重试仍验证实际内容,不会确认异内容。
|
||||
列表、详情和丢弃只开放给该操作的管理员本人,响应仅含公开题面、样例、语言预算与有限错误。
|
||||
`ready` 尚未发布,不会加入学生题库,也不改变既有题目版本。
|
||||
|
||||
Web格式固定单层ZIP的stored/deflate,清单名为`problem.json`;公开文件与隐藏用例文件
|
||||
必须显式分别声明且不得混用。文件名只接受有界ASCII单层普通名;未知成员、链接/设备、
|
||||
加密、嵌套归档、额外前缀/尾随内容、损坏CRC与虚假大小被拒。旧受控目录CLI格式保持原义。
|
||||
压缩与实际解压各64 MiB、最多534文件,上传最多4096帧/每帧1 MiB/30秒;
|
||||
解析期限60秒。staging总预约256 MiB,每次128 MiB,单全局上传/解析槽位,
|
||||
本人最多10个/全局32个活跃候选,操作24小时到期,解析死亡最多领取两次。
|
||||
真实限额由管理capabilities返回;缺少配置或新鲜Worker资源证明时明确不可用。
|
||||
|
||||
`uv run --locked kf-problem-import-worker` 是普通用户、独立数据库登录的组合入口。
|
||||
配置`KF_OJ_IMPORT_STAGING_ROOT`、`KF_OJ_ARTIFACT_ROOT`、可信准入和
|
||||
`KF_OJ_IMPORT_WORKER_CGROUP`;staging与工件目录必须归服务用户所有且权限0700。
|
||||
服务不会sudo、执行上传内容/学生源码、联网安装依赖或读写评测状态。启动及每次解析
|
||||
读回128 MiB/swap0/PIDs16/CPU不超过一核及文件/FD/core上限,错误身份/无界环境拒绝服务。
|
||||
解析子进程继承同一个内核文件锁,父进程死亡不能让GC在存活子进程仍持有文件时释放容量。
|
||||
租约与fence防止旧结果写入;完整工件持久化、候选和引用原子落库后才返回ready。
|
||||
丢弃/失败/到期先清理私有暂存字节再释放预约,保留不可变操作记录与追加式有限审计。
|
||||
|
||||
资源约束服务模板见`deploy/problem-import-worker.service.example`。API和Worker需可访问
|
||||
同一普通用户的私有目录;数据库采用各自最小角色,导入Worker只通过固定身份公开端口
|
||||
验证已受理的维护授权。PrivateNetwork模板要求Unix PostgreSQL socket;实际部署前由运维
|
||||
设置外部私有配置、专用登录与目录,不包含默认口令,也不代表本轮已经生产部署。
|
||||
|
||||
管理员发布使用 `/api/v1/admin/problem-publications`,绑定独立操作 UUID、本人导入操作、
|
||||
候选 UUID 与预期活动版本(首次发布为 null)。`publication.py` 为 CLI 与 Web 共用的
|
||||
准入与不可变版本写入路径;Web 在事务内再次校验实时维护权限、候选摘要、工件实际字节
|
||||
和可信节点配置。CLI 保留本机受控入口与原有导入格式,不借用网页会话。
|
||||
|
||||
成功维护回执只追加保存,UUID 同内容重试返回原快照,不重复创建 revision 或审计。
|
||||
UUID 绑定不同内容或活动版本变化返回409;前端应核对最新状态后建立新的操作凭据。
|
||||
题目状态写入同样要求预期活动版本及 enabled;下架不修改历史版本或删除任何提交。
|
||||
新提交持有逻辑题目的共享锁直到提交事务结束,下架/恢复持有排他锁,二者明确排序。
|
||||
|
||||
管理列表按 slug、版本历史按递减 revision 分页,每页至多100条,仅返回公开元数据。
|
||||
候选仍通过本人导入操作读取;维护回执仅本人可读。隐藏测试字节、摘要和宿主路径不进入
|
||||
这些投影。所有管理读取也经实时维护权限检查,缺失发布配置明确503。
|
||||
|
||||
`for_accepted_execution` 仅在 submissions 已核对有效租约、源码归属及不可变尝试快照后
|
||||
调用,读取已发布原版本时允许逻辑题目已下架。它没有外部 UUID 直读端点,不放宽未发布
|
||||
版本或无有效租约的访问。公开读取与新提交仍检查 enabled。
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Pure data validation shared by the bounded child and fenced candidate transaction."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from uuid import uuid5
|
||||
|
||||
from kf_oj.contracts.admin_problems import CandidatePreview, LegacyImportBudget
|
||||
from kf_oj.contracts.judging import ArtifactReference, TestCase
|
||||
from kf_oj.contracts.problems import PublicLanguageConfiguration
|
||||
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport
|
||||
from kf_oj.modules.problems.publication import admit_version
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ValidatedCandidate:
|
||||
manifest: dict
|
||||
preview: CandidatePreview
|
||||
references: dict[str, ArtifactReference]
|
||||
|
||||
@property
|
||||
def digest(self):
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
self.manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=True
|
||||
).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def validate_candidate(manifest_data, preview_data, references, policy) -> ValidatedCandidate:
|
||||
manifest = ImportEnvelope.model_validate(manifest_data).root
|
||||
preview = CandidatePreview.model_validate(preview_data)
|
||||
refs = {k: ArtifactReference.model_validate(v) for k, v in references.items()}
|
||||
legacy = isinstance(manifest, ProblemImport)
|
||||
if (
|
||||
preview.slug != manifest.slug
|
||||
or preview.title != manifest.title
|
||||
or preview.case_count != len(manifest.cases)
|
||||
or preview.judging_schema_version != manifest.schema_version
|
||||
or tuple(v.value for v in preview.languages) != tuple(v.value for v in manifest.languages)
|
||||
):
|
||||
raise ValueError("invalid candidate projection")
|
||||
expected_budget = (
|
||||
LegacyImportBudget(
|
||||
compile_limits=manifest.compile_limits,
|
||||
run_limits=manifest.run_limits,
|
||||
job_wall_time_us=manifest.job_wall_time_us,
|
||||
)
|
||||
if legacy
|
||||
else None
|
||||
)
|
||||
configs = (
|
||||
()
|
||||
if legacy
|
||||
else tuple(
|
||||
PublicLanguageConfiguration.from_configuration(c)
|
||||
for c in manifest.language_configurations
|
||||
)
|
||||
)
|
||||
if preview.legacy_budget != expected_budget or preview.language_configurations != configs:
|
||||
raise ValueError("invalid candidate budget")
|
||||
names = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
|
||||
if set(refs) != names:
|
||||
raise ValueError("invalid hidden references")
|
||||
cases = tuple(
|
||||
TestCase(
|
||||
case_id=uuid5(preview.candidate_id, c.key),
|
||||
input=refs[c.input_file],
|
||||
expected=refs[c.expected_file],
|
||||
)
|
||||
for c in manifest.cases
|
||||
)
|
||||
admit_version(policy, manifest, preview.candidate_id, preview.candidate_id, cases)
|
||||
return ValidatedCandidate(manifest.model_dump(mode="json"), preview, refs)
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
from typing import Annotated, Literal, Self
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
from pydantic import Field, RootModel, model_validator
|
||||
|
||||
from kf_oj.contracts import judging_v2 as v2
|
||||
from kf_oj.contracts.judging import (
|
||||
MAX_CASES,
|
||||
MAX_JOB_WALL_US,
|
||||
@@ -52,3 +53,34 @@ class ProblemImport(WireModel):
|
||||
if len({(c.input_file, c.expected_file) for c in self.cases}) != len(self.cases):
|
||||
raise ValueError("duplicate testcase reference")
|
||||
return self
|
||||
|
||||
|
||||
class ProblemImportV2(WireModel):
|
||||
schema_version: v2.Version
|
||||
slug: Label
|
||||
title: Annotated[str, Field(strict=True, min_length=1, max_length=160)]
|
||||
statement_file: FileName
|
||||
language_configurations: Annotated[
|
||||
tuple[v2.LanguageConfiguration, ...], Field(min_length=1, max_length=4)
|
||||
]
|
||||
comparison: Literal["exact_bytes"]
|
||||
cases: Annotated[tuple[ImportCase, ...], Field(min_length=1, max_length=MAX_CASES)]
|
||||
samples: Annotated[tuple[ImportSample, ...], Field(max_length=10)]
|
||||
|
||||
@property
|
||||
def languages(self) -> tuple[v2.Language, ...]:
|
||||
return tuple(config.language for config in self.language_configurations)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def unique(self) -> Self:
|
||||
if len(set(self.languages)) != len(self.languages):
|
||||
raise ValueError("duplicate language configuration")
|
||||
if len({case.key for case in self.cases}) != len(self.cases):
|
||||
raise ValueError("duplicate testcase key")
|
||||
if len({(case.input_file, case.expected_file) for case in self.cases}) != len(self.cases):
|
||||
raise ValueError("duplicate testcase reference")
|
||||
return self
|
||||
|
||||
|
||||
class ImportEnvelope(RootModel[ProblemImport | ProblemImportV2]):
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Private durable imports and immutable candidates; no grading-state write authority."""
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
CheckConstraint,
|
||||
Computed,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
ForeignKeyConstraint,
|
||||
Index,
|
||||
Integer,
|
||||
SmallInteger,
|
||||
String,
|
||||
UniqueConstraint,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from kf_oj.infra.db.base import Base
|
||||
from kf_oj.infra.db.columns import CreatedAt
|
||||
from kf_oj.modules.problems.models import ProblemVersion
|
||||
|
||||
|
||||
class ImportPool(Base):
|
||||
__tablename__ = "problem_import_pool"
|
||||
__table_args__ = (
|
||||
CheckConstraint("id = 1", name="singleton"),
|
||||
CheckConstraint("reserved_bytes BETWEEN 0 AND 268435456", name="capacity"),
|
||||
)
|
||||
id: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
|
||||
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="0")
|
||||
worker_ready_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
worker_policy_digest: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class ProblemImportOperation(CreatedAt, Base):
|
||||
__tablename__ = "problem_import_operations"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"state IN ('uploading','uploaded','validating','ready','failed',"
|
||||
"'published','discarded','expired')",
|
||||
name="state",
|
||||
),
|
||||
CheckConstraint("archive_digest ~ '^[0-9a-f]{64}$'", name="digest"),
|
||||
CheckConstraint("archive_bytes BETWEEN 1 AND 67108864", name="bytes"),
|
||||
CheckConstraint("authorization_version > 0", name="authority"),
|
||||
CheckConstraint("attempts BETWEEN 0 AND 2 AND fence >= 0", name="attempts"),
|
||||
CheckConstraint("reserved_bytes IN (0,134217728)", name="reservation"),
|
||||
CheckConstraint("expires_at > created_at AND updated_at >= created_at", name="times"),
|
||||
Index("ix_problem_import_owner", "owner_id", "id"),
|
||||
Index("ix_problem_import_queue", "state", "created_at"),
|
||||
Index("ix_problem_import_expiry", "expires_at"),
|
||||
Index("ix_problem_import_reservations", "id", postgresql_where=text("reserved_bytes>0")),
|
||||
)
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
owner_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
|
||||
authorization_version: Mapped[int] = mapped_column(BigInteger)
|
||||
archive_digest: Mapped[str] = mapped_column(String(64))
|
||||
archive_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
state: Mapped[str] = mapped_column(String(16), server_default="uploading")
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
attempts: Mapped[int] = mapped_column(Integer, server_default="0")
|
||||
fence: Mapped[int] = mapped_column(BigInteger, server_default="0")
|
||||
reserved_bytes: Mapped[int] = mapped_column(BigInteger, server_default="134217728")
|
||||
failure_code: Mapped[str | None] = mapped_column(String(32))
|
||||
candidate_id: Mapped[UUID | None] = mapped_column(
|
||||
ForeignKey("problem_candidates.id", use_alter=True, name="fk_problem_import_candidate")
|
||||
)
|
||||
published_version_id: Mapped[UUID | None] = mapped_column(ForeignKey(ProblemVersion.id))
|
||||
|
||||
|
||||
class ProblemCandidate(CreatedAt, Base):
|
||||
__tablename__ = "problem_candidates"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("operation_id"),
|
||||
CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="digest"),
|
||||
CheckConstraint("case_count BETWEEN 1 AND 256", name="cases"),
|
||||
CheckConstraint("octet_length(preview::text) <= 4194304", name="preview"),
|
||||
CheckConstraint("octet_length(manifest::text) <= 262144", name="manifest"),
|
||||
)
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
operation_id: Mapped[UUID] = mapped_column(ForeignKey("problem_import_operations.id"))
|
||||
manifest_digest: Mapped[str] = mapped_column(String(64))
|
||||
manifest: Mapped[dict] = mapped_column(JSONB)
|
||||
preview: Mapped[dict] = mapped_column(JSONB)
|
||||
case_count: Mapped[int] = mapped_column(Integer)
|
||||
|
||||
|
||||
class CandidateCase(Base):
|
||||
__tablename__ = "problem_candidate_cases"
|
||||
__table_args__ = (
|
||||
CheckConstraint("position BETWEEN 0 AND 255", name="position"),
|
||||
CheckConstraint(
|
||||
"input_bytes BETWEEN 0 AND 67108864 AND expected_bytes BETWEEN 0 AND 67108864",
|
||||
name="bytes",
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["input_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["expected_digest", "artifact_state"],
|
||||
["artifacts.digest", "artifacts.state"],
|
||||
name="fk_candidate_cases_expected_artifacts",
|
||||
),
|
||||
)
|
||||
candidate_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey("problem_candidates.id"), primary_key=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(64))
|
||||
input_digest: Mapped[str] = mapped_column(String(64))
|
||||
input_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
expected_digest: Mapped[str] = mapped_column(String(64))
|
||||
expected_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
artifact_state: Mapped[str] = mapped_column(String(8), Computed("'ready'", persisted=True))
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Shared import availability error and exact policy identity; no lifecycle imports."""
|
||||
|
||||
import hashlib
|
||||
|
||||
|
||||
class ImportUnavailable(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def policy_digest(policy):
|
||||
return hashlib.sha256(policy.model_dump_json().encode()).hexdigest()
|
||||
@@ -0,0 +1,430 @@
|
||||
"""Fenced ordinary-user import consumption. No student execution or evaluation DB access."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, text
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
|
||||
from kf_oj.contracts.admin_problems import MAX_IMPORT_ATTEMPTS, VALIDATION_DEADLINE_SECONDS
|
||||
from kf_oj.infra.artifacts.store import ArtifactError
|
||||
from kf_oj.modules.identity.public import MaintenanceDenied
|
||||
from kf_oj.modules.problems.candidate_validation import validate_candidate
|
||||
from kf_oj.modules.problems.import_models import (
|
||||
CandidateCase,
|
||||
ImportPool,
|
||||
ProblemCandidate,
|
||||
)
|
||||
from kf_oj.modules.problems.import_models import (
|
||||
ProblemImportOperation as Operation,
|
||||
)
|
||||
from kf_oj.modules.problems.imports import ACTIVE, ImportUnavailable, ProblemImports, policy_digest
|
||||
|
||||
|
||||
def verify_worker_limits(configured_group: Path):
|
||||
if os.getuid() == 0:
|
||||
raise ImportUnavailable("import worker must be ordinary")
|
||||
actual = next(
|
||||
line.split(":", 2)[2]
|
||||
for line in Path("/proc/self/cgroup").read_text().splitlines()
|
||||
if line.startswith("0::")
|
||||
)
|
||||
group = Path("/sys/fs/cgroup") / actual.lstrip("/")
|
||||
if group.resolve() != configured_group.absolute().resolve():
|
||||
raise ImportUnavailable("import worker cgroup mismatch")
|
||||
caps = {
|
||||
n: (group / n).read_text().strip()
|
||||
for n in ("memory.max", "memory.swap.max", "pids.max", "cpu.max")
|
||||
}
|
||||
if (
|
||||
caps["memory.max"] == "max"
|
||||
or not 0 < int(caps["memory.max"]) <= 134217728
|
||||
or caps["memory.swap.max"] != "0"
|
||||
or caps["pids.max"] == "max"
|
||||
or not 0 < int(caps["pids.max"]) <= 16
|
||||
):
|
||||
raise ImportUnavailable("import worker resource limits unavailable")
|
||||
quota, period = caps["cpu.max"].split()
|
||||
if quota == "max" or int(quota) > int(period):
|
||||
raise ImportUnavailable("import worker CPU limit unavailable")
|
||||
for name, maximum in (
|
||||
(resource.RLIMIT_FSIZE, 67108864),
|
||||
(resource.RLIMIT_NOFILE, 64),
|
||||
(resource.RLIMIT_CORE, 0),
|
||||
):
|
||||
limits = resource.getrlimit(name)
|
||||
if any(v == resource.RLIM_INFINITY or v > maximum for v in limits):
|
||||
raise ImportUnavailable("import worker process limits unavailable")
|
||||
return caps
|
||||
|
||||
|
||||
class ImportWorker:
|
||||
def __init__(self, sessions, staging, artifacts, authority, audit, policy, cgroup):
|
||||
(
|
||||
self.sessions,
|
||||
self.staging,
|
||||
self.artifacts,
|
||||
self.authority,
|
||||
self.audit,
|
||||
self.policy,
|
||||
self.cgroup,
|
||||
) = sessions, staging, artifacts, authority, audit, policy, cgroup
|
||||
|
||||
async def heartbeat(self):
|
||||
verify_worker_limits(self.cgroup)
|
||||
async with self.sessions.begin() as session:
|
||||
safe = await session.scalar(
|
||||
text("""
|
||||
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
|
||||
OR rolreplication OR rolbypassrls)
|
||||
AND pg_has_role(current_user,'kf_oj_problem_import','USAGE')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
|
||||
FROM pg_roles WHERE rolname=current_user
|
||||
""")
|
||||
)
|
||||
if not safe:
|
||||
raise ImportUnavailable("dedicated unprivileged import login required")
|
||||
pool = await session.get(ImportPool, 1, with_for_update=True)
|
||||
if pool is None:
|
||||
raise ImportUnavailable("import pool unavailable")
|
||||
pool.worker_ready_until = await session.scalar(
|
||||
select(func.clock_timestamp())
|
||||
) + timedelta(seconds=90)
|
||||
pool.worker_policy_digest = policy_digest(self.policy)
|
||||
|
||||
async def _release(self, operation_id):
|
||||
await asyncio.to_thread(self.staging.remove, operation_id)
|
||||
async with self.sessions.begin() as session:
|
||||
pool = await ProblemImports.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
pool.reserved_bytes -= op.reserved_bytes
|
||||
op.reserved_bytes = 0
|
||||
op.lease_until = None
|
||||
|
||||
async def collect(self):
|
||||
# Caller owns the root flock, so a live upload/parser cannot be unlinked or
|
||||
# have its reservation reused, even if its DB lease expired while suspended.
|
||||
async with self.sessions() as session:
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
names = list(
|
||||
await session.scalars(
|
||||
select(Operation.id)
|
||||
.where(
|
||||
or_(
|
||||
Operation.reserved_bytes > 0,
|
||||
Operation.state.in_(ACTIVE) & (Operation.expires_at <= now),
|
||||
)
|
||||
)
|
||||
.order_by(Operation.updated_at, Operation.id)
|
||||
.limit(32)
|
||||
)
|
||||
)
|
||||
garbage = list(
|
||||
await session.scalars(
|
||||
select(ProblemCandidate.operation_id)
|
||||
.join(Operation, Operation.id == ProblemCandidate.operation_id)
|
||||
.where(Operation.state.in_(("discarded", "expired", "failed")))
|
||||
.limit(32)
|
||||
)
|
||||
)
|
||||
names = list(dict.fromkeys(names + garbage))[:32]
|
||||
for identifier in names:
|
||||
async with self.sessions.begin() as session:
|
||||
await ProblemImports.pool(session)
|
||||
op = await session.get(Operation, identifier, with_for_update=True)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if op.state in ACTIVE and op.expires_at <= now:
|
||||
op.state = "expired"
|
||||
op.failure_code = "expired"
|
||||
op.updated_at = now
|
||||
op.candidate_id = None
|
||||
op.fence += 1
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "import_expired"
|
||||
)
|
||||
elif op.state == "uploading" and op.lease_until and op.lease_until <= now:
|
||||
op.state = "failed"
|
||||
op.failure_code = "upload_interrupted"
|
||||
op.updated_at = now
|
||||
op.fence += 1
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "import_failed"
|
||||
)
|
||||
elif op.state == "validating" and op.lease_until and op.lease_until <= now:
|
||||
if op.attempts < MAX_IMPORT_ATTEMPTS:
|
||||
op.state = "uploaded"
|
||||
op.failure_code = None
|
||||
op.lease_until = None
|
||||
op.fence += 1
|
||||
op.updated_at = now
|
||||
await asyncio.to_thread(self.staging.reset_validation, op.id)
|
||||
else:
|
||||
op.state = "failed"
|
||||
op.failure_code = "validation_unavailable"
|
||||
op.updated_at = now
|
||||
op.fence += 1
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "import_failed"
|
||||
)
|
||||
if op.state in ("discarded", "expired", "failed"):
|
||||
op.candidate_id = None
|
||||
await session.flush()
|
||||
candidate = await session.scalar(
|
||||
select(ProblemCandidate).where(ProblemCandidate.operation_id == op.id)
|
||||
)
|
||||
if candidate:
|
||||
await session.execute(
|
||||
delete(CandidateCase).where(CandidateCase.candidate_id == candidate.id)
|
||||
)
|
||||
await session.delete(candidate)
|
||||
release = (
|
||||
op.state in ("ready", "published", "discarded", "expired", "failed")
|
||||
and op.reserved_bytes > 0
|
||||
)
|
||||
if release:
|
||||
await self._release(identifier)
|
||||
# Uncommitted staging objects after a failed transaction are removed only under
|
||||
# the same kernel lock and only from this service's private, bounded namespace.
|
||||
for identifier in self.staging.operation_ids():
|
||||
async with self.sessions() as session:
|
||||
exists = await session.get(Operation, identifier)
|
||||
if exists is None:
|
||||
await asyncio.to_thread(self.staging.remove, identifier)
|
||||
|
||||
async def once(self):
|
||||
await self.heartbeat()
|
||||
try:
|
||||
with self.staging.exclusive() as lock:
|
||||
await self.collect()
|
||||
async with self.sessions.begin() as session:
|
||||
await ProblemImports.pool(session)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
busy = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Operation)
|
||||
.where(
|
||||
Operation.state.in_(("uploading", "validating")),
|
||||
Operation.lease_until > now,
|
||||
)
|
||||
)
|
||||
if busy:
|
||||
return False
|
||||
op = await session.scalar(
|
||||
select(Operation)
|
||||
.where(
|
||||
Operation.state == "uploaded",
|
||||
Operation.attempts < MAX_IMPORT_ATTEMPTS,
|
||||
Operation.expires_at > now,
|
||||
)
|
||||
.order_by(Operation.created_at, Operation.id)
|
||||
.limit(1)
|
||||
.with_for_update()
|
||||
)
|
||||
if op is None:
|
||||
return False
|
||||
op.attempts += 1
|
||||
op.fence += 1
|
||||
op.state = "validating"
|
||||
op.updated_at = now
|
||||
op.lease_until = now + timedelta(seconds=VALIDATION_DEADLINE_SECONDS + 15)
|
||||
identifier, fence, size, digest, actor, epoch = (
|
||||
op.id,
|
||||
op.fence,
|
||||
op.archive_bytes,
|
||||
op.archive_digest,
|
||||
op.owner_id,
|
||||
op.authorization_version,
|
||||
)
|
||||
try:
|
||||
async with self.sessions.begin() as session:
|
||||
await self.authority.require_reserved_maintenance(session, actor, epoch)
|
||||
candidate_id = uuid4()
|
||||
result = await self.parse(identifier, size, digest, candidate_id, lock)
|
||||
if "failure_code" in result:
|
||||
await self.fail(identifier, fence, result["failure_code"])
|
||||
return True
|
||||
candidate = validate_candidate(
|
||||
result["manifest"], result["preview"], result["references"], self.policy
|
||||
)
|
||||
if (
|
||||
candidate.digest != result["manifest_digest"]
|
||||
or candidate.preview.candidate_id != candidate_id
|
||||
):
|
||||
raise ValueError("invalid candidate receipt")
|
||||
directory = self.staging.operation(identifier)
|
||||
members = self.staging.members(directory)
|
||||
try:
|
||||
async with self.sessions.begin() as session:
|
||||
await self.authority.require_reserved_maintenance(session, actor, epoch)
|
||||
await ProblemImports.pool(session)
|
||||
op = await session.get(Operation, identifier, with_for_update=True)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if (
|
||||
op.state != "validating"
|
||||
or op.fence != fence
|
||||
or op.lease_until <= now
|
||||
or op.expires_at <= now
|
||||
):
|
||||
raise ValueError("stale candidate receipt")
|
||||
for name, reference in sorted(
|
||||
candidate.references.items(), key=lambda item: item[1].digest
|
||||
):
|
||||
fd = os.open(
|
||||
name,
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||
dir_fd=members,
|
||||
)
|
||||
try:
|
||||
await self.artifacts.put_file(session, fd, reference)
|
||||
finally:
|
||||
os.close(fd)
|
||||
session.add(
|
||||
ProblemCandidate(
|
||||
id=candidate_id,
|
||||
operation_id=identifier,
|
||||
manifest_digest=candidate.digest,
|
||||
manifest=candidate.manifest,
|
||||
preview=candidate.preview.model_dump(mode="json"),
|
||||
case_count=candidate.preview.case_count,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
for position, case in enumerate(candidate.manifest["cases"]):
|
||||
input_ref, answer_ref = (
|
||||
candidate.references[case["input_file"]],
|
||||
candidate.references[case["expected_file"]],
|
||||
)
|
||||
session.add(
|
||||
CandidateCase(
|
||||
candidate_id=candidate_id,
|
||||
position=position,
|
||||
key=case["key"],
|
||||
input_digest=input_ref.digest,
|
||||
input_bytes=input_ref.bytes,
|
||||
expected_digest=answer_ref.digest,
|
||||
expected_bytes=answer_ref.bytes,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
# Adoption must not turn an expired lease into a ready candidate.
|
||||
completed_at = await session.scalar(select(func.clock_timestamp()))
|
||||
if op.lease_until <= completed_at or op.expires_at <= completed_at:
|
||||
raise ValueError("candidate completion lease expired")
|
||||
op.state = "ready"
|
||||
op.candidate_id = candidate_id
|
||||
op.updated_at = completed_at
|
||||
op.lease_until = None
|
||||
await self.audit.record(
|
||||
session, actor, epoch, identifier, "import_ready", candidate_id
|
||||
)
|
||||
finally:
|
||||
os.close(members)
|
||||
os.close(directory)
|
||||
await self._release(identifier)
|
||||
except MaintenanceDenied:
|
||||
await self.fail(identifier, fence, "authority_revoked")
|
||||
except (ArtifactError, OSError):
|
||||
await self.fail(identifier, fence, "storage_unavailable")
|
||||
except (ValueError, DBAPIError):
|
||||
await self.fail(identifier, fence, "validation_unavailable")
|
||||
return True
|
||||
except ArtifactError:
|
||||
return False # Another live writer holds the one global slot.
|
||||
|
||||
async def fail(self, identifier, fence, code):
|
||||
from kf_oj.contracts.admin_problems import ImportFailure
|
||||
|
||||
code = ImportFailure(code).value
|
||||
async with self.sessions.begin() as session:
|
||||
await ProblemImports.pool(session)
|
||||
op = await session.get(Operation, identifier, with_for_update=True)
|
||||
if op.state != "validating" or op.fence != fence:
|
||||
return
|
||||
op.state = "failed"
|
||||
op.failure_code = code
|
||||
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||
op.lease_until = None
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "import_failed"
|
||||
)
|
||||
await self._release(identifier)
|
||||
|
||||
async def parse(self, identifier, size, digest, candidate, lock):
|
||||
verify_worker_limits(self.cgroup)
|
||||
directory = self.staging.operation(identifier)
|
||||
archive = members = None
|
||||
try:
|
||||
archive = os.open(
|
||||
"archive",
|
||||
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
|
||||
dir_fd=directory,
|
||||
)
|
||||
members = self.staging.members(directory, create=True)
|
||||
# Preserve only the trusted runtime import roots, without inheriting DB/
|
||||
# service credentials. Curated runtimes may keep dependencies outside site-packages.
|
||||
env = {
|
||||
"PATH": str(Path(sys.executable).parent),
|
||||
"PYTHONPATH": os.pathsep.join(
|
||||
dict.fromkeys(
|
||||
[str(Path(__file__).parents[3])]
|
||||
+ [p for p in os.environ.get("PYTHONPATH", "").split(os.pathsep) if p]
|
||||
)
|
||||
),
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"PYTHONNOUSERSITE": "1",
|
||||
"LC_ALL": "C.UTF-8",
|
||||
}
|
||||
# Minimal environment may require the trusted dynamic-loader path in a curated VM.
|
||||
if "LD_LIBRARY_PATH" in os.environ:
|
||||
env["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH"]
|
||||
child = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-m",
|
||||
"kf_oj.workers._archive_child",
|
||||
str(archive),
|
||||
str(members),
|
||||
str(lock),
|
||||
str(size),
|
||||
digest,
|
||||
str(candidate),
|
||||
self.policy.model_dump_json(),
|
||||
env=env,
|
||||
pass_fds=(archive, members, lock),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
async with asyncio.timeout(VALIDATION_DEADLINE_SECONDS + 5):
|
||||
output, _ = await child.communicate()
|
||||
except BaseException:
|
||||
child.kill()
|
||||
await child.wait()
|
||||
raise
|
||||
if child.returncode != 0 or not 0 < len(output) <= 4 * 1024**2:
|
||||
return {
|
||||
"failure_code": "validation_deadline"
|
||||
if child.returncode == -14
|
||||
else "validation_unavailable"
|
||||
}
|
||||
result = json.loads(output)
|
||||
if not isinstance(result, dict):
|
||||
raise ValueError("invalid child result")
|
||||
return result
|
||||
except TimeoutError:
|
||||
return {"failure_code": "validation_deadline"}
|
||||
finally:
|
||||
if members is not None:
|
||||
os.close(members)
|
||||
if archive is not None:
|
||||
os.close(archive)
|
||||
os.close(directory)
|
||||
@@ -0,0 +1,432 @@
|
||||
"""Authenticated durable import lifecycle; filesystem work remains behind private adapters."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import timedelta
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
IMPORT_LIFETIME_SECONDS,
|
||||
MAX_OWNER_CANDIDATES,
|
||||
MAX_TOTAL_CANDIDATES,
|
||||
MAX_UPLOAD_CHUNK_BYTES,
|
||||
MAX_UPLOAD_CHUNKS,
|
||||
STAGING_MAX_BYTES,
|
||||
STAGING_RESERVATION_BYTES,
|
||||
UPLOAD_DEADLINE_SECONDS,
|
||||
CandidatePreview,
|
||||
CreateProblemImport,
|
||||
ProblemImportCapabilities,
|
||||
ProblemImportDetail,
|
||||
ProblemImportPage,
|
||||
ProblemImportSummary,
|
||||
)
|
||||
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||
from kf_oj.infra.artifacts.store import ArtifactError
|
||||
from kf_oj.modules.identity.public import MaintenanceDenied, ProblemMaintenancePort
|
||||
from kf_oj.modules.operations.public import MaintenanceAuditPort
|
||||
from kf_oj.modules.problems.import_models import ImportPool, ProblemCandidate
|
||||
from kf_oj.modules.problems.import_models import ProblemImportOperation as Operation
|
||||
from kf_oj.modules.problems.import_policy import ImportUnavailable as ImportUnavailable
|
||||
from kf_oj.modules.problems.import_policy import policy_digest as policy_digest
|
||||
|
||||
ACTIVE = ("uploading", "uploaded", "validating", "ready")
|
||||
|
||||
|
||||
class ImportConflict(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ImportBadDigest(ImportConflict):
|
||||
pass
|
||||
|
||||
|
||||
class ImportMissing(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def summary(operation) -> ProblemImportSummary:
|
||||
return ProblemImportSummary(
|
||||
operation_id=operation.id,
|
||||
state=operation.state,
|
||||
expected_archive_digest=operation.archive_digest,
|
||||
archive_bytes=operation.archive_bytes,
|
||||
created_at=operation.created_at,
|
||||
expires_at=operation.expires_at,
|
||||
updated_at=operation.updated_at,
|
||||
failure_code=operation.failure_code,
|
||||
candidate_id=operation.candidate_id,
|
||||
published_version_id=operation.published_version_id,
|
||||
)
|
||||
|
||||
|
||||
class ProblemImports:
|
||||
def __init__(
|
||||
self,
|
||||
sessions,
|
||||
staging: ImportStaging,
|
||||
identity: ProblemMaintenancePort,
|
||||
audit: MaintenanceAuditPort,
|
||||
policy,
|
||||
):
|
||||
self.sessions, self.staging, self.identity, self.audit, self.policy = (
|
||||
sessions,
|
||||
staging,
|
||||
identity,
|
||||
audit,
|
||||
policy,
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def write(self, principal):
|
||||
async with self.sessions.begin() as session:
|
||||
await session.execute(text("SET LOCAL statement_timeout='5s'"))
|
||||
await session.execute(text("SET LOCAL lock_timeout='2s'"))
|
||||
await self.identity.require_problem_maintenance(session, principal)
|
||||
yield session
|
||||
|
||||
@staticmethod
|
||||
async def pool(session):
|
||||
pool = await session.get(ImportPool, 1, with_for_update=True)
|
||||
if pool is None:
|
||||
raise ImportUnavailable("import capacity unavailable")
|
||||
actual = await session.scalar(
|
||||
select(func.coalesce(func.sum(Operation.reserved_bytes), 0)).where(
|
||||
Operation.reserved_bytes > 0
|
||||
)
|
||||
)
|
||||
if pool.reserved_bytes != actual:
|
||||
raise ImportUnavailable("import capacity inconsistent")
|
||||
return pool
|
||||
|
||||
async def capabilities(self):
|
||||
async with self.sessions() as session:
|
||||
pool = await session.get(ImportPool, 1)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
available = bool(
|
||||
pool
|
||||
and pool.worker_ready_until
|
||||
and pool.worker_ready_until > now
|
||||
and pool.worker_policy_digest == policy_digest(self.policy)
|
||||
)
|
||||
return ProblemImportCapabilities(
|
||||
uploads_available=available, validation_available=available
|
||||
)
|
||||
|
||||
async def create(self, principal, request: CreateProblemImport):
|
||||
# Immutable receipt retries need no filesystem slot, including while a parser/GC
|
||||
# owns the root lock. Missing receipts still enter the locked creation path below.
|
||||
async with self.write(principal) as session:
|
||||
existing = await session.get(Operation, request.operation_id)
|
||||
if existing:
|
||||
if (
|
||||
existing.owner_id != principal.account_id
|
||||
or existing.archive_digest != request.archive_digest
|
||||
or existing.archive_bytes != request.archive_bytes
|
||||
):
|
||||
raise ImportConflict("operation key already bound")
|
||||
return summary(existing)
|
||||
try:
|
||||
with self.staging.exclusive():
|
||||
await asyncio.to_thread(self.staging.usage)
|
||||
async with self.write(principal) as session:
|
||||
pool = await self.pool(session)
|
||||
existing = await session.get(
|
||||
Operation, request.operation_id, with_for_update=True
|
||||
)
|
||||
if existing:
|
||||
if (
|
||||
existing.owner_id != principal.account_id
|
||||
or existing.archive_digest != request.archive_digest
|
||||
or existing.archive_bytes != request.archive_bytes
|
||||
):
|
||||
raise ImportConflict("operation key already bound")
|
||||
return summary(existing)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if (
|
||||
not pool.worker_ready_until
|
||||
or pool.worker_ready_until <= now
|
||||
or pool.worker_policy_digest != policy_digest(self.policy)
|
||||
):
|
||||
raise ImportUnavailable("import validation unavailable")
|
||||
global_count = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Operation)
|
||||
.where(Operation.state.in_(ACTIVE))
|
||||
)
|
||||
owner_count = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Operation)
|
||||
.where(
|
||||
Operation.state.in_(ACTIVE), Operation.owner_id == principal.account_id
|
||||
)
|
||||
)
|
||||
busy = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(Operation)
|
||||
.where(
|
||||
Operation.state.in_(("uploading", "validating")),
|
||||
Operation.lease_until > now,
|
||||
)
|
||||
)
|
||||
if (
|
||||
global_count >= MAX_TOTAL_CANDIDATES
|
||||
or owner_count >= MAX_OWNER_CANDIDATES
|
||||
or busy
|
||||
or pool.reserved_bytes + STAGING_RESERVATION_BYTES > STAGING_MAX_BYTES
|
||||
):
|
||||
raise ImportUnavailable("import capacity exhausted")
|
||||
operation = Operation(
|
||||
id=request.operation_id,
|
||||
owner_id=principal.account_id,
|
||||
authorization_version=principal.authorization_version,
|
||||
archive_digest=request.archive_digest,
|
||||
archive_bytes=request.archive_bytes,
|
||||
state="uploading",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
expires_at=now + timedelta(seconds=IMPORT_LIFETIME_SECONDS),
|
||||
lease_until=now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15),
|
||||
attempts=0,
|
||||
fence=0,
|
||||
reserved_bytes=STAGING_RESERVATION_BYTES,
|
||||
)
|
||||
session.add(operation)
|
||||
pool.reserved_bytes += STAGING_RESERVATION_BYTES
|
||||
await self.audit.record(
|
||||
session,
|
||||
principal.account_id,
|
||||
principal.authorization_version,
|
||||
operation.id,
|
||||
"upload_created",
|
||||
)
|
||||
await session.flush()
|
||||
return summary(operation)
|
||||
except ArtifactError:
|
||||
raise ImportUnavailable("import storage unavailable") from None
|
||||
|
||||
async def detail(self, principal, operation_id):
|
||||
# Recheck live authority even when a principal was injected by an internal caller.
|
||||
async with self.write(principal) as session:
|
||||
op = await session.get(Operation, operation_id)
|
||||
if op is None or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
candidate = (
|
||||
await session.get(ProblemCandidate, op.candidate_id) if op.candidate_id else None
|
||||
)
|
||||
return ProblemImportDetail(
|
||||
**summary(op).model_dump(),
|
||||
candidate=CandidatePreview.model_validate(candidate.preview) if candidate else None,
|
||||
)
|
||||
|
||||
async def list(self, principal, after=None, limit=20):
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError("invalid page limit")
|
||||
async with self.write(principal) as session:
|
||||
query = select(Operation).where(Operation.owner_id == principal.account_id)
|
||||
if after:
|
||||
query = query.where(Operation.id > after)
|
||||
rows = list(await session.scalars(query.order_by(Operation.id).limit(limit + 1)))
|
||||
return ProblemImportPage(
|
||||
items=tuple(summary(o) for o in rows[:limit]),
|
||||
next_after=rows[limit - 1].id if len(rows) > limit else None,
|
||||
)
|
||||
|
||||
async def upload(self, principal, operation_id: UUID, chunks, content_length: int | None):
|
||||
# Reject missing/foreign receipts before consulting shared filesystem capacity.
|
||||
# The locked transaction below still rechecks authority and ownership before writes.
|
||||
async with self.write(principal) as session:
|
||||
op = await session.get(Operation, operation_id)
|
||||
if op is None or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
if content_length is not None and content_length != op.archive_bytes:
|
||||
raise ImportConflict("archive length mismatch")
|
||||
descriptor = operation_fd = None
|
||||
fence = None
|
||||
received = 0
|
||||
chunk_count = 0
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
with self.staging.exclusive():
|
||||
async with self.write(principal) as session:
|
||||
await self.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
if not op or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
accepted = op.state in ("uploaded", "validating", "ready", "published")
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if (
|
||||
not accepted
|
||||
and (
|
||||
op.state != "uploading"
|
||||
or op.lease_until is None
|
||||
or op.lease_until <= now
|
||||
)
|
||||
) or op.expires_at <= now:
|
||||
raise ImportConflict("upload no longer available")
|
||||
if content_length is not None and content_length != op.archive_bytes:
|
||||
raise ImportConflict("archive length mismatch")
|
||||
if not accepted:
|
||||
op.fence += 1
|
||||
fence = op.fence
|
||||
op.lease_until = now + timedelta(seconds=UPLOAD_DEADLINE_SECONDS + 15)
|
||||
expected_bytes, expected_digest = op.archive_bytes, op.archive_digest
|
||||
if not accepted:
|
||||
operation_fd = self.staging.operation(operation_id, create=True)
|
||||
descriptor = os.open(
|
||||
"archive",
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC,
|
||||
0o600,
|
||||
dir_fd=operation_fd,
|
||||
)
|
||||
async with asyncio.timeout(UPLOAD_DEADLINE_SECONDS):
|
||||
async for chunk in chunks:
|
||||
chunk_count += 1
|
||||
received += len(chunk)
|
||||
if (
|
||||
type(chunk) is not bytes
|
||||
or len(chunk) > MAX_UPLOAD_CHUNK_BYTES
|
||||
or chunk_count > MAX_UPLOAD_CHUNKS
|
||||
or received > expected_bytes
|
||||
):
|
||||
raise ImportConflict("archive bytes outside policy")
|
||||
digest.update(chunk)
|
||||
if accepted:
|
||||
continue
|
||||
# Cancellation waits for the finite writer outside the event loop.
|
||||
task = asyncio.create_task(asyncio.to_thread(write_all, descriptor, chunk))
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
except BaseException:
|
||||
await task
|
||||
raise
|
||||
if received != expected_bytes or digest.hexdigest() != expected_digest:
|
||||
raise ImportBadDigest("archive digest or length mismatch")
|
||||
if accepted:
|
||||
async with self.write(principal) as session:
|
||||
op = await session.get(Operation, operation_id)
|
||||
if not op or op.state not in (
|
||||
"uploaded",
|
||||
"validating",
|
||||
"ready",
|
||||
"published",
|
||||
):
|
||||
raise ImportConflict("import receipt no longer available")
|
||||
return summary(op)
|
||||
await asyncio.to_thread(os.fsync, descriptor)
|
||||
os.close(descriptor)
|
||||
descriptor = None
|
||||
os.fsync(operation_fd)
|
||||
async with self.write(principal) as session:
|
||||
await self.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
now = await session.scalar(select(func.clock_timestamp()))
|
||||
if op.state != "uploading" or op.fence != fence or op.lease_until <= now:
|
||||
raise ImportConflict("upload receipt expired")
|
||||
op.state = "uploaded"
|
||||
op.lease_until = None
|
||||
op.updated_at = now
|
||||
await self.audit.record(
|
||||
session, op.owner_id, op.authorization_version, op.id, "upload_received"
|
||||
)
|
||||
result = summary(op)
|
||||
return result
|
||||
except BaseException as error:
|
||||
# Failure bookkeeping cannot authorize a new business write. It only fences this
|
||||
# accepted upload and releases capacity after physical bytes have been removed.
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
descriptor = None
|
||||
if operation_fd is not None:
|
||||
os.close(operation_fd)
|
||||
operation_fd = None
|
||||
if fence is not None:
|
||||
try:
|
||||
with self.staging.exclusive():
|
||||
await asyncio.to_thread(self.staging.remove, operation_id)
|
||||
async with self.sessions.begin() as session:
|
||||
pool = await self.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
if op and op.state == "uploading" and op.fence == fence:
|
||||
op.state = "failed"
|
||||
op.failure_code = (
|
||||
"upload_deadline"
|
||||
if isinstance(error, TimeoutError)
|
||||
else "digest_mismatch"
|
||||
if isinstance(error, ImportBadDigest)
|
||||
else "authority_revoked"
|
||||
if isinstance(error, MaintenanceDenied)
|
||||
else "storage_unavailable"
|
||||
if isinstance(error, OSError)
|
||||
else "upload_interrupted"
|
||||
)
|
||||
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||
op.lease_until = None
|
||||
pool.reserved_bytes -= op.reserved_bytes
|
||||
op.reserved_bytes = 0
|
||||
await self.audit.record(
|
||||
session,
|
||||
op.owner_id,
|
||||
op.authorization_version,
|
||||
op.id,
|
||||
"import_failed",
|
||||
)
|
||||
except (ArtifactError, OSError):
|
||||
pass # Durable reservation retained for fenced GC.
|
||||
raise
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
if operation_fd is not None:
|
||||
os.close(operation_fd)
|
||||
|
||||
async def discard(self, principal, operation_id):
|
||||
async with self.write(principal) as session:
|
||||
op = await session.get(Operation, operation_id)
|
||||
if op is None or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
if op.state == "discarded":
|
||||
return summary(op)
|
||||
try:
|
||||
with self.staging.exclusive():
|
||||
async with self.write(principal) as session:
|
||||
pool = await self.pool(session)
|
||||
op = await session.get(Operation, operation_id, with_for_update=True)
|
||||
if not op or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("import unavailable")
|
||||
if op.state == "published":
|
||||
raise ImportConflict("published import cannot be discarded")
|
||||
if op.state == "discarded":
|
||||
return summary(op)
|
||||
await asyncio.to_thread(self.staging.remove, operation_id)
|
||||
op.state = "discarded"
|
||||
op.candidate_id = None
|
||||
op.failure_code = None
|
||||
op.lease_until = None
|
||||
op.fence += 1
|
||||
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||
pool.reserved_bytes -= op.reserved_bytes
|
||||
op.reserved_bytes = 0
|
||||
await self.audit.record(
|
||||
session,
|
||||
principal.account_id,
|
||||
principal.authorization_version,
|
||||
op.id,
|
||||
"import_discarded",
|
||||
)
|
||||
return summary(op)
|
||||
except ArtifactError:
|
||||
raise ImportUnavailable("import storage unavailable") from None
|
||||
|
||||
|
||||
def write_all(descriptor, data):
|
||||
remaining = memoryview(data)
|
||||
while remaining:
|
||||
size = os.write(descriptor, remaining)
|
||||
if size <= 0:
|
||||
raise OSError("short staging write")
|
||||
remaining = remaining[size:]
|
||||
@@ -0,0 +1,324 @@
|
||||
"""Administrator transactions; no evaluation writes or private data in projections."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import uuid4, uuid5
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
|
||||
from kf_oj.contracts.admin_problems import CandidatePreview
|
||||
from kf_oj.contracts.judging import ArtifactReference, TestCase
|
||||
from kf_oj.contracts.problem_maintenance import (
|
||||
MaintainedProblem,
|
||||
MaintainedProblemPage,
|
||||
MaintainedVersion,
|
||||
MaintainedVersionPage,
|
||||
MaintenanceReceipt,
|
||||
)
|
||||
from kf_oj.modules.identity.public import ProblemMaintenancePort
|
||||
from kf_oj.modules.operations.public import MaintenanceAuditPort
|
||||
from kf_oj.modules.problems import repository
|
||||
from kf_oj.modules.problems.candidate_validation import validate_candidate
|
||||
from kf_oj.modules.problems.import_format import ImportEnvelope
|
||||
from kf_oj.modules.problems.import_models import (
|
||||
CandidateCase,
|
||||
ProblemCandidate,
|
||||
ProblemImportOperation,
|
||||
)
|
||||
from kf_oj.modules.problems.imports import ImportConflict, ImportMissing, ImportUnavailable
|
||||
from kf_oj.modules.problems.maintenance_models import ProblemMaintenanceOperation
|
||||
from kf_oj.modules.problems.models import Problem, ProblemCase, ProblemVersion
|
||||
from kf_oj.modules.problems.publication import admit_version, publish_version
|
||||
from kf_oj.modules.problems.service import ProblemService
|
||||
|
||||
|
||||
def project_problem(problem, version):
|
||||
return MaintainedProblem(
|
||||
id=problem.id,
|
||||
slug=problem.slug,
|
||||
enabled=problem.enabled,
|
||||
active_version_id=problem.active_version_id,
|
||||
revision=version.revision if version else 0,
|
||||
title=version.title if version else None,
|
||||
created_at=problem.created_at,
|
||||
)
|
||||
|
||||
|
||||
class ProblemMaintenance:
|
||||
def __init__(
|
||||
self,
|
||||
sessions,
|
||||
identity: ProblemMaintenancePort,
|
||||
audit: MaintenanceAuditPort,
|
||||
artifacts=None,
|
||||
policy=None,
|
||||
):
|
||||
self.sessions, self.identity, self.audit = sessions, identity, audit
|
||||
self.artifacts, self.policy = artifacts, policy
|
||||
self._publishing = asyncio.Semaphore(2)
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self, principal):
|
||||
async with self.sessions.begin() as session:
|
||||
await session.execute(text("SET LOCAL statement_timeout='5s'"))
|
||||
await session.execute(text("SET LOCAL lock_timeout='2s'"))
|
||||
await self.identity.require_problem_maintenance(session, principal)
|
||||
yield session
|
||||
|
||||
async def page(self, principal, *, after=None, limit=20, enabled=None):
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError("invalid page limit")
|
||||
async with self.transaction(principal) as session:
|
||||
q = select(Problem, ProblemVersion).outerjoin(
|
||||
ProblemVersion, Problem.active_version_id == ProblemVersion.id
|
||||
)
|
||||
if after is not None:
|
||||
q = q.where(Problem.slug > after)
|
||||
if enabled is not None:
|
||||
q = q.where(Problem.enabled == enabled)
|
||||
rows = list(await session.execute(q.order_by(Problem.slug).limit(limit + 1)))
|
||||
return MaintainedProblemPage(
|
||||
items=tuple(project_problem(*r) for r in rows[:limit]),
|
||||
next_after=rows[limit - 1][0].slug if len(rows) > limit else None,
|
||||
)
|
||||
|
||||
async def by_slug(self, principal, slug):
|
||||
async with self.transaction(principal) as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(Problem, ProblemVersion)
|
||||
.outerjoin(ProblemVersion, Problem.active_version_id == ProblemVersion.id)
|
||||
.where(Problem.slug == slug)
|
||||
)
|
||||
).one_or_none()
|
||||
if row is None:
|
||||
raise ImportMissing("problem unavailable")
|
||||
return project_problem(*row)
|
||||
|
||||
async def detail(self, principal, problem_id):
|
||||
async with self.transaction(principal) as session:
|
||||
problem = await session.get(Problem, problem_id)
|
||||
if problem is None:
|
||||
raise ImportMissing("problem unavailable")
|
||||
version = (
|
||||
await session.get(ProblemVersion, problem.active_version_id)
|
||||
if problem.active_version_id
|
||||
else None
|
||||
)
|
||||
return project_problem(problem, version)
|
||||
|
||||
async def versions(self, principal, problem_id, *, before=None, limit=20):
|
||||
if not 1 <= limit <= 100 or (before is not None and before < 1):
|
||||
raise ValueError("invalid page limit")
|
||||
async with self.transaction(principal) as session:
|
||||
problem = await session.get(Problem, problem_id)
|
||||
if problem is None:
|
||||
raise ImportMissing("problem unavailable")
|
||||
count = (
|
||||
select(func.count())
|
||||
.where(ProblemCase.version_id == ProblemVersion.id)
|
||||
.scalar_subquery()
|
||||
)
|
||||
q = select(ProblemVersion, count).where(
|
||||
ProblemVersion.problem_id == problem_id, ProblemVersion.published_at.is_not(None)
|
||||
)
|
||||
if before is not None:
|
||||
q = q.where(ProblemVersion.revision < before)
|
||||
rows = list(
|
||||
await session.execute(q.order_by(ProblemVersion.revision.desc()).limit(limit + 1))
|
||||
)
|
||||
return MaintainedVersionPage(
|
||||
items=tuple(
|
||||
MaintainedVersion(
|
||||
**ProblemService._summary(problem, v).model_dump(),
|
||||
published_at=v.published_at,
|
||||
case_count=n,
|
||||
)
|
||||
for v, n in rows[:limit]
|
||||
),
|
||||
next_before=rows[limit - 1][0].revision if len(rows) > limit else None,
|
||||
)
|
||||
|
||||
async def receipt(self, principal, operation_id):
|
||||
async with self.transaction(principal) as session:
|
||||
row = await session.get(ProblemMaintenanceOperation, operation_id)
|
||||
if row is None or row.owner_id != principal.account_id:
|
||||
raise ImportMissing("operation unavailable")
|
||||
return MaintenanceReceipt.model_validate(row.receipt)
|
||||
|
||||
async def _replay(self, session, principal, action, body, target):
|
||||
digest = hashlib.sha256(
|
||||
json.dumps(
|
||||
{"action": action, "target": str(target), "request": body.model_dump(mode="json")},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
).hexdigest()
|
||||
key = int.from_bytes(hashlib.sha256(body.operation_id.bytes).digest()[:4], signed=True)
|
||||
await session.execute(text("SELECT pg_advisory_xact_lock(1262907470,:key)"), {"key": key})
|
||||
row = await session.get(ProblemMaintenanceOperation, body.operation_id)
|
||||
if row is not None:
|
||||
if row.owner_id != principal.account_id:
|
||||
raise ImportMissing("operation unavailable")
|
||||
if row.action != action or row.request_digest != digest:
|
||||
raise ImportConflict("operation UUID already bound")
|
||||
return digest, MaintenanceReceipt.model_validate(row.receipt)
|
||||
return digest, None
|
||||
|
||||
async def _record(
|
||||
self, session, principal, request, action, digest, problem, version, published=None
|
||||
):
|
||||
receipt = MaintenanceReceipt(
|
||||
operation_id=request.operation_id,
|
||||
action=action,
|
||||
completed_at=await session.scalar(select(func.clock_timestamp())),
|
||||
problem=project_problem(problem, version),
|
||||
published_version_id=published,
|
||||
)
|
||||
session.add(
|
||||
ProblemMaintenanceOperation(
|
||||
id=request.operation_id,
|
||||
owner_id=principal.account_id,
|
||||
authorization_version=principal.authorization_version,
|
||||
action=action,
|
||||
request_digest=digest,
|
||||
receipt=receipt.model_dump(mode="json"),
|
||||
)
|
||||
)
|
||||
await self.audit.record(
|
||||
session,
|
||||
principal.account_id,
|
||||
principal.authorization_version,
|
||||
request.operation_id,
|
||||
{"publish": "published", "disable": "disabled", "restore": "restored"}[action],
|
||||
published if action == "publish" else problem.id,
|
||||
)
|
||||
await session.flush()
|
||||
return receipt
|
||||
|
||||
async def publish(self, principal, request):
|
||||
if self.artifacts is None or self.policy is None or self._publishing.locked():
|
||||
raise ImportUnavailable("publication unavailable")
|
||||
async with self._publishing:
|
||||
async with self.transaction(principal) as session:
|
||||
digest, replay = await self._replay(
|
||||
session, principal, "publish", request, request.import_operation_id
|
||||
)
|
||||
if replay:
|
||||
return replay
|
||||
op = await session.get(
|
||||
ProblemImportOperation, request.import_operation_id, with_for_update=True
|
||||
)
|
||||
if op is None or op.owner_id != principal.account_id:
|
||||
raise ImportMissing("candidate unavailable")
|
||||
if (
|
||||
op.state != "ready"
|
||||
or op.candidate_id != request.candidate_id
|
||||
or op.authorization_version != principal.authorization_version
|
||||
or op.expires_at <= await session.scalar(select(func.clock_timestamp()))
|
||||
):
|
||||
raise ImportConflict("candidate no longer publishable")
|
||||
candidate = await session.get(ProblemCandidate, op.candidate_id)
|
||||
if candidate is None or candidate.operation_id != op.id:
|
||||
raise ImportUnavailable("candidate unavailable")
|
||||
manifest = ImportEnvelope.model_validate(candidate.manifest).root
|
||||
preview = CandidatePreview.model_validate(candidate.preview)
|
||||
rows = list(
|
||||
await session.scalars(
|
||||
select(CandidateCase)
|
||||
.where(CandidateCase.candidate_id == candidate.id)
|
||||
.order_by(CandidateCase.position)
|
||||
)
|
||||
)
|
||||
if len(rows) != len(manifest.cases) or len(rows) != candidate.case_count:
|
||||
raise ImportUnavailable("candidate references unavailable")
|
||||
refs = {}
|
||||
for position, (row, c) in enumerate(zip(rows, manifest.cases, strict=True)):
|
||||
if row.position != position or row.key != c.key:
|
||||
raise ImportUnavailable("candidate references unavailable")
|
||||
for name, ref in (
|
||||
(
|
||||
c.input_file,
|
||||
ArtifactReference(digest=row.input_digest, bytes=row.input_bytes),
|
||||
),
|
||||
(
|
||||
c.expected_file,
|
||||
ArtifactReference(digest=row.expected_digest, bytes=row.expected_bytes),
|
||||
),
|
||||
):
|
||||
if name in refs and refs[name] != ref:
|
||||
raise ImportUnavailable("candidate references unavailable")
|
||||
refs[name] = ref
|
||||
validated = validate_candidate(
|
||||
candidate.manifest, candidate.preview, refs, self.policy
|
||||
)
|
||||
if (
|
||||
validated.digest != candidate.manifest_digest
|
||||
or preview.candidate_id != candidate.id
|
||||
):
|
||||
raise ImportUnavailable("candidate integrity unavailable")
|
||||
await repository.lock_slug(session, manifest.slug)
|
||||
problem = await repository.lock_problem(session, manifest.slug)
|
||||
actual = problem.active_version_id if problem else None
|
||||
if actual != request.expected_active_version_id or (
|
||||
problem and not problem.enabled
|
||||
):
|
||||
raise ImportConflict("active version or availability changed")
|
||||
if problem is None:
|
||||
problem = Problem(id=uuid4(), slug=manifest.slug)
|
||||
session.add(problem)
|
||||
await session.flush()
|
||||
version_id = uuid4()
|
||||
cases = tuple(
|
||||
TestCase(
|
||||
case_id=uuid5(version_id, c.key),
|
||||
input=refs[c.input_file],
|
||||
expected=refs[c.expected_file],
|
||||
)
|
||||
for c in manifest.cases
|
||||
)
|
||||
admit_version(self.policy, manifest, version_id, problem.id, cases)
|
||||
# Candidate FKs protect all referenced ready metadata from GC. The
|
||||
# private adapter additionally re-hashes disk bytes without buffering them.
|
||||
for ref in sorted(
|
||||
{r.digest: r for r in refs.values()}.values(), key=lambda r: r.digest
|
||||
):
|
||||
await self.artifacts.check_metadata(session, ref)
|
||||
task = asyncio.create_task(asyncio.to_thread(self.artifacts.store.verify, ref))
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
except BaseException:
|
||||
await task
|
||||
raise
|
||||
version = await publish_version(
|
||||
session, problem, manifest, preview, version_id, cases
|
||||
)
|
||||
op.state, op.published_version_id = "published", version.id
|
||||
op.updated_at = await session.scalar(select(func.clock_timestamp()))
|
||||
return await self._record(
|
||||
session, principal, request, "publish", digest, problem, version, version.id
|
||||
)
|
||||
|
||||
async def availability(self, principal, problem_id, request, *, enabled):
|
||||
action = "restore" if enabled else "disable"
|
||||
async with self.transaction(principal) as session:
|
||||
digest, replay = await self._replay(session, principal, action, request, problem_id)
|
||||
if replay:
|
||||
return replay
|
||||
problem = await session.get(Problem, problem_id, with_for_update=True)
|
||||
if problem is None:
|
||||
raise ImportMissing("problem unavailable")
|
||||
if (
|
||||
problem.active_version_id != request.expected_active_version_id
|
||||
or problem.enabled != request.expected_enabled
|
||||
or problem.enabled == enabled
|
||||
):
|
||||
raise ImportConflict("active version or availability changed")
|
||||
version = await session.get(ProblemVersion, problem.active_version_id)
|
||||
if version is None or version.published_at is None:
|
||||
raise ImportUnavailable("published version unavailable")
|
||||
problem.enabled = enabled
|
||||
await session.flush()
|
||||
return await self._record(session, principal, request, action, digest, problem, version)
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Append-only actor-bound receipts; replay never re-applies a published operation."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import BigInteger, CheckConstraint, ForeignKey, String
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from kf_oj.infra.db.base import Base
|
||||
from kf_oj.infra.db.columns import CreatedAt
|
||||
|
||||
|
||||
class ProblemMaintenanceOperation(CreatedAt, Base):
|
||||
__tablename__ = "problem_maintenance_operations"
|
||||
__table_args__ = (
|
||||
CheckConstraint("action IN ('publish','disable','restore')", name="action"),
|
||||
CheckConstraint("request_digest ~ '^[0-9a-f]{64}$'", name="digest"),
|
||||
CheckConstraint("authorization_version > 0", name="authority"),
|
||||
CheckConstraint("octet_length(receipt::text) <= 4096", name="receipt"),
|
||||
)
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
owner_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
|
||||
authorization_version: Mapped[int] = mapped_column(BigInteger)
|
||||
action: Mapped[str] = mapped_column(String(16))
|
||||
request_digest: Mapped[str] = mapped_column(String(64))
|
||||
receipt: Mapped[dict] = mapped_column(JSONB)
|
||||
@@ -13,6 +13,7 @@ from sqlalchemy import (
|
||||
ForeignKeyConstraint,
|
||||
Index,
|
||||
Integer,
|
||||
SmallInteger,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
@@ -54,6 +55,17 @@ class ProblemVersion(CreatedAt, Base):
|
||||
CheckConstraint("char_length(title) BETWEEN 1 AND 160", name="title_size"),
|
||||
CheckConstraint("comparison = 'exact_bytes'", name="comparison"),
|
||||
CheckConstraint("job_wall_time_us BETWEEN 1000 AND 3600000000", name="job_wall"),
|
||||
CheckConstraint("judging_schema_version IN (1,2)", name="judging_schema"),
|
||||
CheckConstraint(
|
||||
"(judging_schema_version = 1 AND language_configurations IS NULL "
|
||||
"AND toolchain IS NOT NULL AND compile_limits IS NOT NULL "
|
||||
"AND run_limits IS NOT NULL AND job_wall_time_us IS NOT NULL) OR "
|
||||
"(judging_schema_version = 2 AND language_configurations IS NOT NULL "
|
||||
"AND toolchain IS NULL AND compile_limits IS NULL "
|
||||
"AND run_limits IS NULL AND job_wall_time_us IS NULL "
|
||||
"AND public.kf_valid_language_configurations(languages, language_configurations))",
|
||||
name="language_configurations",
|
||||
),
|
||||
Index("ix_problem_versions_public", "problem_id", "published_at"),
|
||||
)
|
||||
|
||||
@@ -63,10 +75,12 @@ class ProblemVersion(CreatedAt, Base):
|
||||
title: Mapped[str] = mapped_column(String(160))
|
||||
statement: Mapped[str] = mapped_column(Text)
|
||||
languages: Mapped[list[str]] = mapped_column(JSONB)
|
||||
toolchain: Mapped[dict] = mapped_column(JSONB)
|
||||
compile_limits: Mapped[dict] = mapped_column(JSONB)
|
||||
run_limits: Mapped[dict] = mapped_column(JSONB)
|
||||
job_wall_time_us: Mapped[int] = mapped_column(BigInteger)
|
||||
judging_schema_version: Mapped[int] = mapped_column(SmallInteger, server_default="1")
|
||||
language_configurations: Mapped[list[dict] | None] = mapped_column(JSONB(none_as_null=True))
|
||||
toolchain: Mapped[dict | None] = mapped_column(JSONB(none_as_null=True))
|
||||
compile_limits: Mapped[dict | None] = mapped_column(JSONB(none_as_null=True))
|
||||
run_limits: Mapped[dict | None] = mapped_column(JSONB(none_as_null=True))
|
||||
job_wall_time_us: Mapped[int | None] = mapped_column(BigInteger)
|
||||
comparison: Mapped[str] = mapped_column(String(16), server_default="exact_bytes")
|
||||
public_samples: Mapped[list[dict]] = mapped_column(JSONB)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Read-only import projection for the ordinary operator CLI; no private archive data."""
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from kf_oj.modules.problems.import_models import ImportPool
|
||||
from kf_oj.modules.problems.import_policy import ImportUnavailable, policy_digest
|
||||
|
||||
|
||||
class ImportOperationQueries:
|
||||
def __init__(self, sessions, policy):
|
||||
self.sessions, self.policy = sessions, policy
|
||||
|
||||
async def status(self) -> dict:
|
||||
async with self.sessions() as session, session.begin():
|
||||
await session.execute(text("SET TRANSACTION READ ONLY"))
|
||||
await session.execute(text("SET LOCAL statement_timeout='1500'"))
|
||||
await session.execute(text("SET LOCAL lock_timeout='500'"))
|
||||
authorized = await session.scalar(
|
||||
text("""
|
||||
SELECT rolcanlogin AND NOT(rolsuper OR rolcreatedb OR rolcreaterole
|
||||
OR rolreplication OR rolbypassrls)
|
||||
AND pg_has_role(current_user,'kf_oj_problem_import','USAGE')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_platform','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_coordinator','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_operator','MEMBER')
|
||||
AND NOT pg_has_role(current_user,'kf_oj_identity_authority_owner','MEMBER')
|
||||
FROM pg_roles WHERE rolname=session_user
|
||||
""")
|
||||
)
|
||||
if authorized is not True:
|
||||
raise ImportUnavailable("dedicated import observation role required")
|
||||
pool = await session.get(ImportPool, 1)
|
||||
row = (
|
||||
(
|
||||
await session.execute(
|
||||
text("""
|
||||
SELECT clock_timestamp() now,
|
||||
(SELECT count(*) FROM (SELECT 1 FROM problem_import_operations
|
||||
WHERE state='uploaded' LIMIT 129) q) uploaded,
|
||||
(SELECT count(*) FROM (SELECT 1 FROM problem_import_operations
|
||||
WHERE state='validating' LIMIT 129) q) validating,
|
||||
(SELECT coalesce(sum(reserved_bytes),0) FROM (
|
||||
SELECT reserved_bytes FROM problem_import_operations
|
||||
WHERE reserved_bytes>0 LIMIT 129) q) actual_reserved
|
||||
""")
|
||||
)
|
||||
)
|
||||
.mappings()
|
||||
.one()
|
||||
)
|
||||
if pool is None or pool.reserved_bytes != row["actual_reserved"]:
|
||||
raise ImportUnavailable("import reservation consistency unknown")
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"worker_ready": bool(
|
||||
pool.worker_ready_until
|
||||
and pool.worker_ready_until > row["now"]
|
||||
and pool.worker_policy_digest == policy_digest(self.policy)
|
||||
),
|
||||
"reserved_bytes": pool.reserved_bytes,
|
||||
"uploaded_capped": min(row["uploaded"], 128),
|
||||
"validating_capped": min(row["validating"], 128),
|
||||
"truncated": row["uploaded"] > 128 or row["validating"] > 128,
|
||||
}
|
||||
@@ -4,7 +4,9 @@ from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from kf_oj.contracts.judging import Language, RunLimits, TestCase, Toolchain
|
||||
from kf_oj.contracts import judging as v1
|
||||
from kf_oj.contracts import judging_v2 as v2
|
||||
from kf_oj.contracts.judging_versions import Manifest
|
||||
|
||||
|
||||
class ProblemUnavailable(ValueError):
|
||||
@@ -15,13 +17,68 @@ class ProblemUnavailable(ValueError):
|
||||
class JudgingVersion:
|
||||
version_id: UUID
|
||||
problem_id: UUID
|
||||
languages: tuple[Language, ...]
|
||||
toolchain: Toolchain
|
||||
compile_limits: RunLimits
|
||||
run_limits: RunLimits
|
||||
job_wall_time_us: int
|
||||
cases: tuple[TestCase, ...]
|
||||
languages: tuple[v1.Language | v2.Language, ...]
|
||||
toolchain: v1.Toolchain | None
|
||||
compile_limits: v1.RunLimits | None
|
||||
run_limits: v1.RunLimits | None
|
||||
job_wall_time_us: int | None
|
||||
cases: tuple[v1.TestCase, ...]
|
||||
judging_schema_version: int = 1
|
||||
language_configurations: tuple[v2.LanguageConfiguration, ...] = ()
|
||||
|
||||
def manifest(
|
||||
self,
|
||||
evaluation_id: UUID,
|
||||
submission_id: UUID,
|
||||
language: v1.Language | v2.Language,
|
||||
source: v1.ArtifactReference,
|
||||
) -> Manifest:
|
||||
"""Build only from the immutable authorized version, preserving v1 wire bytes."""
|
||||
if language not in self.languages:
|
||||
raise ProblemUnavailable("language unavailable")
|
||||
fields = dict(
|
||||
schema_version=self.judging_schema_version,
|
||||
kind="judge.manifest",
|
||||
evaluation_id=evaluation_id,
|
||||
submission_id=submission_id,
|
||||
problem_version_id=self.version_id,
|
||||
language=language,
|
||||
source=source,
|
||||
cases=self.cases,
|
||||
comparison="exact_bytes",
|
||||
)
|
||||
if self.judging_schema_version == 1:
|
||||
if self.language_configurations:
|
||||
raise ProblemUnavailable("inconsistent legacy version")
|
||||
return v1.JobManifest(
|
||||
**fields,
|
||||
toolchain=self.toolchain,
|
||||
compile_limits=self.compile_limits,
|
||||
run_limits=self.run_limits,
|
||||
job_wall_time_us=self.job_wall_time_us,
|
||||
)
|
||||
if self.judging_schema_version != 2 or any(
|
||||
field is not None
|
||||
for field in (
|
||||
self.toolchain,
|
||||
self.compile_limits,
|
||||
self.run_limits,
|
||||
self.job_wall_time_us,
|
||||
)
|
||||
):
|
||||
raise ProblemUnavailable("unsupported judging configuration")
|
||||
if tuple(c.language for c in self.language_configurations) != self.languages:
|
||||
raise ProblemUnavailable("inconsistent language configurations")
|
||||
configuration = next(c for c in self.language_configurations if c.language == language)
|
||||
return v2.JobManifest(
|
||||
**fields,
|
||||
**configuration.model_dump(exclude={"language"}),
|
||||
)
|
||||
|
||||
|
||||
class ProblemPort(Protocol):
|
||||
async def for_submission(self, session, version_id: UUID) -> JudgingVersion: ...
|
||||
|
||||
async def for_accepted_execution(self, session, version_id: UUID) -> JudgingVersion:
|
||||
"""Require an already-validated live execution lease; allow a disabled published version."""
|
||||
...
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
"""One admission and immutable publication path for local and web imports."""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from kf_oj.contracts import judging_v2 as v2
|
||||
from kf_oj.contracts.judging import MAX_SOURCE_BYTES, ArtifactReference
|
||||
from kf_oj.modules.problems import repository
|
||||
from kf_oj.modules.problems.import_format import ProblemImport
|
||||
from kf_oj.modules.problems.models import ProblemCase, ProblemVersion
|
||||
from kf_oj.modules.problems.public import JudgingVersion, ProblemUnavailable
|
||||
|
||||
|
||||
def admit_version(policy, manifest, version_id, problem_id, cases):
|
||||
legacy = isinstance(manifest, ProblemImport)
|
||||
projection = JudgingVersion(
|
||||
version_id,
|
||||
problem_id,
|
||||
manifest.languages,
|
||||
manifest.toolchain if legacy else None,
|
||||
manifest.compile_limits if legacy else None,
|
||||
manifest.run_limits if legacy else None,
|
||||
manifest.job_wall_time_us if legacy else None,
|
||||
cases,
|
||||
manifest.schema_version,
|
||||
() if legacy else manifest.language_configurations,
|
||||
)
|
||||
for language in manifest.languages:
|
||||
v2.admit_manifest(
|
||||
policy,
|
||||
projection.manifest(
|
||||
uuid4(),
|
||||
uuid4(),
|
||||
language,
|
||||
ArtifactReference(digest="0" * 64, bytes=MAX_SOURCE_BYTES),
|
||||
),
|
||||
1,
|
||||
)
|
||||
return projection
|
||||
|
||||
|
||||
async def publish_version(session, problem, manifest, preview, version_id, cases):
|
||||
"""Caller holds the logical slug/row locks and has made all artifacts durable."""
|
||||
if not problem.enabled:
|
||||
raise ProblemUnavailable("problem disabled")
|
||||
legacy = isinstance(manifest, ProblemImport)
|
||||
version = ProblemVersion(
|
||||
id=version_id,
|
||||
problem_id=problem.id,
|
||||
revision=await repository.next_revision(session, problem.id),
|
||||
title=manifest.title,
|
||||
statement=preview.statement,
|
||||
languages=[lang.value for lang in manifest.languages],
|
||||
judging_schema_version=manifest.schema_version,
|
||||
language_configurations=None
|
||||
if legacy
|
||||
else [c.model_dump(mode="json") for c in manifest.language_configurations],
|
||||
toolchain=manifest.toolchain.model_dump(mode="json") if legacy else None,
|
||||
compile_limits=manifest.compile_limits.model_dump(mode="json") if legacy else None,
|
||||
run_limits=manifest.run_limits.model_dump(mode="json") if legacy else None,
|
||||
job_wall_time_us=manifest.job_wall_time_us if legacy else None,
|
||||
comparison=manifest.comparison,
|
||||
public_samples=[s.model_dump(mode="json") for s in preview.samples],
|
||||
)
|
||||
session.add(version)
|
||||
await session.flush()
|
||||
for position, case in enumerate(cases):
|
||||
session.add(
|
||||
ProblemCase(
|
||||
id=case.case_id,
|
||||
version_id=version_id,
|
||||
position=position,
|
||||
input_digest=case.input.digest,
|
||||
expected_digest=case.expected.digest,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
version.published_at = await session.scalar(select(func.now()))
|
||||
problem.active_version_id = version_id
|
||||
await session.flush()
|
||||
return version
|
||||
@@ -24,7 +24,22 @@ async def next_revision(session: AsyncSession, problem_id: UUID) -> int:
|
||||
return (latest or 0) + 1
|
||||
|
||||
|
||||
async def public_version(session: AsyncSession, version_id: UUID):
|
||||
async def published_version(session: AsyncSession, version_id: UUID, *, admission=False):
|
||||
query = (
|
||||
select(ProblemVersion, Problem)
|
||||
.join(Problem, Problem.id == ProblemVersion.problem_id)
|
||||
.where(ProblemVersion.id == version_id, ProblemVersion.published_at.is_not(None))
|
||||
)
|
||||
if admission:
|
||||
# A new submission commits while this shared problem lock is held. A
|
||||
# concurrent disable either precedes admission or follows that commit.
|
||||
query = query.where(Problem.enabled.is_(True)).with_for_update(read=True, of=Problem)
|
||||
return (await session.execute(query)).one_or_none()
|
||||
|
||||
|
||||
async def public_version(session: AsyncSession, version_id: UUID, *, admission=False):
|
||||
if admission:
|
||||
return await published_version(session, version_id, admission=True)
|
||||
return (
|
||||
await session.execute(
|
||||
select(ProblemVersion, Problem)
|
||||
|
||||
@@ -1,30 +1,37 @@
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID, uuid4, uuid5
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from kf_oj.contracts import judging_v2 as v2
|
||||
from kf_oj.contracts.judging import (
|
||||
MAX_PACKAGE_BYTES,
|
||||
MAX_SOURCE_BYTES,
|
||||
AdmissionPolicy,
|
||||
ArtifactReference,
|
||||
JobManifest,
|
||||
Language,
|
||||
RunLimits,
|
||||
TestCase,
|
||||
Toolchain,
|
||||
decode_wire,
|
||||
)
|
||||
from kf_oj.contracts.problems import ProblemDetail, ProblemPage, ProblemSummary, PublicSample
|
||||
from kf_oj.contracts.problems import (
|
||||
ProblemDetail,
|
||||
ProblemPage,
|
||||
ProblemSummary,
|
||||
PublicLanguageConfiguration,
|
||||
PublicSample,
|
||||
)
|
||||
from kf_oj.infra.artifacts.models import Artifact
|
||||
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||
from kf_oj.modules.problems import repository
|
||||
from kf_oj.modules.problems.import_format import ProblemImport
|
||||
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport
|
||||
from kf_oj.modules.problems.local_files import LocalImportFiles
|
||||
from kf_oj.modules.problems.models import Problem, ProblemCase, ProblemVersion
|
||||
from kf_oj.modules.problems.models import Problem
|
||||
from kf_oj.modules.problems.public import JudgingVersion, ProblemUnavailable
|
||||
from kf_oj.modules.problems.publication import admit_version, publish_version
|
||||
|
||||
|
||||
class ProblemService:
|
||||
@@ -32,7 +39,7 @@ class ProblemService:
|
||||
self,
|
||||
sessions: async_sessionmaker,
|
||||
artifacts: ArtifactRepository | None = None,
|
||||
policy: AdmissionPolicy | None = None,
|
||||
policy: v2.NodeAdmission | None = None,
|
||||
):
|
||||
self.sessions, self.artifacts, self.policy = sessions, artifacts, policy
|
||||
|
||||
@@ -41,7 +48,12 @@ class ProblemService:
|
||||
raise ValueError("artifact store and trusted node policy required for import")
|
||||
files = LocalImportFiles(root)
|
||||
try:
|
||||
manifest = decode_wire(ProblemImport, files.read(manifest_file, 256 * 1024))
|
||||
manifest = decode_wire(ImportEnvelope, files.read(manifest_file, 256 * 1024)).root
|
||||
stdout_limit = (
|
||||
manifest.run_limits.stdout_bytes
|
||||
if isinstance(manifest, ProblemImport)
|
||||
else min(c.run_limits.stdout_bytes for c in manifest.language_configurations)
|
||||
)
|
||||
statement = files.read(manifest.statement_file, 256 * 1024).decode("utf-8")
|
||||
if not statement or "\x00" in statement or "\x00" in manifest.title:
|
||||
raise ValueError("invalid public text")
|
||||
@@ -52,9 +64,7 @@ class ProblemService:
|
||||
for case in manifest.cases:
|
||||
data = files.read(case.input_file, remaining)
|
||||
remaining -= len(data)
|
||||
answer = files.read(
|
||||
case.expected_file, min(remaining, manifest.run_limits.stdout_bytes)
|
||||
)
|
||||
answer = files.read(case.expected_file, min(remaining, stdout_limit))
|
||||
remaining -= len(answer)
|
||||
payloads.append((data, answer))
|
||||
samples = tuple(
|
||||
@@ -75,22 +85,7 @@ class ProblemService:
|
||||
)
|
||||
for case, (data, answer) in zip(manifest.cases, payloads, strict=True)
|
||||
)
|
||||
check = JobManifest(
|
||||
schema_version=1,
|
||||
kind="judge.manifest",
|
||||
evaluation_id=uuid4(),
|
||||
submission_id=uuid4(),
|
||||
problem_version_id=version_id,
|
||||
language=manifest.languages[0],
|
||||
toolchain=manifest.toolchain,
|
||||
source=ArtifactReference(digest="0" * 64, bytes=MAX_SOURCE_BYTES),
|
||||
cases=cases,
|
||||
comparison=manifest.comparison,
|
||||
compile_limits=manifest.compile_limits,
|
||||
run_limits=manifest.run_limits,
|
||||
job_wall_time_us=manifest.job_wall_time_us,
|
||||
)
|
||||
self.policy.admit(check, 1)
|
||||
admit_version(self.policy, manifest, version_id, uuid4(), cases)
|
||||
async with self.sessions.begin() as session:
|
||||
# Lock existing and not-yet-existing logical slugs before assigning revisions.
|
||||
await repository.lock_slug(session, manifest.slug)
|
||||
@@ -101,24 +96,6 @@ class ProblemService:
|
||||
await session.flush()
|
||||
if not problem.enabled:
|
||||
raise ProblemUnavailable("problem disabled")
|
||||
revision = await repository.next_revision(session, problem.id)
|
||||
session.add(
|
||||
ProblemVersion(
|
||||
id=version_id,
|
||||
problem_id=problem.id,
|
||||
revision=revision,
|
||||
title=manifest.title,
|
||||
statement=statement,
|
||||
languages=[lang.value for lang in manifest.languages],
|
||||
toolchain=manifest.toolchain.model_dump(mode="json"),
|
||||
compile_limits=manifest.compile_limits.model_dump(mode="json"),
|
||||
run_limits=manifest.run_limits.model_dump(mode="json"),
|
||||
job_wall_time_us=manifest.job_wall_time_us,
|
||||
comparison=manifest.comparison,
|
||||
public_samples=[s.model_dump() for s in samples],
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
# Consistent digest lock order avoids deadlocks between overlapping imports/jobs.
|
||||
unique = {
|
||||
ref.digest: (data, ref)
|
||||
@@ -128,20 +105,14 @@ class ProblemService:
|
||||
for digest in sorted(unique):
|
||||
data, reference = unique[digest]
|
||||
await self.artifacts.put(session, data, reference)
|
||||
for position, case in enumerate(cases):
|
||||
session.add(
|
||||
ProblemCase(
|
||||
id=case.case_id,
|
||||
version_id=version_id,
|
||||
position=position,
|
||||
input_digest=case.input.digest,
|
||||
expected_digest=case.expected.digest,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
version = await session.get(ProblemVersion, version_id)
|
||||
version.published_at = await session.scalar(select(func.now()))
|
||||
problem.active_version_id = version_id
|
||||
await publish_version(
|
||||
session,
|
||||
problem,
|
||||
manifest,
|
||||
SimpleNamespace(statement=statement, samples=samples),
|
||||
version_id,
|
||||
cases,
|
||||
)
|
||||
return version_id
|
||||
|
||||
@staticmethod
|
||||
@@ -176,13 +147,32 @@ class ProblemService:
|
||||
**self._summary(problem, version).model_dump(),
|
||||
statement=version.statement,
|
||||
samples=tuple(PublicSample.model_validate(s) for s in version.public_samples),
|
||||
run_limits=RunLimits.model_validate(version.run_limits),
|
||||
run_limits=(
|
||||
RunLimits.model_validate(version.run_limits)
|
||||
if version.judging_schema_version == 1
|
||||
else None
|
||||
),
|
||||
language_configurations=tuple(
|
||||
PublicLanguageConfiguration.from_configuration(
|
||||
v2.LanguageConfiguration.model_validate(c)
|
||||
)
|
||||
for c in version.language_configurations or ()
|
||||
),
|
||||
)
|
||||
|
||||
async def for_submission(self, session, version_id: UUID) -> JudgingVersion:
|
||||
if self.artifacts is None:
|
||||
raise ProblemUnavailable("artifact service unavailable")
|
||||
row = await repository.public_version(session, version_id)
|
||||
row = await repository.public_version(session, version_id, admission=True)
|
||||
return await self._judging_version(session, row)
|
||||
|
||||
async def for_accepted_execution(self, session, version_id: UUID) -> JudgingVersion:
|
||||
# Only WorkerAccess calls this after validating the current lease and its
|
||||
# immutable submission/attempt binding; this is never a public UUID lookup.
|
||||
row = await repository.published_version(session, version_id)
|
||||
return await self._judging_version(session, row)
|
||||
|
||||
async def _judging_version(self, session, row):
|
||||
if row is None:
|
||||
raise ProblemUnavailable("problem version unavailable")
|
||||
version, problem = row
|
||||
@@ -215,10 +205,24 @@ class ProblemService:
|
||||
return JudgingVersion(
|
||||
version.id,
|
||||
problem.id,
|
||||
tuple(Language(v) for v in version.languages),
|
||||
Toolchain.model_validate(version.toolchain),
|
||||
RunLimits.model_validate(version.compile_limits),
|
||||
RunLimits.model_validate(version.run_limits),
|
||||
tuple(
|
||||
(Language if version.judging_schema_version == 1 else v2.Language)(v)
|
||||
for v in version.languages
|
||||
),
|
||||
Toolchain.model_validate(version.toolchain)
|
||||
if version.judging_schema_version == 1
|
||||
else None,
|
||||
RunLimits.model_validate(version.compile_limits)
|
||||
if version.judging_schema_version == 1
|
||||
else None,
|
||||
RunLimits.model_validate(version.run_limits)
|
||||
if version.judging_schema_version == 1
|
||||
else None,
|
||||
version.job_wall_time_us,
|
||||
tuple(refs),
|
||||
version.judging_schema_version,
|
||||
tuple(
|
||||
v2.LanguageConfiguration.model_validate(c)
|
||||
for c in version.language_configurations or ()
|
||||
),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Stricter Web package declarations; the accepted local CLI format remains unchanged."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from kf_oj.contracts.judging import decode_wire
|
||||
from kf_oj.modules.problems.import_format import ImportEnvelope, ProblemImport, ProblemImportV2
|
||||
|
||||
ARCHIVE_SUFFIXES = (".zip", ".jar", ".war", ".tar", ".tgz", ".gz", ".bz2", ".xz", ".7z", ".rar")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WebDeclaration:
|
||||
manifest: ProblemImport | ProblemImportV2
|
||||
members: frozenset[str]
|
||||
|
||||
|
||||
def decode_web_declaration(data: bytes) -> WebDeclaration:
|
||||
# decode_wire rejects duplicate JSON keys, unknown fields and noncanonical
|
||||
# protocol values before either local paths or node policy can be considered.
|
||||
manifest = decode_wire(ImportEnvelope, data).root
|
||||
hidden = {name for c in manifest.cases for name in (c.input_file, c.expected_file)}
|
||||
samples = {name for s in manifest.samples for name in (s.input_file, s.output_file)}
|
||||
metadata = {"problem.json", manifest.statement_file}
|
||||
if len(metadata) != 2 or hidden & (samples | metadata) or samples & metadata:
|
||||
raise ValueError("public and hidden package declarations must be separate")
|
||||
members = frozenset(metadata | hidden | samples)
|
||||
if len({name.lower() for name in members}) != len(members):
|
||||
raise ValueError("package names must be case-insensitively unique")
|
||||
if any(name.lower().endswith(ARCHIVE_SUFFIXES) for name in members):
|
||||
raise ValueError("nested archive declarations are unsupported")
|
||||
return WebDeclaration(manifest, members)
|
||||
@@ -23,3 +23,28 @@ submission、queued evaluation 和 outbox;同键同内容复用既有提交,
|
||||
Coordinator 通过正式 claim 创建短期领取权,Worker 不写业务数据库。
|
||||
真实服务链与安全条件必须在独立 VM 验收;数据库 fixture 的租约或合成报告只能
|
||||
验证控制规则、授权和投影,不能作为真实程序执行/停止证明。
|
||||
|
||||
`performance.py` 提供本人提交的只读匿名聚合。Coordinator 在受验证最终 AC 的同一事务
|
||||
写入不可变 `evaluation_performance`;平台只有 SELECT 权限。比较组绑定不可变版本、
|
||||
语言、精确镜像/工具链、编译与运行预算,以及运维核验后配置的测量环境纪元。
|
||||
没有原纪元的旧尝试不会通过当前配置被补造可信性,仍保留个人历史与正常成绩。
|
||||
单条受控回填与纪元变更规则见[协调器说明](../../../../../judge/crates/coordinator/README.md)。
|
||||
|
||||
历史可按 `problem_version_id` 与可选 wire `language` 过滤,始终带当前账号条件。
|
||||
`GET /api/v1/submissions/{submission_id}/performance` 每次先检查会话和所有权;
|
||||
匿名拒绝、其他主体与不存在记录同为不可用,私有响应为 `no-store`。
|
||||
缓存命中仍查归属。每进程最多一个聚合查询,不排等待队列;SQL 期限两秒、请求总期限
|
||||
三秒、锁等待至多半秒。聚合 DTO 缓存三十秒且最多128项,新 AC 的索引水位改变即失效。
|
||||
缓存不保存逐条样本数组,不向浏览器发送其他账号的身份、源码或逐条用量。
|
||||
|
||||
每指标在同一个只读可重复读快照内,排除当前提交,取最近最多10,000条非缺测的其他
|
||||
有效 AC,响应带快照、最早样本及截断标志。少于五条时仅显示本次真实用量。
|
||||
CPU 为测试点运行 CPU 微秒之和,内存为测试点运行峰值字节的最大值,不包括编译。
|
||||
用量与桶边界以十进制字符串传输;零合法、缺测不补零。严格优于数按原整数的 `>`
|
||||
计算,同值独立计数。响应至多24个连续桶,只有最后上界包含端点;浏览器仅格式化单位
|
||||
和绘制坐标,比例以服务端严格计数除以样本数,不按人数统计。
|
||||
|
||||
下架只阻止新提交与公开题目读取。Worker 的版本读取通过 `ProblemPort.for_accepted_execution`,
|
||||
在本模块验证当前 Coordinator 租约/尝试/源码绑定之后才调用;生成并比较原 canonical
|
||||
快照,再次核对租约后才释放字节。无有效租约、外来工件、未发布版本和其他提交仍拒绝。
|
||||
本人历史和源码读取保留所有权检查,不依赖题目当前 enabled 状态。
|
||||
|
||||
@@ -15,6 +15,7 @@ from sqlalchemy import (
|
||||
String,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
@@ -28,7 +29,7 @@ class Submission(CreatedAt, Base):
|
||||
__table_args__ = (
|
||||
UniqueConstraint("account_id", "idempotency_key"),
|
||||
UniqueConstraint("id", "problem_version_id"),
|
||||
CheckConstraint("language IN ('c17', 'cpp20')", name="language"),
|
||||
CheckConstraint("language IN ('c17','cpp20','python313','java21')", name="language"),
|
||||
CheckConstraint("request_digest ~ '^[0-9a-f]{64}$'", name="request_digest"),
|
||||
ForeignKeyConstraint(
|
||||
["problem_version_id", "problem_id"],
|
||||
@@ -38,13 +39,20 @@ class Submission(CreatedAt, Base):
|
||||
["source_digest", "artifact_state"], ["artifacts.digest", "artifacts.state"]
|
||||
),
|
||||
Index("ix_submissions_account_history", "account_id", "created_at", "id"),
|
||||
Index(
|
||||
"ix_submissions_owned_problem_history",
|
||||
"account_id",
|
||||
"problem_version_id",
|
||||
text("created_at DESC"),
|
||||
text("id DESC"),
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
account_id: Mapped[UUID] = mapped_column(ForeignKey("accounts.id"))
|
||||
problem_id: Mapped[UUID]
|
||||
problem_version_id: Mapped[UUID]
|
||||
language: Mapped[str] = mapped_column(String(8))
|
||||
language: Mapped[str] = mapped_column(String(16))
|
||||
source_digest: Mapped[str] = mapped_column(String(64))
|
||||
artifact_state: Mapped[str] = mapped_column(String(8), Computed("'ready'", persisted=True))
|
||||
idempotency_key: Mapped[UUID]
|
||||
@@ -112,6 +120,10 @@ class JudgeAttempt(CreatedAt, Base):
|
||||
CheckConstraint("fencing_version > 0", name="fence"),
|
||||
CheckConstraint("lease_token_hash ~ '^[0-9a-f]{64}$'", name="token_digest"),
|
||||
CheckConstraint("manifest_digest ~ '^[0-9a-f]{64}$'", name="manifest_digest"),
|
||||
CheckConstraint(
|
||||
"measurement_epoch IS NULL OR measurement_epoch ~ '^[0-9a-f]{64}$'",
|
||||
name="measurement_epoch",
|
||||
),
|
||||
CheckConstraint("sequence >= 0", name="sequence"),
|
||||
Index("ix_judge_attempts_expiry", "expires_at", "evaluation_id"),
|
||||
)
|
||||
@@ -132,6 +144,64 @@ class JudgeAttempt(CreatedAt, Base):
|
||||
report: Mapped[dict | None] = mapped_column(JSONB)
|
||||
report_digest: Mapped[str | None] = mapped_column(String(64))
|
||||
cleanup_complete: Mapped[bool | None] = mapped_column(Boolean)
|
||||
measurement_epoch: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class EvaluationPerformance(Base):
|
||||
"""Coordinator-only immutable final projection; the platform only reads it."""
|
||||
|
||||
__tablename__ = "evaluation_performance"
|
||||
__table_args__ = (
|
||||
ForeignKeyConstraint(
|
||||
["evaluation_id", "problem_version_id"],
|
||||
["evaluations.id", "evaluations.problem_version_id"],
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["submission_id", "problem_version_id"],
|
||||
["submissions.id", "submissions.problem_version_id"],
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["attempt_id", "evaluation_id"], ["judge_attempts.id", "judge_attempts.evaluation_id"]
|
||||
),
|
||||
CheckConstraint("language IN ('c17','cpp20','python313','java21')", name="language"),
|
||||
CheckConstraint("comparison_group ~ '^[0-9a-f]{64}$'", name="comparison_group"),
|
||||
CheckConstraint("measurement_epoch ~ '^[0-9a-f]{64}$'", name="measurement_epoch"),
|
||||
CheckConstraint("report_digest ~ '^[0-9a-f]{64}$'", name="report_digest"),
|
||||
CheckConstraint("cpu_time_us IS NULL OR cpu_time_us >= 0", name="cpu"),
|
||||
CheckConstraint("memory_peak_bytes IS NULL OR memory_peak_bytes >= 0", name="memory"),
|
||||
Index(
|
||||
"ix_performance_group_watermark",
|
||||
"comparison_group",
|
||||
text("finished_at DESC"),
|
||||
text("evaluation_id DESC"),
|
||||
),
|
||||
Index(
|
||||
"ix_performance_cpu_time_us_cohort",
|
||||
"comparison_group",
|
||||
text("finished_at DESC"),
|
||||
text("evaluation_id DESC"),
|
||||
postgresql_where=text("cpu_time_us IS NOT NULL"),
|
||||
),
|
||||
Index(
|
||||
"ix_performance_memory_peak_bytes_cohort",
|
||||
"comparison_group",
|
||||
text("finished_at DESC"),
|
||||
text("evaluation_id DESC"),
|
||||
postgresql_where=text("memory_peak_bytes IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
|
||||
evaluation_id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
submission_id: Mapped[UUID] = mapped_column(unique=True)
|
||||
attempt_id: Mapped[UUID] = mapped_column(unique=True)
|
||||
problem_version_id: Mapped[UUID]
|
||||
language: Mapped[str] = mapped_column(String(16))
|
||||
comparison_group: Mapped[str] = mapped_column(String(64))
|
||||
measurement_epoch: Mapped[str] = mapped_column(String(64))
|
||||
report_digest: Mapped[str] = mapped_column(String(64))
|
||||
finished_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
cpu_time_us: Mapped[int | None] = mapped_column(BigInteger)
|
||||
memory_peak_bytes: Mapped[int | None] = mapped_column(BigInteger)
|
||||
|
||||
|
||||
class CaseResult(Base):
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Read-only, bounded anonymous cohorts from Coordinator's immutable projection."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from kf_oj.contracts.judging import I64_MAX
|
||||
from kf_oj.contracts.judging_versions import decode_report
|
||||
from kf_oj.contracts.submission_performance import (
|
||||
MAX_PERFORMANCE_SAMPLES,
|
||||
MIN_PERFORMANCE_SAMPLES,
|
||||
PerformanceBucket,
|
||||
PerformanceMetric,
|
||||
SubmissionPerformance,
|
||||
)
|
||||
from kf_oj.modules.identity.public import Principal
|
||||
from kf_oj.modules.submissions.models import Evaluation, EvaluationPerformance, Submission
|
||||
from kf_oj.modules.submissions.public import SubmissionUnavailable
|
||||
|
||||
CACHE_TTL_SECONDS = 30
|
||||
CACHE_CAPACITY = 128
|
||||
HISTOGRAM_BUCKETS = 24
|
||||
|
||||
|
||||
def metric(unit, current, rows=(), *, reason=None) -> PerformanceMetric:
|
||||
"""Integer arithmetic throughout; conversion for display belongs to the client."""
|
||||
truncated = len(rows) > MAX_PERFORMANCE_SAMPLES
|
||||
rows = rows[:MAX_PERFORMANCE_SAMPLES]
|
||||
count = len(rows)
|
||||
base = dict(
|
||||
unit=unit,
|
||||
current=None if current is None else str(current),
|
||||
sample_count=count,
|
||||
window_truncated=truncated,
|
||||
oldest_sample_at=rows[-1][1] if rows else None,
|
||||
)
|
||||
if reason is None:
|
||||
reason = (
|
||||
"measurement_missing"
|
||||
if current is None
|
||||
else ("insufficient_samples" if count < MIN_PERFORMANCE_SAMPLES else None)
|
||||
)
|
||||
if reason is not None:
|
||||
return PerformanceMetric(
|
||||
**base,
|
||||
available=False,
|
||||
reason=reason,
|
||||
strictly_slower_count=None,
|
||||
tied_count=None,
|
||||
buckets=(),
|
||||
)
|
||||
values = [row[0] for row in rows]
|
||||
end = max(1, current, max(values))
|
||||
step = (end + HISTOGRAM_BUCKETS - 1) // HISTOGRAM_BUCKETS
|
||||
# Last upper bound includes the maximum, including I64_MAX. No overflow/padding.
|
||||
boundaries = list(range(0, end, step)) + [end]
|
||||
counts = [0] * (len(boundaries) - 1)
|
||||
for value in values:
|
||||
counts[min(value // step, len(counts) - 1)] += 1
|
||||
return PerformanceMetric(
|
||||
**base,
|
||||
available=True,
|
||||
reason=None,
|
||||
strictly_slower_count=sum(value > current for value in values),
|
||||
tied_count=sum(value == current for value in values),
|
||||
buckets=tuple(
|
||||
PerformanceBucket(
|
||||
start=str(start),
|
||||
end=str(stop),
|
||||
upper_inclusive=index == len(counts) - 1,
|
||||
count=counts[index],
|
||||
)
|
||||
for index, (start, stop) in enumerate(zip(boundaries, boundaries[1:], strict=False))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SubmissionPerformanceQueries:
|
||||
def __init__(self, sessions: async_sessionmaker):
|
||||
self.sessions = sessions
|
||||
# No queued aggregate workload. At most one bounded in-memory row cohort per process.
|
||||
self._slot = asyncio.Lock()
|
||||
# Cache contains small immutable aggregate DTOs, never 10,000-row arrays.
|
||||
self._cache: OrderedDict[tuple, tuple[float, SubmissionPerformance]] = OrderedDict()
|
||||
|
||||
async def detail(self, principal: Principal, submission_id: UUID) -> SubmissionPerformance:
|
||||
if self._slot.locked():
|
||||
raise ValueError("performance query capacity unavailable")
|
||||
async with self._slot:
|
||||
try:
|
||||
async with asyncio.timeout(3), self.sessions() as session, session.begin():
|
||||
await session.execute(
|
||||
text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY")
|
||||
)
|
||||
await session.execute(text("SET LOCAL statement_timeout='2s'"))
|
||||
await session.execute(text("SET LOCAL lock_timeout='500ms'"))
|
||||
# Ownership checked even on a cache hit. Session authentication is upstream.
|
||||
row = (
|
||||
await session.execute(
|
||||
select(Submission, Evaluation, EvaluationPerformance)
|
||||
.join(Evaluation, Evaluation.submission_id == Submission.id)
|
||||
.outerjoin(
|
||||
EvaluationPerformance,
|
||||
EvaluationPerformance.evaluation_id == Evaluation.id,
|
||||
)
|
||||
.where(
|
||||
Submission.id == submission_id,
|
||||
Submission.account_id == principal.account_id,
|
||||
)
|
||||
)
|
||||
).one_or_none()
|
||||
if row is None:
|
||||
raise SubmissionUnavailable("submission unavailable")
|
||||
submission, evaluation, projection = row
|
||||
as_of = await session.scalar(text("SELECT clock_timestamp()"))
|
||||
accepted = (
|
||||
evaluation.state == "finished"
|
||||
and evaluation.verdict == "accepted"
|
||||
and evaluation.failure_code is None
|
||||
)
|
||||
if projection is not None and (
|
||||
not accepted
|
||||
or projection.attempt_id != evaluation.current_attempt_id
|
||||
or projection.report_digest != evaluation.final_report_digest
|
||||
):
|
||||
raise ValueError("stored performance binding invalid")
|
||||
cpu = memory = None
|
||||
reason = None if accepted else "not_accepted"
|
||||
if projection is not None:
|
||||
cpu, memory = projection.cpu_time_us, projection.memory_peak_bytes
|
||||
elif accepted:
|
||||
# A single owned legacy report, never a full-table JSON scan or promotion.
|
||||
report = decode_report(
|
||||
json.dumps(evaluation.final_report, ensure_ascii=False).encode()
|
||||
)
|
||||
if (
|
||||
report.binding.evaluation_id != evaluation.id
|
||||
or report.binding.attempt_id != evaluation.current_attempt_id
|
||||
or report.verdict.value != "accepted"
|
||||
or not report.cleanup_complete
|
||||
):
|
||||
raise ValueError("stored result binding invalid")
|
||||
cpu, memory = report.run_cpu_time_us, report.run_memory_peak_bytes
|
||||
reason = "environment_unverified"
|
||||
if any(v is not None and not 0 <= v <= I64_MAX for v in (cpu, memory)):
|
||||
raise ValueError("stored measurement out of range")
|
||||
base = dict(
|
||||
submission_id=submission.id,
|
||||
problem_version_id=submission.problem_version_id,
|
||||
language=submission.language,
|
||||
as_of=as_of,
|
||||
comparison_group=None
|
||||
if projection is None
|
||||
else projection.comparison_group,
|
||||
)
|
||||
if projection is None:
|
||||
return SubmissionPerformance(
|
||||
**base,
|
||||
cpu=metric("cpu_microseconds", cpu, reason=reason),
|
||||
memory=metric("bytes", memory, reason=reason),
|
||||
)
|
||||
group = projection.comparison_group
|
||||
watermark = (
|
||||
await session.execute(
|
||||
select(
|
||||
EvaluationPerformance.finished_at,
|
||||
EvaluationPerformance.evaluation_id,
|
||||
)
|
||||
.where(EvaluationPerformance.comparison_group == group)
|
||||
.order_by(
|
||||
EvaluationPerformance.finished_at.desc(),
|
||||
EvaluationPerformance.evaluation_id.desc(),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
).one()
|
||||
key = (submission.id, projection.report_digest, group, tuple(watermark))
|
||||
cached = self._cache.get(key)
|
||||
now = time.monotonic()
|
||||
if cached is not None and now - cached[0] < CACHE_TTL_SECONDS:
|
||||
self._cache.move_to_end(key)
|
||||
return cached[1]
|
||||
metrics = []
|
||||
for column, unit, current in [
|
||||
(EvaluationPerformance.cpu_time_us, "cpu_microseconds", cpu),
|
||||
(EvaluationPerformance.memory_peak_bytes, "bytes", memory),
|
||||
]:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(column, EvaluationPerformance.finished_at)
|
||||
.where(
|
||||
EvaluationPerformance.comparison_group == group,
|
||||
EvaluationPerformance.evaluation_id != evaluation.id,
|
||||
column.is_not(None),
|
||||
)
|
||||
.order_by(
|
||||
EvaluationPerformance.finished_at.desc(),
|
||||
EvaluationPerformance.evaluation_id.desc(),
|
||||
)
|
||||
.limit(MAX_PERFORMANCE_SAMPLES + 1)
|
||||
)
|
||||
).all()
|
||||
metrics.append(metric(unit, current, rows))
|
||||
result = SubmissionPerformance(**base, cpu=metrics[0], memory=metrics[1])
|
||||
self._cache[key] = (now, result)
|
||||
self._cache.move_to_end(key)
|
||||
while len(self._cache) > CACHE_CAPACITY:
|
||||
self._cache.popitem(last=False)
|
||||
return result
|
||||
except (SQLAlchemyError, TimeoutError) as error:
|
||||
raise ValueError("performance database unavailable") from error
|
||||
@@ -1,11 +1,13 @@
|
||||
import base64
|
||||
import json
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, tuple_
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from kf_oj.contracts.judging import MAX_SOURCE_BYTES, ArtifactReference, JudgementReport
|
||||
from kf_oj.contracts.judging import MAX_SOURCE_BYTES, ArtifactReference
|
||||
from kf_oj.contracts.judging_versions import decode_report
|
||||
from kf_oj.contracts.submissions import (
|
||||
CancellationAccepted,
|
||||
PublicCaseResult,
|
||||
@@ -86,7 +88,14 @@ class SubmissionQueries:
|
||||
.where(Submission.id == submission_id, Submission.account_id == principal.account_id)
|
||||
)
|
||||
|
||||
async def page(self, principal: Principal, after: str | None, limit: int) -> SubmissionPage:
|
||||
async def page(
|
||||
self,
|
||||
principal: Principal,
|
||||
after: str | None,
|
||||
limit: int,
|
||||
problem_version_id: UUID | None = None,
|
||||
language: str | None = None,
|
||||
) -> SubmissionPage:
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError("invalid history limit")
|
||||
query = (
|
||||
@@ -96,6 +105,10 @@ class SubmissionQueries:
|
||||
.order_by(Submission.created_at.desc(), Submission.id.desc())
|
||||
.limit(limit + 1)
|
||||
)
|
||||
if problem_version_id is not None:
|
||||
query = query.where(Submission.problem_version_id == problem_version_id)
|
||||
if language is not None:
|
||||
query = query.where(Submission.language == language)
|
||||
if after is not None:
|
||||
query = query.where(tuple_(Submission.created_at, Submission.id) < cursor_decode(after))
|
||||
async with self.sessions() as session:
|
||||
@@ -115,7 +128,11 @@ class SubmissionQueries:
|
||||
if evaluation.state in {"finished", "failed", "cancelled"} and evaluation.final_report:
|
||||
# A single joined snapshot, strict validation and explicit projection prevent leaking
|
||||
# internal bindings, artifacts, hidden case identifiers or partial attempt reports.
|
||||
report = JudgementReport.model_validate(evaluation.final_report)
|
||||
report = decode_report(
|
||||
json.dumps(
|
||||
evaluation.final_report, ensure_ascii=False, separators=(",", ":")
|
||||
).encode()
|
||||
)
|
||||
if str(report.binding.evaluation_id) != str(evaluation.id):
|
||||
raise ValueError("stored result binding invalid")
|
||||
if report.verdict.value != evaluation.verdict:
|
||||
|
||||
@@ -6,7 +6,7 @@ from uuid import uuid4
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
|
||||
from kf_oj.contracts.judging import JobManifest
|
||||
from kf_oj.contracts.judging_v2 import admit_manifest
|
||||
from kf_oj.contracts.submissions import CreateSubmission, SubmissionCreated
|
||||
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||
from kf_oj.infra.messaging.models import Outbox
|
||||
@@ -38,7 +38,9 @@ class SubmissionService:
|
||||
|
||||
async def create(self, principal: Principal, request: CreateSubmission) -> SubmissionCreated:
|
||||
# Validate even trusted non-HTTP callers. No path or argv is accepted.
|
||||
request = CreateSubmission.model_validate(request.model_dump())
|
||||
request = CreateSubmission.model_validate(
|
||||
request.model_dump(exclude={"language"}) | {"language": request.language}
|
||||
)
|
||||
source = request.source.encode("utf-8")
|
||||
request_digest = hashlib.sha256(
|
||||
json.dumps(
|
||||
@@ -92,22 +94,8 @@ class SubmissionService:
|
||||
# but cannot leave an evaluation that refers to non-durable bytes.
|
||||
reference = await self.artifacts.put(session, source)
|
||||
submission_id, evaluation_id = uuid4(), uuid4()
|
||||
manifest = JobManifest(
|
||||
schema_version=1,
|
||||
kind="judge.manifest",
|
||||
evaluation_id=evaluation_id,
|
||||
submission_id=submission_id,
|
||||
problem_version_id=version.version_id,
|
||||
language=request.language,
|
||||
toolchain=version.toolchain,
|
||||
source=reference,
|
||||
cases=version.cases,
|
||||
comparison="exact_bytes",
|
||||
compile_limits=version.compile_limits,
|
||||
run_limits=version.run_limits,
|
||||
job_wall_time_us=version.job_wall_time_us,
|
||||
)
|
||||
self.admission.admit(manifest, 1)
|
||||
manifest = version.manifest(evaluation_id, submission_id, request.language, reference)
|
||||
admit_manifest(self.admission, manifest, 1)
|
||||
submission = Submission(
|
||||
id=submission_id,
|
||||
account_id=principal.account_id,
|
||||
|
||||
@@ -7,10 +7,10 @@ from contextlib import asynccontextmanager
|
||||
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from kf_oj.contracts.judging import (
|
||||
JobManifest,
|
||||
LeaseBinding,
|
||||
decode_wire,
|
||||
from kf_oj.contracts.judging import LeaseBinding
|
||||
from kf_oj.contracts.judging_versions import (
|
||||
Manifest,
|
||||
decode_manifest,
|
||||
manifest_bytes,
|
||||
)
|
||||
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||
@@ -40,7 +40,7 @@ class WorkerAccess:
|
||||
raise WorkerAccessDenied("worker grant rejected")
|
||||
return hashlib.sha256(token.encode("ascii")).hexdigest()
|
||||
|
||||
async def _manifest(self, session, binding: LeaseBinding, token: str) -> JobManifest:
|
||||
async def _manifest(self, session, binding: LeaseBinding, token: str) -> Manifest:
|
||||
row = await repository.leased_submission(session, binding, self._token_hash(token))
|
||||
if row is None:
|
||||
raise WorkerAccessDenied("worker grant rejected")
|
||||
@@ -49,7 +49,7 @@ class WorkerAccess:
|
||||
# runtime roles; use canonical bytes for both retrieval and digest verification.
|
||||
|
||||
try:
|
||||
manifest = decode_wire(JobManifest, json.dumps(attempt.manifest).encode())
|
||||
manifest = decode_manifest(json.dumps(attempt.manifest).encode())
|
||||
if (
|
||||
manifest.evaluation_id != evaluation.id
|
||||
or manifest.submission_id != submission.id
|
||||
@@ -62,17 +62,16 @@ class WorkerAccess:
|
||||
)
|
||||
):
|
||||
raise WorkerAccessDenied("worker snapshot rejected")
|
||||
version = await self.problems.for_submission(session, manifest.problem_version_id)
|
||||
version = await self.problems.for_accepted_execution(
|
||||
session, manifest.problem_version_id
|
||||
)
|
||||
await self.artifacts.check_metadata(session, manifest.source)
|
||||
if (
|
||||
version.problem_id != submission.problem_id
|
||||
or manifest.language not in version.languages
|
||||
or manifest.toolchain != version.toolchain
|
||||
or manifest.cases != version.cases
|
||||
or manifest.compile_limits != version.compile_limits
|
||||
or manifest.run_limits != version.run_limits
|
||||
or manifest.job_wall_time_us != version.job_wall_time_us
|
||||
):
|
||||
expected = version.manifest(
|
||||
manifest.evaluation_id, manifest.submission_id, manifest.language, manifest.source
|
||||
)
|
||||
if version.problem_id != submission.problem_id or manifest_bytes(
|
||||
manifest
|
||||
) != manifest_bytes(expected):
|
||||
raise WorkerAccessDenied("worker snapshot rejected")
|
||||
return manifest
|
||||
except (ValueError, ProblemUnavailable):
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Fixed data-only child. It inherits no DB credentials and executes no uploaded contents."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
VALIDATION_DEADLINE_SECONDS,
|
||||
CandidatePreview,
|
||||
LegacyImportBudget,
|
||||
)
|
||||
from kf_oj.contracts.judging_v2 import NodeAdmission
|
||||
from kf_oj.contracts.problems import PublicLanguageConfiguration, PublicSample
|
||||
from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader
|
||||
from kf_oj.modules.problems.candidate_validation import validate_candidate
|
||||
from kf_oj.modules.problems.import_format import ProblemImport
|
||||
from kf_oj.modules.problems.web_format import decode_web_declaration
|
||||
|
||||
|
||||
def main():
|
||||
# These are ordinary downward-only process limits, independent of the parent's life.
|
||||
resource.setrlimit(resource.RLIMIT_AS, (128 * 1024**2, 128 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (60, 60))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024**2, 64 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
signal.alarm(VALIDATION_DEADLINE_SECONDS)
|
||||
archive, directory, lock, size, digest, candidate, policy_json = sys.argv[1:]
|
||||
archive, directory, lock = int(archive), int(directory), int(lock)
|
||||
os.fstat(lock) # Held inherited flock prevents GC releasing capacity while any child lives.
|
||||
deadline = time.monotonic() + VALIDATION_DEADLINE_SECONDS
|
||||
result = {"failure_code": "invalid_archive"}
|
||||
try:
|
||||
reader = ArchiveReader(archive, int(size), deadline=deadline)
|
||||
reader.verify_digest(digest)
|
||||
result["failure_code"] = "invalid_manifest"
|
||||
declared = decode_web_declaration(reader.read("problem.json", 256 * 1024))
|
||||
if {m.name for m in reader.members} != set(declared.members):
|
||||
raise ValueError("unexpected package members")
|
||||
manifest = declared.manifest
|
||||
result["failure_code"] = "invalid_data"
|
||||
statement = reader.read(manifest.statement_file, 256 * 1024).decode("utf-8")
|
||||
sample_data = {
|
||||
name: reader.read(name, 65536)
|
||||
for name in sorted({n for s in manifest.samples for n in (s.input_file, s.output_file)})
|
||||
}
|
||||
samples = tuple(
|
||||
PublicSample(
|
||||
input=base64.b64encode(sample_data[s.input_file]).decode("ascii"),
|
||||
output=base64.b64encode(sample_data[s.output_file]).decode("ascii"),
|
||||
)
|
||||
for s in manifest.samples
|
||||
)
|
||||
references = reader.extract_remaining(directory)
|
||||
reader.verify_unchanged()
|
||||
legacy = isinstance(manifest, ProblemImport)
|
||||
preview = CandidatePreview(
|
||||
candidate_id=UUID(candidate),
|
||||
slug=manifest.slug,
|
||||
title=manifest.title,
|
||||
statement=statement,
|
||||
samples=samples,
|
||||
judging_schema_version=manifest.schema_version,
|
||||
languages=manifest.languages,
|
||||
legacy_budget=LegacyImportBudget(
|
||||
compile_limits=manifest.compile_limits,
|
||||
run_limits=manifest.run_limits,
|
||||
job_wall_time_us=manifest.job_wall_time_us,
|
||||
)
|
||||
if legacy
|
||||
else None,
|
||||
language_configurations=()
|
||||
if legacy
|
||||
else tuple(
|
||||
PublicLanguageConfiguration.from_configuration(c)
|
||||
for c in manifest.language_configurations
|
||||
),
|
||||
case_count=len(manifest.cases),
|
||||
)
|
||||
result["failure_code"] = "policy_rejected"
|
||||
validated = validate_candidate(
|
||||
manifest.model_dump(mode="json"),
|
||||
preview.model_dump(mode="json"),
|
||||
{k: {"digest": v[0], "bytes": v[1]} for k, v in references.items()},
|
||||
TypeAdapter(NodeAdmission).validate_json(policy_json),
|
||||
)
|
||||
result = {
|
||||
"manifest": validated.manifest,
|
||||
"preview": validated.preview.model_dump(mode="json"),
|
||||
"references": {k: v.model_dump(mode="json") for k, v in validated.references.items()},
|
||||
"manifest_digest": validated.digest,
|
||||
}
|
||||
except ArchiveError as error:
|
||||
result = {
|
||||
"failure_code": "digest_mismatch"
|
||||
if str(error) == "archive_digest_mismatch"
|
||||
else "invalid_archive"
|
||||
}
|
||||
except (ValueError, OSError, MemoryError):
|
||||
result = {"failure_code": result["failure_code"]}
|
||||
payload = json.dumps(result, separators=(",", ":"), ensure_ascii=True).encode()
|
||||
if len(payload) > 4 * 1024**2:
|
||||
payload = b'{"failure_code":"invalid_data"}'
|
||||
sys.stdout.buffer.write(payload)
|
||||
sys.stdout.buffer.flush()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Read only publisher-owned counts; never claim, publish or acknowledge a task."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
from kf_oj.infra.messaging.outbox_queries import OutboxQueries
|
||||
|
||||
|
||||
async def run() -> dict:
|
||||
engine = build_engine(Settings())
|
||||
try:
|
||||
return await OutboxQueries(build_session_factory(engine)).status()
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
print(json.dumps(asyncio.run(asyncio.wait_for(run(), timeout=6)), sort_keys=True))
|
||||
except Exception:
|
||||
raise SystemExit("outbox status unavailable") from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Ordinary import service composition. Provision its dedicated DB role and cgroup separately."""
|
||||
|
||||
import asyncio
|
||||
import signal
|
||||
import sys
|
||||
|
||||
from sqlalchemy.exc import DBAPIError
|
||||
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.artifacts.repository import ArtifactRepository
|
||||
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||
from kf_oj.infra.artifacts.store import ArtifactStore
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
from kf_oj.modules.identity.service import ReservedMaintenance
|
||||
from kf_oj.modules.operations.service import MaintenanceAudit
|
||||
from kf_oj.modules.problems.import_worker import ImportWorker, verify_worker_limits
|
||||
|
||||
|
||||
async def run():
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for event in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(event, stop.set)
|
||||
settings = Settings()
|
||||
if (
|
||||
settings.import_staging_root is None
|
||||
or settings.artifact_root is None
|
||||
or settings.judge_admission is None
|
||||
or settings.import_worker_cgroup is None
|
||||
):
|
||||
raise ValueError("import service configuration required")
|
||||
verify_worker_limits(settings.import_worker_cgroup)
|
||||
engine = build_engine(settings)
|
||||
store = ArtifactStore(
|
||||
settings.artifact_root,
|
||||
max_bytes=settings.artifact_max_bytes,
|
||||
max_files=settings.artifact_max_files,
|
||||
)
|
||||
staging = ImportStaging(settings.import_staging_root)
|
||||
try:
|
||||
worker = ImportWorker(
|
||||
build_session_factory(engine),
|
||||
staging,
|
||||
ArtifactRepository(store),
|
||||
ReservedMaintenance(),
|
||||
MaintenanceAudit(),
|
||||
settings.judge_admission,
|
||||
settings.import_worker_cgroup,
|
||||
)
|
||||
while not stop.is_set():
|
||||
if not await worker.once():
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=1)
|
||||
except TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
staging.close()
|
||||
store.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
asyncio.run(run())
|
||||
except (ValueError, OSError, DBAPIError):
|
||||
print(
|
||||
"import service unavailable; check trusted configuration and resource limits",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Ordinary import-only read status; no parser, staging mutation or judging writes."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.infra.db.session import build_engine, build_session_factory
|
||||
from kf_oj.modules.problems.operations import ImportOperationQueries
|
||||
|
||||
|
||||
async def run() -> dict:
|
||||
settings = Settings()
|
||||
if settings.judge_admission is None:
|
||||
raise ValueError("import admission required")
|
||||
engine = build_engine(settings)
|
||||
try:
|
||||
return await ImportOperationQueries(
|
||||
build_session_factory(engine), settings.judge_admission
|
||||
).status()
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
result = asyncio.run(asyncio.wait_for(run(), timeout=6))
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
except Exception:
|
||||
raise SystemExit("import status unavailable") from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -39,14 +41,20 @@ def postgres_url():
|
||||
|
||||
|
||||
def migrate(url: str, command: str, target: str | None = None) -> None:
|
||||
subprocess.run(
|
||||
[str(Path(__file__).resolve().parents[1] / ".venv/bin/alembic"), command]
|
||||
+ ([target] if target else []),
|
||||
cwd=Path(__file__).resolve().parents[1],
|
||||
env=os.environ | {"KF_OJ_DATABASE_URL": url},
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
try:
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", command] + ([target] if target else []),
|
||||
cwd=Path(__file__).resolve().parents[1],
|
||||
env=os.environ | {"KF_OJ_DATABASE_URL": url},
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as error:
|
||||
diagnostic = ((error.stdout or b"") + (error.stderr or b"")).decode(errors="replace")[
|
||||
-8192:
|
||||
]
|
||||
diagnostic = re.sub(r"postgresql(?:\+asyncpg)?://[^\s]+", "[test database URL]", diagnostic)
|
||||
raise AssertionError(f"test migration {command} failed: " + diagnostic) from None
|
||||
|
||||
|
||||
async def connect(url: str):
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Finite adverse archive/private-storage acceptance: independent capped VM only."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import struct
|
||||
import time
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from test_admin_import_vm import independent_import_vm as independent_import_vm
|
||||
from test_admin_import_vm import package
|
||||
|
||||
from kf_oj.contracts.judging_v2 import AdmissionRegistry
|
||||
from kf_oj.infra.artifacts.archive_reader import ArchiveError, ArchiveReader
|
||||
from kf_oj.infra.artifacts.staging import ImportStaging
|
||||
from kf_oj.infra.artifacts.store import ArtifactError, ArtifactStore
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def policy():
|
||||
return AdmissionRegistry.model_validate_json(
|
||||
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
|
||||
)
|
||||
|
||||
|
||||
def proof(name, value):
|
||||
print(
|
||||
"KF_FINAL_ACCEPTANCE " + json.dumps({"name": name, "result": value}, default=str),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def test_adverse_zip_and_private_storage_boundaries(independent_import_vm, tmp_path):
|
||||
outside = tmp_path / "outside-sentinel"
|
||||
outside.write_bytes(b"unchanged synthetic sentinel")
|
||||
original = hashlib.sha256(outside.read_bytes()).hexdigest()
|
||||
good, _ = package(policy())
|
||||
cases = {}
|
||||
for name in ("../outside-sentinel", "/outside-sentinel", "a/b", r"a\b", "a:b"):
|
||||
cases["path-" + repr(name)] = package(
|
||||
policy(), payload_mutation=lambda p, n=name: p.update({n: b"bad"})
|
||||
)[0]
|
||||
for kind in (
|
||||
stat.S_IFLNK,
|
||||
stat.S_IFCHR,
|
||||
stat.S_IFBLK,
|
||||
stat.S_IFIFO,
|
||||
stat.S_IFSOCK,
|
||||
stat.S_IFDIR,
|
||||
):
|
||||
out = io.BytesIO()
|
||||
with zipfile.ZipFile(out, "w") as archive:
|
||||
entry = zipfile.ZipInfo("unsafe")
|
||||
entry.create_system = 3
|
||||
entry.external_attr = (kind | 0o600) << 16
|
||||
archive.writestr(entry, b"synthetic")
|
||||
cases[f"type-{kind}"] = out.getvalue()
|
||||
for duplicate in ("statement.md", "STATEMENT.MD"):
|
||||
out = io.BytesIO(good)
|
||||
with zipfile.ZipFile(out, "a") as archive:
|
||||
archive.writestr(duplicate, b"different")
|
||||
cases["duplicate-" + duplicate] = out.getvalue()
|
||||
for signature in (
|
||||
b"PK\x03\x04",
|
||||
b"\x1f\x8b",
|
||||
b"BZh",
|
||||
b"\xfd7zXZ\x00",
|
||||
b"Rar!\x1a\x07",
|
||||
b"7z\xbc\xaf\x27\x1c",
|
||||
b"X" * 257 + b"ustar",
|
||||
):
|
||||
cases["nested-" + signature[:6].hex()] = package(
|
||||
policy(),
|
||||
payload_mutation=lambda p, s=signature: p.update({"hidden.in": s + b"synthetic"}),
|
||||
)[0]
|
||||
central = good.index(b"PK\x01\x02")
|
||||
for label, local_offset, central_offset, value, fmt in (
|
||||
("encrypted", 6, 8, 1, "<H"),
|
||||
("unsupported", 8, 10, 99, "<H"),
|
||||
("declared-size", 22, 24, 67108865, "<I"),
|
||||
("crc", 14, 16, 1, "<I"),
|
||||
):
|
||||
blob = bytearray(good)
|
||||
struct.pack_into(fmt, blob, local_offset, value)
|
||||
struct.pack_into(fmt, blob, central + central_offset, value)
|
||||
cases[label] = bytes(blob)
|
||||
for label, extra in (("zip64", b"\x01\x00\x00\x00"), ("unknown-extra", b"\x99\x99\x00\x00")):
|
||||
out = io.BytesIO()
|
||||
with zipfile.ZipFile(out, "w") as archive:
|
||||
entry = zipfile.ZipInfo("data")
|
||||
entry.extra = extra
|
||||
archive.writestr(entry, b"synthetic")
|
||||
cases[label] = out.getvalue()
|
||||
out = io.BytesIO()
|
||||
with zipfile.ZipFile(out, "w") as archive:
|
||||
for i in range(535):
|
||||
archive.writestr(f"f{i}", b"")
|
||||
cases["file-count"] = out.getvalue()
|
||||
# Actual bounded inflation exceeds the forged declared size. The 65 MiB stream
|
||||
# is produced in chunks under the VM cap; no 65 MiB object is allocated.
|
||||
out = io.BytesIO()
|
||||
with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
with archive.open("inflate", "w") as member:
|
||||
for _ in range(1040):
|
||||
member.write(b"A" * 65536)
|
||||
blob = bytearray(out.getvalue())
|
||||
center = blob.index(b"PK\x01\x02")
|
||||
struct.pack_into("<I", blob, 22, 67108864)
|
||||
struct.pack_into("<I", blob, center + 24, 67108864)
|
||||
cases["actual-inflate-over-64MiB"] = bytes(blob)
|
||||
refused = []
|
||||
for index, (label, blob) in enumerate(cases.items()):
|
||||
file = tmp_path / f"archive-{index}"
|
||||
file.write_bytes(blob)
|
||||
file.chmod(0o600)
|
||||
fd = os.open(file, os.O_RDONLY)
|
||||
try:
|
||||
with pytest.raises(ArchiveError) as error:
|
||||
reader = ArchiveReader(fd, len(blob), deadline=time.monotonic() + 60)
|
||||
for member in reader.members:
|
||||
for _ in reader.chunks(member.name, 67108864):
|
||||
pass
|
||||
assert str(error.value) in ("invalid_archive", "archive_deadline")
|
||||
refused.append(label)
|
||||
finally:
|
||||
os.close(fd)
|
||||
file.unlink()
|
||||
file = tmp_path / "mutable"
|
||||
file.write_bytes(good)
|
||||
file.chmod(0o600)
|
||||
fd = os.open(file, os.O_RDONLY)
|
||||
try:
|
||||
reader = ArchiveReader(fd, len(good), deadline=time.monotonic() + 60)
|
||||
with file.open("r+b") as writer:
|
||||
writer.write(b"X")
|
||||
with pytest.raises(ArchiveError, match="archive_changed"):
|
||||
reader.verify_unchanged()
|
||||
finally:
|
||||
os.close(fd)
|
||||
store = ArtifactStore(tmp_path / "objects", max_bytes=4, max_files=2)
|
||||
try:
|
||||
ref = store.put(b"1234")
|
||||
with pytest.raises(ArtifactError, match="capacity"):
|
||||
store.put(b"5")
|
||||
assert store.read(ref) == b"1234"
|
||||
store.remove(ref.digest)
|
||||
recovered = store.put(b"5")
|
||||
assert store.read(recovered) == b"5"
|
||||
finally:
|
||||
store.close()
|
||||
link_root = tmp_path / "linked-store"
|
||||
link_root.symlink_to(tmp_path / "objects", target_is_directory=True)
|
||||
with pytest.raises((ArtifactError, OSError)):
|
||||
ArtifactStore(link_root)
|
||||
staging = ImportStaging(tmp_path / "staging")
|
||||
operation_id = uuid4()
|
||||
(tmp_path / "staging" / str(operation_id)).symlink_to(tmp_path, target_is_directory=True)
|
||||
try:
|
||||
with pytest.raises((ArtifactError, OSError)):
|
||||
staging.operation(operation_id)
|
||||
finally:
|
||||
staging.close()
|
||||
assert hashlib.sha256(outside.read_bytes()).hexdigest() == original
|
||||
proof(
|
||||
"archive_and_storage",
|
||||
{
|
||||
"refused": refused,
|
||||
"outside_unchanged": True,
|
||||
"content_never_executed": True,
|
||||
"changed_descriptor_rejected": True,
|
||||
"quota_recovered": True,
|
||||
"symlink_roots_rejected": True,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Upgrade, adverse storage and cross-process admission: capped independent VM only."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from conftest import connect
|
||||
from sqlalchemy.engine import make_url
|
||||
from test_admin_import_vm import independent_import_vm as independent_import_vm
|
||||
|
||||
from kf_oj.contracts.judging_v2 import AdmissionRegistry
|
||||
from kf_oj.modules.identity.passwords import hash_password
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def policy():
|
||||
return AdmissionRegistry.model_validate_json(
|
||||
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
|
||||
)
|
||||
|
||||
|
||||
def proof(name, value):
|
||||
print(
|
||||
"KF_FINAL_ACCEPTANCE " + json.dumps({"name": name, "result": value}, default=str),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def test_cross_process_admission_and_worker_identity(independent_import_vm, postgres_url, tmp_path):
|
||||
groups = independent_import_vm
|
||||
roles = {name: "final_" + name + "_" + uuid4().hex for name in ("api", "worker")}
|
||||
users = {f"capacity_admin_{i}": uuid4() for i in range(5)}
|
||||
password = "Synthetic capacity acceptance only 2026!"
|
||||
artifact, staging = tmp_path / "artifacts", tmp_path / "staging"
|
||||
artifact.mkdir(mode=0o700)
|
||||
staging.mkdir(mode=0o700)
|
||||
processes, logs = {}, {}
|
||||
base = {
|
||||
k: v
|
||||
for k, v in os.environ.items()
|
||||
if k
|
||||
in (
|
||||
"PATH",
|
||||
"PYTHONPATH",
|
||||
"LD_LIBRARY_PATH",
|
||||
"LC_ALL",
|
||||
"PYTHONUNBUFFERED",
|
||||
"PYTHONDONTWRITEBYTECODE",
|
||||
)
|
||||
}
|
||||
|
||||
def enter(group):
|
||||
def apply():
|
||||
(group / "cgroup.procs").write_text(str(os.getpid()))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (67108864, 67108864))
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
|
||||
return apply
|
||||
|
||||
def start(name, *, bad_identity=False, bad_group=False):
|
||||
worker = name == "worker"
|
||||
group = groups["import-worker" if worker else "api"]
|
||||
env = base | {
|
||||
"KF_OJ_DATABASE_URL": make_url(postgres_url)
|
||||
.set(username=roles["api" if bad_identity or not worker else "worker"])
|
||||
.render_as_string(hide_password=False),
|
||||
"KF_OJ_ENVIRONMENT": "test",
|
||||
"KF_OJ_ARTIFACT_ROOT": str(artifact),
|
||||
"KF_OJ_IMPORT_STAGING_ROOT": str(staging),
|
||||
"KF_OJ_JUDGE_ADMISSION": policy().model_dump_json(),
|
||||
"KF_OJ_BROWSER_ORIGINS": '["http://127.0.0.1:58000"]',
|
||||
}
|
||||
if worker:
|
||||
env["KF_OJ_IMPORT_WORKER_CGROUP"] = str(groups["api"] if bad_group else group)
|
||||
command = [sys.executable, "-m", "kf_oj.workers.problem_import"]
|
||||
else:
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
"kf_oj.api.app:create_app",
|
||||
"--factory",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
"58000" if name == "api0" else "58001",
|
||||
"--no-proxy-headers",
|
||||
"--limit-concurrency",
|
||||
"4",
|
||||
"--backlog",
|
||||
"16",
|
||||
]
|
||||
log = (tmp_path / (name + ".log")).open("a")
|
||||
logs[name] = log
|
||||
processes[name] = subprocess.Popen(
|
||||
command, env=env, stdout=log, stderr=log, preexec_fn=enter(group)
|
||||
)
|
||||
|
||||
def stop(name):
|
||||
process = processes.pop(name, None)
|
||||
if process and process.poll() is None:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=20)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=10)
|
||||
if name in logs:
|
||||
logs.pop(name).close()
|
||||
|
||||
async def run():
|
||||
connection = await connect(postgres_url)
|
||||
try:
|
||||
for name, membership in (("api", "kf_oj_platform"), ("worker", "kf_oj_problem_import")):
|
||||
await connection.execute(
|
||||
f"CREATE ROLE {roles[name]} LOGIN NOSUPERUSER "
|
||||
"NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS"
|
||||
)
|
||||
await connection.execute(f"GRANT {membership} TO {roles[name]}")
|
||||
for name, identifier in users.items():
|
||||
await connection.execute(
|
||||
"INSERT INTO accounts(id,username,password_hash) VALUES($1,$2,$3)",
|
||||
identifier,
|
||||
name,
|
||||
hash_password(name, password),
|
||||
)
|
||||
await connection.execute(
|
||||
"SELECT * FROM kf_manage_account_authority($1,$2)",
|
||||
name,
|
||||
"bootstrap" if name.endswith("_0") else "grant",
|
||||
)
|
||||
for wrong in ("identity", "cgroup"):
|
||||
start("worker", bad_identity=wrong == "identity", bad_group=wrong == "cgroup")
|
||||
assert await asyncio.to_thread(processes["worker"].wait, 120) != 0
|
||||
assert not await connection.fetchval(
|
||||
"SELECT worker_ready_until>clock_timestamp() "
|
||||
"FROM problem_import_pool WHERE id=1"
|
||||
)
|
||||
stop("worker")
|
||||
start("worker")
|
||||
deadline = time.monotonic() + 180
|
||||
while not await connection.fetchval(
|
||||
"SELECT worker_ready_until>clock_timestamp() FROM problem_import_pool WHERE id=1"
|
||||
):
|
||||
assert time.monotonic() < deadline and processes["worker"].poll() is None
|
||||
await asyncio.sleep(0.1)
|
||||
# Quiesce consumption. The actual verified heartbeat is retained for
|
||||
# finite admission checks; synthetic capacity rows cannot be parsed.
|
||||
stop("worker")
|
||||
start("api0")
|
||||
start("api1")
|
||||
prefix = "/api/v1/admin/problem-imports"
|
||||
async with (
|
||||
httpx.AsyncClient(base_url="http://127.0.0.1:58000", timeout=30) as first,
|
||||
httpx.AsyncClient(base_url="http://127.0.0.1:58001", timeout=30) as second,
|
||||
):
|
||||
for client in (first, second):
|
||||
deadline = time.monotonic() + 300
|
||||
while True:
|
||||
try:
|
||||
if (await client.get("/api/v1/health/live")).status_code == 200:
|
||||
break
|
||||
except httpx.ConnectError:
|
||||
pass
|
||||
assert time.monotonic() < deadline
|
||||
await asyncio.sleep(0.1)
|
||||
result = await client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={
|
||||
"username": "capacity_admin_0",
|
||||
"password": password,
|
||||
},
|
||||
headers={"Origin": "http://127.0.0.1:58000"},
|
||||
)
|
||||
assert result.status_code == 200, result.text
|
||||
|
||||
def headers(client):
|
||||
return {
|
||||
"Origin": "http://127.0.0.1:58000",
|
||||
"X-CSRF-Token": client.cookies["kf_csrf"],
|
||||
}
|
||||
|
||||
def body(key=None):
|
||||
return {
|
||||
"operation_id": str(key or uuid4()),
|
||||
"archive_digest": "a" * 64,
|
||||
"archive_bytes": 100,
|
||||
}
|
||||
|
||||
async def refresh_worker():
|
||||
previous = await connection.fetchval(
|
||||
"SELECT worker_ready_until FROM problem_import_pool WHERE id=1"
|
||||
)
|
||||
start("worker")
|
||||
deadline = time.monotonic() + 120
|
||||
while True:
|
||||
current = await connection.fetchval(
|
||||
"SELECT worker_ready_until FROM problem_import_pool WHERE id=1"
|
||||
)
|
||||
if current and (previous is None or current > previous):
|
||||
break
|
||||
assert time.monotonic() < deadline
|
||||
assert processes["worker"].poll() is None
|
||||
await asyncio.sleep(0.1)
|
||||
stop("worker")
|
||||
|
||||
await refresh_worker()
|
||||
request = body()
|
||||
replies = await asyncio.gather(
|
||||
*(
|
||||
client.post(prefix, json=request, headers=headers(client))
|
||||
for client in (first, second)
|
||||
)
|
||||
)
|
||||
# Identical UUIDs converge across independent API processes. A
|
||||
# transient root-lock refusal is retried once after the winner.
|
||||
assert 201 in [r.status_code for r in replies]
|
||||
retry = await second.post(prefix, json=request, headers=headers(second))
|
||||
assert retry.status_code == 201
|
||||
assert (
|
||||
await connection.fetchval("SELECT count(*) FROM problem_import_operations") == 1
|
||||
)
|
||||
assert (
|
||||
await connection.fetchval("SELECT reserved_bytes FROM problem_import_pool")
|
||||
== 134217728
|
||||
)
|
||||
other = await first.post(prefix, json=body(), headers=headers(first))
|
||||
assert other.status_code == 503
|
||||
discarded = await first.post(
|
||||
prefix + "/" + request["operation_id"] + "/discard", headers=headers(first)
|
||||
)
|
||||
assert discarded.status_code == 200, discarded.text
|
||||
assert (
|
||||
await connection.fetchval("SELECT reserved_bytes FROM problem_import_pool") == 0
|
||||
)
|
||||
|
||||
async def capacity_rows(count):
|
||||
# Explicit unmeasured database fixtures exercise owner/global
|
||||
# admission boundaries; no ready candidate or grading is forged.
|
||||
await connection.execute(
|
||||
"DELETE FROM problem_import_operations WHERE state='uploaded'"
|
||||
)
|
||||
identifiers = list(users.values())
|
||||
for i in range(count):
|
||||
await connection.execute(
|
||||
"""
|
||||
INSERT INTO problem_import_operations(id,owner_id,authorization_version,
|
||||
archive_digest,archive_bytes,state,reserved_bytes,expires_at,updated_at)
|
||||
VALUES($1,$2,2,$3,100,'uploaded',0,
|
||||
clock_timestamp()+interval '1 hour',clock_timestamp())
|
||||
""",
|
||||
uuid4(),
|
||||
identifiers[0] if count == 10 else identifiers[1 + i % 4],
|
||||
"b" * 64,
|
||||
)
|
||||
|
||||
results = {}
|
||||
for label, count in (("owner_limit", 10), ("global_limit", 32)):
|
||||
await capacity_rows(0)
|
||||
await refresh_worker()
|
||||
await capacity_rows(count)
|
||||
before = await connection.fetchval(
|
||||
"SELECT count(*) FROM problem_import_operations"
|
||||
)
|
||||
response = await second.post(prefix, json=body(), headers=headers(second))
|
||||
assert response.status_code == 503
|
||||
assert (
|
||||
await connection.fetchval("SELECT count(*) FROM problem_import_operations")
|
||||
== before
|
||||
)
|
||||
results[label] = {
|
||||
"fixture_count": count,
|
||||
"actual_http_status": 503,
|
||||
"no_new_receipt": True,
|
||||
}
|
||||
await capacity_rows(0)
|
||||
await refresh_worker()
|
||||
resumed = body()
|
||||
result = await second.post(prefix, json=resumed, headers=headers(second))
|
||||
assert result.status_code == 201, result.text
|
||||
assert (
|
||||
await first.post(
|
||||
prefix + "/" + resumed["operation_id"] + "/discard", headers=headers(first)
|
||||
)
|
||||
).status_code == 200
|
||||
assert (
|
||||
await connection.fetchval("SELECT reserved_bytes FROM problem_import_pool") == 0
|
||||
)
|
||||
proof(
|
||||
"cross_process_admission",
|
||||
results
|
||||
| {
|
||||
"two_actual_api_processes": True,
|
||||
"identical_uuid_one_receipt": True,
|
||||
"single_slot_http_status": 503,
|
||||
"wrong_worker_identity_and_group_exit_nonzero": True,
|
||||
"ordinary_worker_heartbeat_verified": True,
|
||||
"recovery_new_operation_201": True,
|
||||
"remaining_reserved_bytes": 0,
|
||||
},
|
||||
)
|
||||
finally:
|
||||
for name in list(processes):
|
||||
stop(name)
|
||||
for role in roles.values():
|
||||
await connection.execute(f"DROP OWNED BY {role}")
|
||||
await connection.execute(f"DROP ROLE {role}")
|
||||
await connection.close()
|
||||
|
||||
asyncio.run(run())
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Contract checks only; upload/parser/security evidence requires the independent VM."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from kf_oj.contracts.admin_problems import (
|
||||
CandidatePreview,
|
||||
CreateProblemImport,
|
||||
LegacyImportBudget,
|
||||
ProblemImportCapabilities,
|
||||
ProblemImportDetail,
|
||||
)
|
||||
from kf_oj.contracts.judging import AdmissionPolicy
|
||||
from kf_oj.modules.problems.import_format import ImportEnvelope
|
||||
from kf_oj.modules.problems.web_format import decode_web_declaration
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def legacy_manifest():
|
||||
policy = AdmissionPolicy.model_validate_json(
|
||||
(ROOT / "tests/fixtures/contracts/judge-admission.valid.json").read_bytes()
|
||||
)
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"slug": "web-fixture",
|
||||
"title": "Public",
|
||||
"statement_file": "statement.md",
|
||||
"languages": ["c17", "cpp20"],
|
||||
"toolchain": policy.toolchain.model_dump(mode="json"),
|
||||
"comparison": "exact_bytes",
|
||||
"compile_limits": policy.compile_ceiling.model_dump(),
|
||||
"run_limits": policy.run_ceiling.model_dump(),
|
||||
"job_wall_time_us": 60000000,
|
||||
"cases": [{"key": "binary", "input_file": "hidden.in", "expected_file": "hidden.out"}],
|
||||
"samples": [{"input_file": "sample.in", "output_file": "sample.out"}],
|
||||
}
|
||||
|
||||
|
||||
def test_web_declaration_separates_public_data_and_preserves_local_format():
|
||||
source = legacy_manifest()
|
||||
original = copy.deepcopy(source)
|
||||
declared = decode_web_declaration(json.dumps(source).encode())
|
||||
assert declared.members == frozenset(
|
||||
("problem.json", "statement.md", "hidden.in", "hidden.out", "sample.in", "sample.out")
|
||||
)
|
||||
assert declared.manifest.model_dump(mode="json") == original
|
||||
assert ImportEnvelope.model_validate(source).root == declared.manifest
|
||||
for mutation in (
|
||||
lambda x: x["samples"][0].update(input_file="hidden.in"),
|
||||
lambda x: x.update(statement_file="problem.json"),
|
||||
lambda x: x["cases"][0].update(input_file="Statement.md"),
|
||||
lambda x: x["cases"][0].update(input_file="hidden.zip"),
|
||||
lambda x: x.update(languages=["rust"]),
|
||||
lambda x: x.update(download_url="https://example.invalid/private"),
|
||||
):
|
||||
value = copy.deepcopy(source)
|
||||
mutation(value)
|
||||
with pytest.raises(ValueError):
|
||||
decode_web_declaration(json.dumps(value).encode())
|
||||
with pytest.raises(ValueError):
|
||||
decode_web_declaration(
|
||||
json.dumps(source)
|
||||
.replace('"schema_version": 1', '"schema_version": 1, "schema_version": 1')
|
||||
.encode()
|
||||
)
|
||||
|
||||
|
||||
def test_import_operation_key_and_preview_have_no_private_payload_fields():
|
||||
operation = {
|
||||
"schema_version": 1,
|
||||
"operation_id": str(uuid4()),
|
||||
"archive_digest": "a" * 64,
|
||||
"archive_bytes": 64 * 1024**2,
|
||||
}
|
||||
assert CreateProblemImport.model_validate(operation).archive_bytes == 64 * 1024**2
|
||||
for field, value in (
|
||||
("archive_bytes", True),
|
||||
("archive_bytes", 64 * 1024**2 + 1),
|
||||
("url", "https://example.invalid"),
|
||||
("role", "admin"),
|
||||
):
|
||||
with pytest.raises(ValidationError):
|
||||
CreateProblemImport.model_validate(operation | {field: value})
|
||||
manifest = ImportEnvelope.model_validate(legacy_manifest()).root
|
||||
preview = CandidatePreview(
|
||||
candidate_id=uuid4(),
|
||||
slug=manifest.slug,
|
||||
title=manifest.title,
|
||||
statement="Public only",
|
||||
samples=(),
|
||||
judging_schema_version=1,
|
||||
languages=manifest.languages,
|
||||
legacy_budget=LegacyImportBudget(
|
||||
compile_limits=manifest.compile_limits,
|
||||
run_limits=manifest.run_limits,
|
||||
job_wall_time_us=manifest.job_wall_time_us,
|
||||
),
|
||||
case_count=1,
|
||||
)
|
||||
now = datetime.now(UTC)
|
||||
response = ProblemImportDetail(
|
||||
operation_id=uuid4(),
|
||||
state="ready",
|
||||
expected_archive_digest="a" * 64,
|
||||
archive_bytes=100,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
expires_at=now + timedelta(days=1),
|
||||
failure_code=None,
|
||||
candidate_id=preview.candidate_id,
|
||||
published_version_id=None,
|
||||
candidate=preview,
|
||||
)
|
||||
assert set(response.candidate.model_dump()) == {
|
||||
"candidate_id",
|
||||
"slug",
|
||||
"title",
|
||||
"statement",
|
||||
"samples",
|
||||
"judging_schema_version",
|
||||
"languages",
|
||||
"legacy_budget",
|
||||
"language_configurations",
|
||||
"case_count",
|
||||
}
|
||||
with pytest.raises(ValidationError):
|
||||
CandidatePreview.model_validate(preview.model_dump() | {"input_digest": "a" * 64})
|
||||
capabilities = ProblemImportCapabilities(uploads_available=False, validation_available=False)
|
||||
assert (
|
||||
capabilities.limits.global_concurrency == 1 and capabilities.limits.owner_candidates == 10
|
||||
)
|
||||
|
||||
|
||||
def test_standalone_import_worker_resolves_all_referenced_metadata():
|
||||
# A separate trusted interpreter avoids accidental model registration by API test imports.
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
script = """
|
||||
import kf_oj.workers.problem_import
|
||||
from kf_oj.infra.db.base import Base
|
||||
for table in Base.metadata.tables.values():
|
||||
for foreign in table.foreign_keys:
|
||||
assert foreign.column.table is not None
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script], capture_output=True, text=True, timeout=15
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,223 @@
|
||||
"""Approved baseline upgrade/retention: actual PostgreSQL in an independent capped VM."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from conftest import connect, migrate
|
||||
from sqlalchemy.engine import make_url
|
||||
from test_admin_import_vm import independent_import_vm as independent_import_vm
|
||||
from test_admin_import_vm import package
|
||||
|
||||
from kf_oj.contracts.judging_v2 import AdmissionRegistry
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def policy():
|
||||
return AdmissionRegistry.model_validate_json(
|
||||
(ROOT / "tests/fixtures/contracts-v2/judge-registry.valid.json").read_bytes()
|
||||
)
|
||||
|
||||
|
||||
def proof(name, value):
|
||||
print(
|
||||
"KF_FINAL_ACCEPTANCE " + json.dumps({"name": name, "result": value}, default=str),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def test_baseline_upgrade_preserves_existing_data(independent_import_vm, postgres_url, tmp_path):
|
||||
baseline = Path("/kf/baseline-platform")
|
||||
assert baseline.is_dir(), "curated approved baseline required"
|
||||
url = make_url(postgres_url).set(database="kf_oj_test_upgrade_" + uuid4().hex)
|
||||
rendered = url.render_as_string(hide_password=False)
|
||||
|
||||
async def manage(command):
|
||||
connection = await connect(postgres_url)
|
||||
try:
|
||||
await connection.execute(command)
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
def old_run(args):
|
||||
result = subprocess.run(
|
||||
[sys.executable, *args],
|
||||
cwd=baseline,
|
||||
env=os.environ
|
||||
| {
|
||||
"PYTHONPATH": "/kf/baseline-platform/src:/kf/site",
|
||||
"KF_OJ_DATABASE_URL": rendered,
|
||||
"KF_BASELINE_FIXTURES": str(tmp_path),
|
||||
},
|
||||
capture_output=True,
|
||||
timeout=300,
|
||||
)
|
||||
assert result.returncode == 0, (result.stdout + result.stderr).decode(errors="replace")[
|
||||
-2000:
|
||||
]
|
||||
return result
|
||||
|
||||
# Real old-schema data is created through the old public services. No student
|
||||
# source executes here; historical report fixtures are explicitly unmeasured.
|
||||
for multi in (False, True):
|
||||
directory = tmp_path / ("v2" if multi else "v1")
|
||||
directory.mkdir(mode=0o700)
|
||||
_, payload = package(policy(), multi=multi)
|
||||
for name, data in payload.items():
|
||||
(directory / name).write_bytes(data)
|
||||
(directory / name).chmod(0o600)
|
||||
seed = """
|
||||
import asyncio,hashlib,json,os,secrets
|
||||
from datetime import datetime,timedelta,timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
from sqlalchemy import text,update
|
||||
from kf_oj.api.app import create_app
|
||||
from kf_oj.core.settings import Settings
|
||||
from kf_oj.contracts.judging import ArtifactReference
|
||||
from kf_oj.contracts.judging_v2 import AdmissionRegistry
|
||||
from kf_oj.contracts.judging_versions import manifest_bytes
|
||||
from kf_oj.contracts.submissions import CreateSubmission
|
||||
from kf_oj.modules.submissions.models import Evaluation,JudgeAttempt
|
||||
async def main():
|
||||
root=Path(os.environ['KF_BASELINE_FIXTURES'])
|
||||
fixture=Path('/kf/tests/fixtures/contracts-v2/judge-registry.valid.json')
|
||||
policy=AdmissionRegistry.model_validate_json(fixture.read_bytes())
|
||||
database=os.environ['KF_OJ_DATABASE_URL']
|
||||
app=create_app(Settings(_env_file=None,database_url=database,
|
||||
artifact_root=root/'objects',judge_admission=policy))
|
||||
async with app.router.lifespan_context(app):
|
||||
password='Synthetic upgrade acceptance only 2026!'
|
||||
user=await app.state.identity.create_account('upgrade_student',password)
|
||||
for name in ('v1','v2'):
|
||||
version=await app.state.problems.import_directory(root/name)
|
||||
request=CreateSubmission(problem_version_id=version,language='c17',
|
||||
idempotency_key=uuid4(),source='int main(void){return 0;}')
|
||||
created=await app.state.submissions.create(user,request)
|
||||
# Controlled, unmeasured historical database fixtures prove preservation of
|
||||
# reports and their digests. They never claim execution or trusted performance.
|
||||
async with app.state.sessions.begin() as session:
|
||||
projected=await app.state.problems.for_submission(session,version)
|
||||
source=ArtifactReference(digest=hashlib.sha256(request.source.encode()).hexdigest(),
|
||||
bytes=len(request.source.encode()))
|
||||
manifest=projected.manifest(created.evaluation_id,created.submission_id,'c17',source)
|
||||
md=hashlib.sha256(manifest_bytes(manifest)).hexdigest()
|
||||
aid,wid=uuid4(),uuid4()
|
||||
template=('contracts' if name=='v1' else 'contracts-v2')
|
||||
file=('judge-result.valid.json' if name=='v1' else 'judge-result.valid-c17.json')
|
||||
report=json.loads((Path('/kf/tests/fixtures')/template/file).read_text())
|
||||
report['binding']={'evaluation_id':str(created.evaluation_id),'attempt_id':str(aid),
|
||||
'worker_id':str(wid),'fencing_version':1}
|
||||
report['manifest_digest']=md
|
||||
report['cases']=[{'case_id':str(case.case_id),'verdict':'accepted',
|
||||
'cpu_time_us':100,'memory_peak_bytes':262144}
|
||||
for case in manifest.cases]
|
||||
report['run_cpu_time_us']=100*len(manifest.cases)
|
||||
report['run_memory_peak_bytes']=262144
|
||||
digest=hashlib.sha256(json.dumps(report,sort_keys=True,separators=(',',':'),
|
||||
ensure_ascii=False).encode()).hexdigest()
|
||||
await session.execute(text('SET LOCAL ROLE kf_oj_coordinator'))
|
||||
session.add(JudgeAttempt(id=aid,evaluation_id=created.evaluation_id,worker_id=wid,
|
||||
number=1,fencing_version=1,lease_token_hash=hashlib.sha256(
|
||||
secrets.token_hex(32).encode()).hexdigest(),
|
||||
expires_at=datetime.now(timezone.utc)+timedelta(minutes=10),
|
||||
manifest=manifest.model_dump(mode='json'),manifest_digest=md,
|
||||
report=report,report_digest=digest,cleanup_complete=True))
|
||||
await session.flush()
|
||||
await session.execute(update(Evaluation).where(Evaluation.id==created.evaluation_id)
|
||||
.values(current_attempt_id=aid,state='finished',attempts_count=1,
|
||||
fencing_version=1,verdict='accepted',final_report=report,
|
||||
final_report_digest=digest))
|
||||
asyncio.run(main())
|
||||
"""
|
||||
asyncio.run(manage(f'CREATE DATABASE "{url.database}"'))
|
||||
try:
|
||||
old_run(["-m", "alembic", "upgrade", "head"])
|
||||
old_run(["-c", seed])
|
||||
|
||||
async def snapshot(columns=None):
|
||||
connection = await connect(rendered)
|
||||
try:
|
||||
if columns is None:
|
||||
rows = await connection.fetch("""
|
||||
SELECT table_name,column_name FROM information_schema.columns
|
||||
WHERE table_schema='public' AND table_name<>'alembic_version'
|
||||
ORDER BY table_name,ordinal_position
|
||||
""")
|
||||
columns = {}
|
||||
for row in rows:
|
||||
columns.setdefault(row["table_name"], []).append(row["column_name"])
|
||||
data = {}
|
||||
for table, names in columns.items():
|
||||
query = (
|
||||
"SELECT " + ",".join('"' + n + '"' for n in names) + ' FROM "' + table + '"'
|
||||
)
|
||||
records = sorted(
|
||||
json.dumps(dict(r), sort_keys=True, default=str)
|
||||
for r in await connection.fetch(query)
|
||||
)
|
||||
data[table] = {
|
||||
"rows": len(records),
|
||||
"sha256": hashlib.sha256("\n".join(records).encode()).hexdigest(),
|
||||
}
|
||||
return columns, data
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
columns, before = asyncio.run(snapshot())
|
||||
assert before["accounts"]["rows"] == 1 and before["problem_versions"]["rows"] == 2
|
||||
assert before["submissions"]["rows"] == 2
|
||||
assert before["judge_attempts"]["rows"] == 2
|
||||
migrate(rendered, "upgrade", "head")
|
||||
_, after = asyncio.run(snapshot(columns))
|
||||
assert after == before
|
||||
migrate(rendered, "check")
|
||||
|
||||
async def defaults():
|
||||
connection = await connect(rendered)
|
||||
try:
|
||||
account = await connection.fetchrow(
|
||||
"SELECT is_administrator,authorization_version FROM accounts"
|
||||
)
|
||||
assert not account["is_administrator"] and account["authorization_version"] == 1
|
||||
assert await connection.fetchval("SELECT count(*) FROM evaluation_performance") == 0
|
||||
assert (
|
||||
await connection.fetchval("SELECT count(*) FROM problem_maintenance_events")
|
||||
== 0
|
||||
)
|
||||
for role in ("kf_oj_identity_operator", "kf_oj_problem_import"):
|
||||
row = await connection.fetchrow("SELECT * FROM pg_roles WHERE rolname=$1", role)
|
||||
assert not any(
|
||||
row[k]
|
||||
for k in (
|
||||
"rolcanlogin",
|
||||
"rolsuper",
|
||||
"rolcreatedb",
|
||||
"rolcreaterole",
|
||||
"rolreplication",
|
||||
"rolbypassrls",
|
||||
)
|
||||
)
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
asyncio.run(defaults())
|
||||
proof(
|
||||
"baseline_upgrade",
|
||||
{
|
||||
"baseline": "5df3c370aab0b12a1c3f8670a0c066f8b77a4078",
|
||||
"retained_tables": before,
|
||||
"old_v1_v2_manifests_and_source_bytes": "exact",
|
||||
"historical_report_retention_fixture_unmeasured": True,
|
||||
"fixture_reports_not_student_execution_or_performance_acceptance": True,
|
||||
"fresh_current_database": postgres_url.split("/")[-1].split("?")[0],
|
||||
},
|
||||
)
|
||||
finally:
|
||||
asyncio.run(manage(f'DROP DATABASE "{url.database}"'))
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Actual temporary PG migration and authority storage; API/CLI acceptance is separate."""
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
from datetime import timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
import pytest
|
||||
from conftest import connect, migrate
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from test_database_foundation import fixture_rows
|
||||
|
||||
from kf_oj.modules.identity.public import AuthenticationFailed, Principal
|
||||
from kf_oj.modules.identity.service import IdentityService, token_digest
|
||||
|
||||
|
||||
class RollBackAuthorityChange(Exception):
|
||||
"""Abort the test transaction before any authority change commits."""
|
||||
|
||||
|
||||
async def submission_snapshot(connection):
|
||||
# Synthetic queued metadata only; this does not claim real program execution.
|
||||
return {
|
||||
table: list(
|
||||
await connection.fetch(
|
||||
f"SELECT to_jsonb(t)::text FROM {table} t ORDER BY to_jsonb(t)::text"
|
||||
)
|
||||
)
|
||||
for table in (
|
||||
"artifacts",
|
||||
"problems",
|
||||
"problem_versions",
|
||||
"problem_cases",
|
||||
"submissions",
|
||||
"evaluations",
|
||||
"outbox",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
async def account_and_session(connection):
|
||||
account = uuid4()
|
||||
token, csrf = uuid4().hex * 2, uuid4().hex * 2
|
||||
await connection.execute(
|
||||
"INSERT INTO accounts(id,username,password_hash) VALUES($1,'storage_user',$2)",
|
||||
account,
|
||||
"synthetic-irreversible-placeholder-hash-" * 2,
|
||||
)
|
||||
now = await connection.fetchval("SELECT clock_timestamp()")
|
||||
await connection.execute(
|
||||
"INSERT INTO sessions(token_hash,csrf_hash,account_id,expires_at) VALUES($1,$2,$3,$4)",
|
||||
token_digest(token),
|
||||
token_digest(csrf),
|
||||
account,
|
||||
now + timedelta(hours=1),
|
||||
)
|
||||
return account, token, csrf
|
||||
|
||||
|
||||
def test_upgrade_preserves_ordinary_account_and_current_session(postgres_url):
|
||||
migrate(postgres_url, "downgrade", "0004_performance_watermark")
|
||||
|
||||
async def seed():
|
||||
connection = await connect(postgres_url)
|
||||
try:
|
||||
account, token, csrf = await account_and_session(connection)
|
||||
await fixture_rows(connection)
|
||||
return account, token, csrf, await submission_snapshot(connection)
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
account, token, csrf, before = asyncio.run(seed())
|
||||
migrate(postgres_url, "upgrade", "head")
|
||||
migrate(postgres_url, "check")
|
||||
|
||||
async def inspect():
|
||||
connection = await connect(postgres_url)
|
||||
engine = create_async_engine(postgres_url)
|
||||
try:
|
||||
row = await connection.fetchrow("SELECT * FROM accounts WHERE id=$1", account)
|
||||
assert row["enabled"] and not row["is_administrator"]
|
||||
assert row["authorization_version"] == 1
|
||||
assert await submission_snapshot(connection) == before
|
||||
service = IdentityService(async_sessionmaker(engine, expire_on_commit=False))
|
||||
assert await service.authenticate(token, csrf_token=csrf) == Principal(
|
||||
account, "storage_user"
|
||||
)
|
||||
finally:
|
||||
await connection.close()
|
||||
await engine.dispose()
|
||||
|
||||
asyncio.run(inspect())
|
||||
|
||||
|
||||
def test_authority_columns_and_import_role_are_not_runtime_write_permissions(postgres_url):
|
||||
async def inspect():
|
||||
connection = await connect(postgres_url)
|
||||
try:
|
||||
for role in ("kf_oj_platform", "kf_oj_coordinator", "kf_oj_problem_import"):
|
||||
for column in ("is_administrator", "authorization_version"):
|
||||
assert not await connection.fetchval(
|
||||
"SELECT has_column_privilege($1,'accounts',$2,'UPDATE')", role, column
|
||||
)
|
||||
assert not await connection.fetchval(
|
||||
"SELECT has_column_privilege($1,'accounts',$2,'INSERT')", role, column
|
||||
)
|
||||
for table in (
|
||||
"evaluations",
|
||||
"judge_attempts",
|
||||
"case_results",
|
||||
"evaluation_performance",
|
||||
):
|
||||
for privilege in ("SELECT", "INSERT", "UPDATE", "DELETE"):
|
||||
assert not await connection.fetchval(
|
||||
"SELECT has_table_privilege('kf_oj_problem_import',$1,$2)", table, privilege
|
||||
)
|
||||
assert await connection.fetchval(
|
||||
"SELECT has_column_privilege('kf_oj_platform','accounts','enabled','UPDATE')"
|
||||
)
|
||||
assert not await connection.fetchval(
|
||||
"SELECT has_column_privilege('kf_oj_identity_operator',"
|
||||
"'accounts','is_administrator','UPDATE')"
|
||||
)
|
||||
assert not await connection.fetchval(
|
||||
"SELECT has_column_privilege('kf_oj_identity_operator',"
|
||||
"'accounts','authorization_version','UPDATE')"
|
||||
)
|
||||
assert not await connection.fetchval(
|
||||
"SELECT pg_has_role('kf_oj_platform','kf_oj_identity_operator','USAGE')"
|
||||
)
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
asyncio.run(inspect())
|
||||
|
||||
|
||||
def test_authority_change_derives_version_and_revokes_old_session_atomically(postgres_url):
|
||||
async def inspect():
|
||||
connection = await connect(postgres_url)
|
||||
engine = create_async_engine(postgres_url)
|
||||
try:
|
||||
account, token, csrf = await account_and_session(connection)
|
||||
service = IdentityService(async_sessionmaker(engine, expire_on_commit=False))
|
||||
await service.authenticate(token, csrf_token=csrf)
|
||||
async with connection.transaction():
|
||||
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||
await connection.execute(
|
||||
"SELECT * FROM public.kf_manage_account_authority('storage_user','bootstrap')"
|
||||
)
|
||||
assert (
|
||||
await connection.fetchval(
|
||||
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||
)
|
||||
== 2
|
||||
)
|
||||
with pytest.raises(AuthenticationFailed):
|
||||
await service.authenticate(token, csrf_token=csrf)
|
||||
async with connection.transaction():
|
||||
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||
await connection.execute(
|
||||
"SELECT * FROM public.kf_manage_account_authority('storage_user','bootstrap')"
|
||||
)
|
||||
assert (
|
||||
await connection.fetchval(
|
||||
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||
)
|
||||
== 2
|
||||
)
|
||||
fresh_token, fresh_csrf = uuid4().hex * 2, uuid4().hex * 2
|
||||
await connection.execute(
|
||||
"INSERT INTO sessions(token_hash,csrf_hash,account_id,expires_at) "
|
||||
"VALUES($1,$2,$3,clock_timestamp()+interval '1 hour')",
|
||||
token_digest(fresh_token),
|
||||
token_digest(fresh_csrf),
|
||||
account,
|
||||
)
|
||||
await connection.execute(
|
||||
"INSERT INTO accounts(id,username,password_hash) "
|
||||
"VALUES(gen_random_uuid(),'storage_recovery',repeat('synthetic-only-',5))"
|
||||
)
|
||||
async with connection.transaction():
|
||||
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||
await connection.execute(
|
||||
"SELECT * FROM public.kf_manage_account_authority('storage_recovery','grant')"
|
||||
)
|
||||
with pytest.raises(RollBackAuthorityChange):
|
||||
async with connection.transaction():
|
||||
await connection.execute(
|
||||
"SELECT * FROM public.kf_manage_account_authority('storage_user','revoke')"
|
||||
)
|
||||
assert (
|
||||
await connection.fetchval(
|
||||
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||
)
|
||||
== 3
|
||||
)
|
||||
raise RollBackAuthorityChange
|
||||
assert await connection.fetchval(
|
||||
"SELECT is_administrator FROM accounts WHERE id=$1", account
|
||||
)
|
||||
assert (
|
||||
await connection.fetchval(
|
||||
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||
)
|
||||
== 2
|
||||
)
|
||||
await service.authenticate(fresh_token, csrf_token=fresh_csrf)
|
||||
async with connection.transaction():
|
||||
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||
await connection.execute(
|
||||
"SELECT * FROM public.kf_manage_account_authority('storage_user','revoke')"
|
||||
)
|
||||
assert (
|
||||
await connection.fetchval(
|
||||
"SELECT authorization_version FROM accounts WHERE id=$1", account
|
||||
)
|
||||
== 3
|
||||
)
|
||||
with pytest.raises(AuthenticationFailed):
|
||||
await service.authenticate(fresh_token, csrf_token=fresh_csrf)
|
||||
finally:
|
||||
await connection.close()
|
||||
await engine.dispose()
|
||||
|
||||
asyncio.run(inspect())
|
||||
|
||||
|
||||
def test_authority_history_cannot_be_erased_by_downgrade(postgres_url):
|
||||
async def change():
|
||||
connection = await connect(postgres_url)
|
||||
try:
|
||||
account, _, _ = await account_and_session(connection)
|
||||
await connection.execute("UPDATE accounts SET enabled=false WHERE id=$1", account)
|
||||
with pytest.raises(asyncpg.CheckViolationError):
|
||||
await connection.execute(
|
||||
"UPDATE accounts SET authorization_version=1 WHERE id=$1", account
|
||||
)
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
asyncio.run(change())
|
||||
with pytest.raises(subprocess.CalledProcessError):
|
||||
migrate(postgres_url, "downgrade", "0004_performance_watermark")
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Trusted adapter checks; real cold PG/role/permission tests remain VM-only."""
|
||||
|
||||
import hashlib
|
||||
|
||||
from kf_oj.contracts.judging import ArtifactReference
|
||||
from kf_oj.infra.artifacts.backup import verify_existing
|
||||
|
||||
|
||||
def test_backup_verifies_binary_and_empty_objects_without_creating_storage_entries(tmp_path):
|
||||
tmp_path.chmod(0o700)
|
||||
values = [b"", b"\0\xff\n" * 50000]
|
||||
expected = {}
|
||||
for value in values:
|
||||
digest = hashlib.sha256(value).hexdigest()
|
||||
path = tmp_path / digest
|
||||
path.write_bytes(value)
|
||||
path.chmod(0o600)
|
||||
expected[digest] = value
|
||||
before = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in tmp_path.iterdir()}
|
||||
for digest, value in expected.items():
|
||||
verify_existing(tmp_path, ArtifactReference(digest=digest, bytes=len(value)))
|
||||
after = {p.name: (p.read_bytes(), p.stat().st_mtime_ns) for p in tmp_path.iterdir()}
|
||||
assert after == before
|
||||
assert not (tmp_path / ".lock").exists()
|
||||
@@ -0,0 +1,219 @@
|
||||
"""Real local authority/CLI transactions; run privilege faults in the independent VM."""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
import pytest
|
||||
from conftest import connect
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
from kf_oj.modules.identity.authority import manage_authority
|
||||
|
||||
|
||||
async def seed(connection, *usernames):
|
||||
for name in usernames:
|
||||
await connection.execute(
|
||||
"INSERT INTO accounts(id,username,password_hash) VALUES($1,$2,repeat('synthetic-',6))",
|
||||
uuid4(),
|
||||
name,
|
||||
)
|
||||
|
||||
|
||||
async def authority(connection, username, operation):
|
||||
async with connection.transaction():
|
||||
await connection.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||
return await connection.fetchrow(
|
||||
"SELECT * FROM public.kf_manage_account_authority($1,$2)", username, operation
|
||||
)
|
||||
|
||||
|
||||
def test_explicit_bootstrap_repeated_grants_recovery_and_last_administrator(postgres_url):
|
||||
async def run():
|
||||
c = await connect(postgres_url)
|
||||
try:
|
||||
await seed(c, "first_admin", "second_admin")
|
||||
assert await c.fetchval("SELECT count(*) FROM accounts WHERE is_administrator") == 0
|
||||
with pytest.raises(asyncpg.CheckViolationError):
|
||||
await authority(c, "first_admin", "grant")
|
||||
first = await authority(c, "first_admin", "bootstrap")
|
||||
assert first["changed"] and first["authorization_version"] == 2
|
||||
assert not (await authority(c, "first_admin", "bootstrap"))["changed"]
|
||||
assert not (await authority(c, "first_admin", "grant"))["changed"]
|
||||
with pytest.raises(asyncpg.CheckViolationError):
|
||||
await authority(c, "second_admin", "bootstrap")
|
||||
for operation in ("disable", "revoke"):
|
||||
with pytest.raises(asyncpg.CheckViolationError):
|
||||
await authority(c, "first_admin", operation)
|
||||
await authority(c, "second_admin", "grant")
|
||||
disabled = await authority(c, "first_admin", "disable")
|
||||
assert not disabled["enabled"] and disabled["is_administrator"]
|
||||
assert disabled["authorization_version"] == 3
|
||||
assert not (await authority(c, "first_admin", "disable"))["changed"]
|
||||
restored = await authority(c, "first_admin", "enable")
|
||||
assert restored["enabled"] and restored["authorization_version"] == 4
|
||||
revoked = await authority(c, "first_admin", "revoke")
|
||||
assert not revoked["is_administrator"] and revoked["authorization_version"] == 5
|
||||
assert not (await authority(c, "first_admin", "revoke"))["changed"]
|
||||
await authority(c, "first_admin", "grant")
|
||||
with pytest.raises(asyncpg.NoDataFoundError):
|
||||
await authority(c, "absent_user", "grant")
|
||||
for role in ("kf_oj_platform", "kf_oj_coordinator", "kf_oj_problem_import"):
|
||||
assert not await c.fetchval(
|
||||
"SELECT has_function_privilege($1,"
|
||||
"'kf_manage_account_authority(text,text)','EXECUTE')",
|
||||
role,
|
||||
)
|
||||
for column in ("password_hash", "is_administrator", "enabled"):
|
||||
assert not await c.fetchval(
|
||||
"SELECT has_column_privilege('kf_oj_identity_operator','accounts',$1,'SELECT')",
|
||||
column,
|
||||
)
|
||||
async with c.transaction():
|
||||
await c.execute("SET LOCAL ROLE kf_oj_identity_operator")
|
||||
for sql in (
|
||||
"SELECT password_hash FROM accounts",
|
||||
"UPDATE accounts SET is_administrator=false",
|
||||
"SELECT token_hash FROM sessions",
|
||||
):
|
||||
with pytest.raises(asyncpg.InsufficientPrivilegeError):
|
||||
async with c.transaction():
|
||||
await c.execute(sql)
|
||||
finally:
|
||||
await c.close()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("isolation", ("read_committed", "repeatable_read", "serializable"))
|
||||
def test_concurrent_different_account_revocations_preserve_one_admin(postgres_url, isolation):
|
||||
async def serialized_run():
|
||||
c, other = await connect(postgres_url), await connect(postgres_url)
|
||||
first_tx = second_tx = pending = None
|
||||
try:
|
||||
await seed(c, "first_admin", "second_admin")
|
||||
await authority(c, "first_admin", "bootstrap")
|
||||
await authority(c, "second_admin", "grant")
|
||||
first_tx, second_tx = (
|
||||
c.transaction(isolation=isolation),
|
||||
other.transaction(isolation=isolation),
|
||||
)
|
||||
await first_tx.start()
|
||||
await second_tx.start()
|
||||
assert (
|
||||
await other.fetchval(
|
||||
"SELECT count(*) FROM accounts WHERE is_administrator AND enabled"
|
||||
)
|
||||
== 2
|
||||
)
|
||||
await authority(c, "first_admin", "revoke")
|
||||
pending = asyncio.create_task(authority(other, "second_admin", "revoke"))
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
if await c.fetchval(
|
||||
"SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND NOT granted)",
|
||||
other.get_server_pid(),
|
||||
):
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
else:
|
||||
raise AssertionError("second authority operation never blocked on PG lock")
|
||||
assert not pending.done()
|
||||
await first_tx.commit()
|
||||
first_tx = None
|
||||
with pytest.raises((asyncpg.CheckViolationError, asyncpg.SerializationError)):
|
||||
await asyncio.wait_for(pending, 5)
|
||||
await second_tx.rollback()
|
||||
second_tx = None
|
||||
assert (
|
||||
await c.fetchval("SELECT count(*) FROM accounts WHERE is_administrator AND enabled")
|
||||
== 1
|
||||
)
|
||||
assert await c.fetchval(
|
||||
"SELECT is_administrator FROM accounts WHERE username='second_admin'"
|
||||
)
|
||||
finally:
|
||||
if pending is not None and not pending.done():
|
||||
pending.cancel()
|
||||
await asyncio.gather(pending, return_exceptions=True)
|
||||
if first_tx is not None:
|
||||
await first_tx.rollback()
|
||||
if second_tx is not None:
|
||||
await second_tx.rollback()
|
||||
await c.close()
|
||||
await other.close()
|
||||
|
||||
asyncio.run(serialized_run())
|
||||
|
||||
|
||||
def test_real_cli_uses_dedicated_login_and_never_prints_database_secret(postgres_url):
|
||||
role = "authority_test_" + uuid4().hex
|
||||
|
||||
async def setup():
|
||||
c = await connect(postgres_url)
|
||||
try:
|
||||
await seed(c, "cli_admin", "cli_backup")
|
||||
await c.execute(
|
||||
f"CREATE ROLE {role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE "
|
||||
"NOREPLICATION NOBYPASSRLS"
|
||||
)
|
||||
await c.execute(f"GRANT kf_oj_identity_operator TO {role}")
|
||||
finally:
|
||||
await c.close()
|
||||
|
||||
asyncio.run(setup())
|
||||
url = make_url(postgres_url).set(username=role).render_as_string(hide_password=False)
|
||||
env = os.environ | {
|
||||
"KF_OJ_DATABASE_URL": postgres_url,
|
||||
"KF_OJ_IDENTITY_OPERATOR_DATABASE_URL": url,
|
||||
}
|
||||
cwd = Path(__file__).resolve().parents[1]
|
||||
|
||||
def invoke(command, username, expected=0):
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "kf_oj.modules.identity.cli", command, username],
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == expected, result.stderr
|
||||
assert url not in result.stdout + result.stderr
|
||||
assert "Traceback" not in result.stderr
|
||||
return result.stdout
|
||||
|
||||
try:
|
||||
assert "changed=True" in invoke("bootstrap-admin", "cli_admin")
|
||||
assert "changed=False" in invoke("bootstrap-admin", "cli_admin")
|
||||
invoke("revoke-admin", "cli_admin", 1)
|
||||
invoke("grant-admin", "cli_backup")
|
||||
invoke("disable", "cli_admin")
|
||||
invoke("enable", "cli_admin")
|
||||
invoke("revoke-admin", "cli_admin")
|
||||
invoke("grant-admin", "cli_admin")
|
||||
finally:
|
||||
|
||||
async def cleanup():
|
||||
c = await connect(postgres_url)
|
||||
try:
|
||||
await c.execute(f"REVOKE kf_oj_identity_operator FROM {role}")
|
||||
await c.execute(f"DROP ROLE {role}")
|
||||
finally:
|
||||
await c.close()
|
||||
|
||||
asyncio.run(cleanup())
|
||||
|
||||
|
||||
def test_bad_cli_operation_rejected_without_connecting():
|
||||
async def run():
|
||||
for username, operation in (("invalid name", "grant"), ("valid_name", "unknown")):
|
||||
with pytest.raises(ValueError):
|
||||
await manage_authority(None, username, operation)
|
||||
|
||||
asyncio.run(run())
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Contract/admission-route tests; Root file/atomicity and drain acceptance are VM-only."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
from kf_oj.api.availability import AvailabilityMiddleware, needs_open
|
||||
from kf_oj.contracts.availability import AvailabilityRecord
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ["open", "draining", "quiescent"])
|
||||
def test_availability_is_exactly_bound_to_a_known_version_and_state(state):
|
||||
path = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "tests/fixtures/contracts/delivery-availability.valid.json"
|
||||
)
|
||||
value = json.loads(path.read_text())
|
||||
value["state"] = state
|
||||
assert AvailabilityRecord.model_validate(value).state == state
|
||||
schema = path.parents[3] / "contracts/schemas/delivery-availability.v1.schema.json"
|
||||
Draft202012Validator(json.loads(schema.read_text())).validate(value)
|
||||
for key, replacement in (
|
||||
("schema_version", True),
|
||||
("release_id", "../release"),
|
||||
("state", "always_allow"),
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
AvailabilityRecord.model_validate({**value, key: replacement})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method,path,blocked",
|
||||
[
|
||||
("POST", "/api/v1/submissions", True),
|
||||
("POST", "/api/v1/admin/problem-imports", True),
|
||||
("POST", "/api/v1/admin/problems/publish", True),
|
||||
("POST", "/api/v1/auth/login", True),
|
||||
("GET", "/api/v1/internal/evaluations/artifacts", False),
|
||||
("POST", "/api/v1/auth/logout", False),
|
||||
("POST", "/api/v1/submissions/11111111-1111-4111-8111-111111111111/cancel", False),
|
||||
(
|
||||
"PUT",
|
||||
"/api/v1/admin/problem-imports/11111111-1111-4111-8111-111111111111/archive",
|
||||
False,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_drain_blocks_new_mutations_without_cutting_accepted_reads_and_cancel(
|
||||
method, path, blocked
|
||||
):
|
||||
assert needs_open(method, path) is blocked
|
||||
|
||||
|
||||
def test_missing_admission_record_rejects_before_body_or_domain_execution(monkeypatch):
|
||||
# Transport policy only: this stub does not verify Root ownership or Linux file safety.
|
||||
calls = []
|
||||
messages = []
|
||||
|
||||
def missing(*_):
|
||||
raise FileNotFoundError("record missing")
|
||||
|
||||
monkeypatch.setattr("kf_oj.api.availability.current", missing)
|
||||
|
||||
async def receive():
|
||||
calls.append("body")
|
||||
return {"type": "http.request", "body": b""}
|
||||
|
||||
async def domain(*_):
|
||||
calls.append("domain")
|
||||
|
||||
async def send(message):
|
||||
messages.append(message)
|
||||
|
||||
settings = SimpleNamespace(
|
||||
environment="production", delivery_release_id=None, delivery_configuration_id=None
|
||||
)
|
||||
asyncio.run(
|
||||
AvailabilityMiddleware(domain, settings=settings)(
|
||||
{"type": "http", "method": "POST", "path": "/api/v1/submissions"}, receive, send
|
||||
)
|
||||
)
|
||||
assert calls == []
|
||||
assert messages[0]["status"] == 503
|
||||
assert (b"cache-control", b"no-store") in messages[0]["headers"]
|
||||
@@ -27,8 +27,16 @@ ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def fixture(postgres_url, tmp_path, monkeypatch):
|
||||
_, policy = import_fixture(tmp_path / "import")
|
||||
async def fixture(
|
||||
postgres_url,
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
*,
|
||||
import_builder=import_fixture,
|
||||
node_policy=None,
|
||||
measurement_epochs=None,
|
||||
):
|
||||
_, policy = import_builder(tmp_path / "import")
|
||||
engine = create_async_engine(postgres_url)
|
||||
platform_role(engine)
|
||||
monkeypatch.setattr("kf_oj.api.app.build_engine", lambda _: engine)
|
||||
@@ -57,6 +65,9 @@ async def fixture(postgres_url, tmp_path, monkeypatch):
|
||||
reserved.bind(("127.0.0.1", 0))
|
||||
port = reserved.getsockname()[1]
|
||||
workers = [dict(id=str(uuid4()), token=secrets.token_hex(32)) for _ in range(2)]
|
||||
if measurement_epochs is not None:
|
||||
for worker, epoch in zip(workers, measurement_epochs, strict=True):
|
||||
worker["measurement_epoch"] = epoch
|
||||
platform = secrets.token_hex(32)
|
||||
config = tmp_path / "coordinator.json"
|
||||
config.write_text(
|
||||
@@ -66,7 +77,7 @@ async def fixture(postgres_url, tmp_path, monkeypatch):
|
||||
database_url=coord_url,
|
||||
workers=workers,
|
||||
platform_token=platform,
|
||||
admission=policy.model_dump(mode="json"),
|
||||
admission=(node_policy or policy).model_dump(mode="json"),
|
||||
lease_seconds=5,
|
||||
recovery_seconds=1,
|
||||
)
|
||||
@@ -134,19 +145,23 @@ async def fixture(postgres_url, tmp_path, monkeypatch):
|
||||
"cpu.max": "50000 100000",
|
||||
}
|
||||
|
||||
async def create():
|
||||
async def create(language="c17"):
|
||||
return await app.state.submissions.create(
|
||||
owner,
|
||||
CreateSubmission(
|
||||
problem_version_id=version,
|
||||
language="c17",
|
||||
language=language,
|
||||
idempotency_key=uuid4(),
|
||||
source="synthetic report fixture; never compiled on host",
|
||||
),
|
||||
)
|
||||
|
||||
client.fixture_app = app
|
||||
client.fixture_account = owner
|
||||
client.fixture_version = version
|
||||
client.coordinator_unit = unit
|
||||
client.coordinator_role = role
|
||||
client.coordinator_config = config
|
||||
yield client, c, workers, platform, create
|
||||
finally:
|
||||
subprocess.run(["systemctl", "--user", "stop", unit], check=True, capture_output=True)
|
||||
@@ -434,6 +449,7 @@ def test_actual_expiry_stopping_quarantine_retry_and_old_attempt(
|
||||
break
|
||||
await asyncio.sleep(0.05)
|
||||
assert pending[0]["binding"] == g["binding"]
|
||||
assert pending == [dict(binding=g["binding"], case_count=2)]
|
||||
stopped = dict(
|
||||
schema_version=1, kind="judge.stopped", binding=g["binding"], cleanup_complete=False
|
||||
)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import json
|
||||
import subprocess
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
@@ -12,9 +13,15 @@ def test_migration_roundtrip_and_role_safety(postgres_url):
|
||||
c = await connect(postgres_url)
|
||||
try:
|
||||
tables = set(await c.fetch("SELECT tablename FROM pg_tables WHERE schemaname='public'"))
|
||||
assert len(tables) == 13 # Twelve business tables and Alembic history.
|
||||
assert len(tables) == 21 # Twenty domain/control tables and Alembic history.
|
||||
roles = await c.fetch("SELECT * FROM pg_roles WHERE rolname LIKE 'kf_oj_%'")
|
||||
assert {row["rolname"] for row in roles} == {"kf_oj_platform", "kf_oj_coordinator"}
|
||||
assert {row["rolname"] for row in roles} == {
|
||||
"kf_oj_platform",
|
||||
"kf_oj_coordinator",
|
||||
"kf_oj_identity_operator",
|
||||
"kf_oj_problem_import",
|
||||
"kf_oj_identity_authority_owner",
|
||||
}
|
||||
assert all(
|
||||
not any(
|
||||
r[k]
|
||||
@@ -38,7 +45,14 @@ def test_migration_roundtrip_and_role_safety(postgres_url):
|
||||
asyncio.run(inspect())
|
||||
migrate(postgres_url, "downgrade", "base")
|
||||
migrate(postgres_url, "upgrade", "head")
|
||||
migrate(postgres_url, "check")
|
||||
try:
|
||||
migrate(postgres_url, "check")
|
||||
except subprocess.CalledProcessError as error:
|
||||
raise AssertionError(
|
||||
"migration model consistency failed: "
|
||||
+ (error.stdout or b"").decode(errors="replace")[-8192:]
|
||||
+ (error.stderr or b"").decode(errors="replace")[-8192:]
|
||||
) from None
|
||||
asyncio.run(inspect())
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,349 @@
|
||||
"""Trusted byte fixtures only; no privileged backup, recovery or RTO acceptance."""
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
backup_manifest = importlib.import_module("scripts.release.backup_manifest")
|
||||
backup_objects = importlib.import_module("scripts.release.backup_objects")
|
||||
policy = importlib.import_module("scripts.release.policy")
|
||||
|
||||
|
||||
def empty_drain_projection():
|
||||
return {
|
||||
"execution": {
|
||||
"database_clock_unix_ms": 1791830000000,
|
||||
**{
|
||||
name: {"count_capped": 0, "truncated": False}
|
||||
for name in ("queued", "in_flight", "cleanup_unconfirmed")
|
||||
},
|
||||
},
|
||||
"outbox": {
|
||||
"schema_version": 1,
|
||||
"pending_capped": 0,
|
||||
"locked_capped": 0,
|
||||
"truncated": False,
|
||||
},
|
||||
"imports": {
|
||||
"schema_version": 1,
|
||||
"worker_ready": True,
|
||||
"reserved_bytes": 0,
|
||||
"uploaded_capped": 0,
|
||||
"validating_capped": 0,
|
||||
"truncated": False,
|
||||
},
|
||||
"broker": {
|
||||
"schema_version": 1,
|
||||
"metadata_verified": True,
|
||||
"messages_ready": 0,
|
||||
"messages_unacknowledged": 0,
|
||||
"message_bytes": 0,
|
||||
"consumers": 1,
|
||||
},
|
||||
"factory": {
|
||||
"schema_version": 1,
|
||||
"configuration_sha256": "b" * 64,
|
||||
"supervisor_authorized": True,
|
||||
"execution_gate": "required_per_execution",
|
||||
"isolation_verified": False,
|
||||
"max_concurrent_runs": 1,
|
||||
"active_helpers": 0,
|
||||
"cleanup_complete": True,
|
||||
"remaining_requests": 4096,
|
||||
"remaining_lifetime_seconds": 3600,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def snapshot():
|
||||
data = b"trusted repeated bytes\n"
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
files = [
|
||||
{
|
||||
"path": path,
|
||||
"uid": uid,
|
||||
"gid": uid,
|
||||
"mode": mode,
|
||||
"bytes": len(data),
|
||||
"sha256": digest,
|
||||
}
|
||||
for path, uid, mode in [
|
||||
("postgres/PG_VERSION", 1203, 0o600),
|
||||
("postgres/global/pg_control", 1203, 0o600),
|
||||
("releases/first", 0, 0o444),
|
||||
("releases/second", 0, 0o555),
|
||||
]
|
||||
]
|
||||
boot = "12345678-1234-1234-1234-123456789abc"
|
||||
selected = {
|
||||
"schema_version": 1,
|
||||
"release_id": "sd-" + "c" * 12,
|
||||
"manifest_sha256": "b" * 64,
|
||||
"configuration_id": "cfg-" + "d" * 24,
|
||||
"receipt_sha256": "e" * 64,
|
||||
}
|
||||
return {
|
||||
"schema_version": 3,
|
||||
"format": backup_manifest.FORMAT,
|
||||
"backup_id": "bk-" + "2" * 32,
|
||||
"source_sha": "c" * 40,
|
||||
"manifest_sha256": selected["manifest_sha256"],
|
||||
"configuration_id": selected["configuration_id"],
|
||||
"created_unix_seconds": 1793070000,
|
||||
"expires_unix_seconds": 1793070000 + 7 * 86400,
|
||||
"selected": selected,
|
||||
"consistent_cut": {
|
||||
"schema_version": 1,
|
||||
"source_sha": "c" * 40,
|
||||
"configuration_id": selected["configuration_id"],
|
||||
"action": "stop",
|
||||
"phase": "complete",
|
||||
"success": True,
|
||||
"events": [],
|
||||
"root_cleanup_confirmed": True,
|
||||
"boot_id": boot,
|
||||
"drain_projection": empty_drain_projection(),
|
||||
},
|
||||
"artifact_references": {
|
||||
"schema_version": 1,
|
||||
"kind": "kf.private.artifact-backup-inventory",
|
||||
"read_only": True,
|
||||
"quiescence_verified": False,
|
||||
"boot_id": boot,
|
||||
"storage_identity": {"device": 1, "inode": 2, "uid": 1200},
|
||||
"objects": [],
|
||||
},
|
||||
"broker_cookie_sha256": "b" * 64,
|
||||
"database_bootstrap_sha256": "e" * 64,
|
||||
"images": {
|
||||
key: {
|
||||
"toolchain_id": key,
|
||||
"image_kind": key,
|
||||
"runtime_version": "1",
|
||||
"image_revision": 1,
|
||||
"image_digest": "f" * 64,
|
||||
}
|
||||
for key in ("gnu", "python313", "java21")
|
||||
},
|
||||
"directories": [
|
||||
{"path": path, "uid": 0, "gid": 0, "mode": 0o700}
|
||||
for path in sorted(backup_manifest.DOMAINS | {"postgres/global"})
|
||||
],
|
||||
"files": files,
|
||||
"payload_bytes": len(data) * len(files),
|
||||
"restored": False,
|
||||
"execution_verified": False,
|
||||
}
|
||||
|
||||
|
||||
def object_format(value):
|
||||
return {
|
||||
**value,
|
||||
"schema_version": 4,
|
||||
"format": backup_manifest.OBJECT_FORMAT,
|
||||
"logical_bytes": value["payload_bytes"],
|
||||
"payload_bytes": value["files"][0]["bytes"],
|
||||
}
|
||||
|
||||
|
||||
def test_v3_reader_and_v4_logical_permissions_survive_content_deduplication(snapshot, tmp_path):
|
||||
old = backup_manifest.validate(snapshot)
|
||||
new = backup_manifest.validate(object_format(snapshot))
|
||||
assert len(backup_manifest.physical_files(old)) == 4
|
||||
assert len(backup_manifest.physical_files(new)) == 1
|
||||
assert backup_manifest.logical_bytes(old) == backup_manifest.logical_bytes(new)
|
||||
assert new["payload_bytes"] * 4 == new["logical_bytes"]
|
||||
assert new["files"][2]["mode"] != new["files"][3]["mode"]
|
||||
for item in new["files"]:
|
||||
assert backup_manifest.member_path(tmp_path, new, item) == (
|
||||
tmp_path / "objects" / item["sha256"]
|
||||
)
|
||||
assert backup_manifest.named_member(tmp_path, old, item["path"]) == (
|
||||
tmp_path / "payload" / item["path"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,change",
|
||||
[
|
||||
("payload_bytes", 1),
|
||||
("logical_bytes", 1),
|
||||
("schema_version", True),
|
||||
("format", backup_manifest.FORMAT),
|
||||
],
|
||||
)
|
||||
def test_v4_cannot_confuse_stored_bytes_with_restored_bytes(snapshot, field, change):
|
||||
value = object_format(snapshot)
|
||||
value[field] = change
|
||||
with pytest.raises(policy.DeliveryError):
|
||||
backup_manifest.validate(value)
|
||||
|
||||
|
||||
def test_v4_preserves_logical_versions_above_v3_limit_without_raising_physical_limit(snapshot):
|
||||
value = object_format(snapshot)
|
||||
size = backup_manifest.MAX_FILE_BYTES
|
||||
value["files"] = [
|
||||
{**value["files"][0], "path": "releases/version-" + str(index), "bytes": size}
|
||||
for index in range(32)
|
||||
] + [{**item, "bytes": size} for item in snapshot["files"][:2]]
|
||||
value["logical_bytes"] = len(value["files"]) * size
|
||||
value["payload_bytes"] = size
|
||||
backup_manifest.validate(value)
|
||||
assert value["logical_bytes"] > backup_manifest.MAX_BYTES
|
||||
old = {
|
||||
**value,
|
||||
"schema_version": 3,
|
||||
"format": backup_manifest.FORMAT,
|
||||
"payload_bytes": value["logical_bytes"],
|
||||
}
|
||||
old.pop("logical_bytes")
|
||||
with pytest.raises(policy.DeliveryError):
|
||||
backup_manifest.validate(old)
|
||||
# Distinct object digests still consume the unchanged physical budget.
|
||||
for index, item in enumerate(value["files"]):
|
||||
item["sha256"] = hashlib.sha256(str(index).encode()).hexdigest()
|
||||
value["payload_bytes"] = value["logical_bytes"]
|
||||
with pytest.raises(policy.DeliveryError):
|
||||
backup_manifest.validate(value)
|
||||
|
||||
|
||||
def test_store_reads_each_source_but_writes_one_single_link_object(tmp_path, monkeypatch):
|
||||
source, repeated, objects = tmp_path / "source", tmp_path / "repeated", tmp_path / "objects"
|
||||
objects.mkdir(mode=0o700)
|
||||
data = b"trusted source\n" * 2000
|
||||
source.write_bytes(data)
|
||||
repeated.write_bytes(data)
|
||||
repeated.chmod(0o444)
|
||||
original = backup_objects.read_stream
|
||||
observed = []
|
||||
|
||||
def read(incoming, before, deadline, outgoing=None):
|
||||
observed.append((before.st_ino, outgoing is not None))
|
||||
return original(incoming, before, deadline, outgoing)
|
||||
|
||||
monkeypatch.setattr(backup_objects, "read_stream", read)
|
||||
seen = {}
|
||||
first, added = backup_objects.store(
|
||||
source, source.stat(), objects, seen, time.monotonic() + 10, remaining_bytes=len(data)
|
||||
)
|
||||
second, repeated_bytes = backup_objects.store(
|
||||
repeated, repeated.stat(), objects, seen, time.monotonic() + 10, remaining_bytes=0
|
||||
)
|
||||
assert first == second == hashlib.sha256(data).hexdigest()
|
||||
assert (added, repeated_bytes) == (len(data), 0)
|
||||
assert observed == [
|
||||
(source.stat().st_ino, False),
|
||||
(source.stat().st_ino, True),
|
||||
(repeated.stat().st_ino, False),
|
||||
]
|
||||
assert list(objects.iterdir()) == [objects / first]
|
||||
assert (objects / first).read_bytes() == data
|
||||
assert (objects / first).stat().st_nlink == 1
|
||||
assert (objects / first).stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
def test_failed_second_source_read_retains_incomplete_private_copy(tmp_path, monkeypatch):
|
||||
source, objects = tmp_path / "source", tmp_path / "objects"
|
||||
source.write_bytes(b"trusted fixture")
|
||||
objects.mkdir(mode=0o700)
|
||||
original = backup_objects.read_stream
|
||||
|
||||
def read(incoming, before, deadline, outgoing=None):
|
||||
if outgoing is not None:
|
||||
raise OSError("fixture write failure")
|
||||
return original(incoming, before, deadline)
|
||||
|
||||
monkeypatch.setattr(backup_objects, "read_stream", read)
|
||||
seen = {}
|
||||
with pytest.raises(OSError):
|
||||
backup_objects.store(
|
||||
source, source.stat(), objects, seen, time.monotonic() + 10, remaining_bytes=100
|
||||
)
|
||||
assert seen == {}
|
||||
assert list(objects.iterdir()) == [objects / ".pending"]
|
||||
assert not (tmp_path / "manifest.json").exists()
|
||||
|
||||
|
||||
def test_restore_expands_repeated_content_into_independent_files_with_own_modes(tmp_path):
|
||||
restore = importlib.import_module("scripts.release.restore")
|
||||
content = tmp_path / "object"
|
||||
data = b"trusted restoration fixture\n"
|
||||
content.write_bytes(data)
|
||||
row = {"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}
|
||||
first, second = tmp_path / "first", tmp_path / "second"
|
||||
for path, mode in ((first, 0o600), (second, 0o444)):
|
||||
witness = restore.write_file(
|
||||
content, path, row, os.getuid(), os.getgid(), mode, time.monotonic() + 10
|
||||
)
|
||||
restore.unchanged_file(path, witness, os.getuid(), os.getgid(), mode)
|
||||
assert path.read_bytes() == data
|
||||
assert path.stat().st_mode & 0o777 == mode
|
||||
assert path.stat().st_nlink == 1
|
||||
assert len({path.stat().st_ino for path in (content, first, second)}) == 3
|
||||
|
||||
|
||||
def test_delayed_filesystem_write_error_is_not_success(tmp_path, monkeypatch):
|
||||
class SyncFailure:
|
||||
def __call__(self, descriptor):
|
||||
assert os.fstat(descriptor).st_ino == tmp_path.stat().st_ino
|
||||
return -1
|
||||
|
||||
class Library:
|
||||
syncfs = SyncFailure()
|
||||
|
||||
monkeypatch.setattr(backup_objects.ctypes, "CDLL", lambda *_, **__: Library())
|
||||
with pytest.raises(policy.DeliveryError, match="filesystem synchronization failed"):
|
||||
backup_objects.sync_filesystem(tmp_path, time.monotonic() + 10)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing", ["releases/older", "releases/older/nested"])
|
||||
def test_every_directory_ancestor_is_required_even_when_file_parent_exists(snapshot, missing):
|
||||
value = object_format(snapshot)
|
||||
value["directories"] = value["directories"] + [
|
||||
{"path": path, "uid": 0, "gid": 0, "mode": 0o700}
|
||||
for path in ("releases/older", "releases/older/nested")
|
||||
if path != missing
|
||||
]
|
||||
value["files"] = [
|
||||
{**row, "path": "releases/older/nested/file"} if row["path"] == "releases/first" else row
|
||||
for row in value["files"]
|
||||
]
|
||||
with pytest.raises(policy.DeliveryError, match="private backup parent missing"):
|
||||
backup_manifest.validate(value)
|
||||
|
||||
|
||||
def test_v4_complete_version_inventory_keeps_an_explicit_logical_entry_bound(snapshot):
|
||||
value = object_format(snapshot)
|
||||
count = backup_manifest.MAX_LOGICAL_ENTRIES - len(value["directories"])
|
||||
template = {**value["files"][0], "bytes": 1, "sha256": hashlib.sha256(b"x").hexdigest()}
|
||||
value["files"] = [{**template, "path": row["path"]} for row in value["files"][:2]] + [
|
||||
{**template, "path": "releases/item-" + str(i)} for i in range(count - 2)
|
||||
]
|
||||
value.update(logical_bytes=count, payload_bytes=1)
|
||||
backup_manifest.validate(value)
|
||||
assert len(value["files"]) + len(value["directories"]) == 98304
|
||||
assert backup_manifest.MAX_BYTES == 7 * 1024**3
|
||||
assert backup_manifest.MAX_LOGICAL_BYTES == 12 * 1024**3
|
||||
old = {**value, "schema_version": 3, "format": backup_manifest.FORMAT, "payload_bytes": count}
|
||||
old.pop("logical_bytes")
|
||||
with pytest.raises(policy.DeliveryError, match="inventory count exceeds bound"):
|
||||
backup_manifest.validate(old)
|
||||
value["files"].append({**template, "path": "releases/one-too-many"})
|
||||
value["logical_bytes"] += 1
|
||||
with pytest.raises(policy.DeliveryError, match="inventory count exceeds bound"):
|
||||
backup_manifest.validate(value)
|
||||
|
||||
|
||||
def test_private_metadata_rejects_a_file_above_the_explicit_byte_limit(tmp_path):
|
||||
path = tmp_path / "metadata.json"
|
||||
path.write_bytes(b" " * (backup_manifest.MAX_METADATA_BYTES + 1))
|
||||
with pytest.raises(policy.DeliveryError, match="metadata byte limit"):
|
||||
backup_manifest.read_metadata(path)
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Ordinary owned source/cache checks; no service, isolation or recovery claims."""
|
||||
|
||||
import importlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
bytecode = importlib.import_module("scripts.release.bytecode")
|
||||
manifest = importlib.import_module("scripts.release.manifest")
|
||||
policy = importlib.import_module("scripts.release.policy")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def compiled(tmp_path):
|
||||
source = tmp_path / "site/toy.py"
|
||||
source.parent.mkdir()
|
||||
source.write_text("VALUE = (17, 'ordinary trusted fixture')\n")
|
||||
source.chmod(0o444)
|
||||
assert bytecode.compile_payload(tmp_path, "sd-" + "b" * 12) == 1
|
||||
cache = source.parent / "__pycache__/toy.cpython-313.pyc"
|
||||
return tmp_path, source, cache
|
||||
|
||||
|
||||
def test_complete_cache_inventory_and_actual_import_without_source_compilation(compiled):
|
||||
payload, source, cache = compiled
|
||||
records = manifest.inventory(payload)
|
||||
assert {r["path"] for r in records} == {
|
||||
"site/toy.py",
|
||||
"site/__pycache__/toy.cpython-313.pyc",
|
||||
}
|
||||
assert all(r["mode"] == "0444" for r in records)
|
||||
# Only this tiny trusted fixture executes. An attempted source compilation
|
||||
# fails, so success demonstrates CPython actually consumes the checked cache.
|
||||
code = """import builtins,sys
|
||||
sys.path.insert(0,sys.argv[1])
|
||||
def refuse(*args,**kwargs): raise AssertionError('unexpected source compilation')
|
||||
builtins.compile=refuse
|
||||
import toy
|
||||
assert toy.VALUE==(17,'ordinary trusted fixture')
|
||||
"""
|
||||
before = {p: bytecode._stamp(p.lstat()) for p in (source, cache)}
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-I", "-B", "-c", code, str(source.parent)],
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert {p: bytecode._stamp(p.lstat()) for p in before} == before
|
||||
|
||||
|
||||
def test_source_change_cannot_reuse_cache_even_with_restored_mtime(compiled):
|
||||
payload, source, _ = compiled
|
||||
before = source.stat()
|
||||
source.chmod(0o600)
|
||||
source.write_text("VALUE = (18, 'ordinary trusted fixture')\n")
|
||||
source.chmod(0o444)
|
||||
os.utime(source, ns=(before.st_atime_ns, before.st_mtime_ns))
|
||||
with pytest.raises(policy.DeliveryError, match="bytecode source hash differs"):
|
||||
manifest.inventory(payload)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offset,value", [(0, 0), (4, 0), (4, 1), (4, 2)])
|
||||
def test_wrong_magic_timestamp_and_unchecked_hash_caches_are_refused(compiled, offset, value):
|
||||
payload, _, cache = compiled
|
||||
data = bytearray(cache.read_bytes())
|
||||
data[offset] = value
|
||||
cache.chmod(0o600)
|
||||
cache.write_bytes(data)
|
||||
cache.chmod(0o444)
|
||||
with pytest.raises(policy.DeliveryError, match="only pinned checked-hash"):
|
||||
manifest.inventory(payload)
|
||||
|
||||
|
||||
def test_builder_refuses_inherited_cache_and_does_not_write_to_external_prefix(
|
||||
compiled, monkeypatch
|
||||
):
|
||||
payload, _, cache = compiled
|
||||
before = cache.read_bytes()
|
||||
monkeypatch.setattr(sys, "pycache_prefix", str(payload / "outside"))
|
||||
with pytest.raises(policy.DeliveryError, match="inherited bytecode cache"):
|
||||
bytecode.compile_payload(payload, "sd-" + "b" * 12)
|
||||
assert cache.read_bytes() == before and not (payload / "outside").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"site/toy.pyc",
|
||||
"site/__pycache__/toy.cpython-314.pyc",
|
||||
"site/__pycache__/toy.cpython-313.opt-1.pyc",
|
||||
"site/__pycache__/toy.py",
|
||||
"site/__pycache__/nested/__pycache__/toy.cpython-313.pyc",
|
||||
"site/../__pycache__/toy.cpython-313.pyc",
|
||||
"images/python313/__pycache__/toy.cpython-313.pyc",
|
||||
"frontend/__pycache__/toy.cpython-313.pyc",
|
||||
],
|
||||
)
|
||||
def test_cache_allowlist_keeps_wrong_versions_domains_and_paths_closed(name):
|
||||
with pytest.raises(policy.DeliveryError):
|
||||
policy.relative_name(name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", policy.BYTECODE_ROOTS)
|
||||
def test_only_source_adjacent_pinned_cache_paths_are_allowed(prefix):
|
||||
name = prefix + "package/__pycache__/toy.cpython-313.pyc"
|
||||
assert policy.relative_name(name) == name
|
||||
assert policy.bytecode_source(name) == prefix + "package/toy.py"
|
||||
directory = prefix + "package/__pycache__/.__directory__"
|
||||
assert policy.relative_name(directory) == directory
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user